From 9d1c0ac395a5509d577cb030148e335b1ee2b5c8 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Wed, 1 Feb 2023 23:43:05 +0000 Subject: [PATCH] Setting wallet dates earlier in the import process --- core/models/wallets.py | 2 +- core/views/scans.py | 31 ++++++------------------------- core/views/wallets_edit.py | 1 + parsers/scans.py | 21 +++++++++++++-------- parsers/survex.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 34 deletions(-) diff --git a/core/models/wallets.py b/core/models/wallets.py index 7c0dea4..a90a737 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -99,7 +99,7 @@ class Wallet(models.Model): # Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it def date(self): - """Reads all the JSON data just to get the JSNON date.""" + """Reads all the JSON data just to get the JSON date.""" if self.walletdate: return self.walletdate if not self.get_json(): diff --git a/core/views/scans.py b/core/views/scans.py index 63b477d..aed38ee 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -11,9 +11,9 @@ from troggle.core.models.survex import SingleScan, SurvexBlock from troggle.core.models.wallets import Wallet from troggle.core.models.troggle import DataIssue, Expedition, Person from troggle.core.views.expo import getmimetype +from troggle.parsers.survex import set_walletdate # from troggle.parsers.people import GetPersonExpeditionNameLookup - # import parsers.surveys """one of these views serves files as binary blobs, and simply set the mime type based on the file extension, @@ -23,7 +23,7 @@ by looking inside the file before being served. need to check if inavlid query string is invalid, or produces multiple replies and render a user-friendly error page. -Note that datewallet(), caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the +Note that caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the manywallets dict. TODO @@ -47,24 +47,7 @@ def populatewallet(w): w.persons = list(set(survexpeople)) -def datewallet(w, earliest): - """Gets the date of the youngest survexblock associated with the wallet - REFACTOR this to do the whole date-getting task - """ - first = earliest - blocks = SurvexBlock.objects.filter(scanswallet=w) - for b in blocks: - if b.date: - if b.date < first: - first = b.date - if first == earliest: - # no date found - w.date = None - else: - w.date = first.isoformat() - return w.date - - + def caveifywallet(w): """Gets the cave from the list of survex files, only selects one of them though. Only used for display. @@ -105,11 +88,7 @@ def fillblankpeople(w): def fillblankothers(w): if not w.walletdate: - earliest = datetime.datetime.now().date() - if not w.date(): # sets .walletdate as a side-effect, gets it from JSON - d = datewallet(w, earliest) # if nothing in JASON, it looks at the survex blocks - w.walletdate = d - w.save() + set_walletdate(w) Gcavelookup = GetCaveLookup() @@ -180,6 +159,8 @@ def walletslistperson(request, first_name, last_name): def setwalletsdates(): + """This sets all the undated wallets, but they should already all be dated on + import or on edit""" wallets = Wallet.objects.filter(walletdate=None) print(f"undated wallets: {len(wallets)}") for w in wallets: diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py index a58aee8..1484639 100644 --- a/core/views/wallets_edit.py +++ b/core/views/wallets_edit.py @@ -842,6 +842,7 @@ def walletedit(request, path=None): }, ) else: # no wallet data: should never happen as there should be default data in all cases + print(f"! - Should never get here ! Problem in wallet editing...") context = { "year": year, "prev": prev, diff --git a/parsers/scans.py b/parsers/scans.py index 00538fe..3f042da 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -6,7 +6,7 @@ from troggle.core.models.survex import SingleScan from troggle.core.models.troggle import DataIssue from troggle.core.models.wallets import Wallet -"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. +"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. Loads all the wallets . """ contentsjson = "contents.json" @@ -16,9 +16,13 @@ git = settings.GIT # to do: Actually read all the JSON files and set the survex file field appropriately! -def setwalletyear(wallet): +def set_walletyear(wallet): _ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear +def set_JSONwalletdate(wallet): + """At this point in the import process, the survex files have not been imported so + we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here.""" + _ = wallet.date() # don't need return value. Sets .walletdate as side effect def load_all_scans(): """This iterates through the scans directories (either here or on the remote server) @@ -104,7 +108,7 @@ def load_all_scans(): print("", flush=True, end="") # Create the wallet object. But we don't have a date for it yet. wallet = Wallet(fpath=fpath, walletname=walletname) - setwalletyear(wallet) + set_walletyear(wallet) wallet.save() wallets[walletname] = wallet @@ -147,17 +151,18 @@ def load_all_scans(): # The wallets found from JSON should all have dates already wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath) wallets[walletname] = wallet - # could now also load the json but we don't. Do later, on-demand - # wallet.walletdate = wallet.date() - # could check if link to svx file is valid too.. but do on-demand later - # But we *do* set the walletyear: - setwalletyear(wallet) + # Now also load the json + # BUT can't check linked survex blocks as they haven't been imported yet + set_JSONwalletdate(wallet) + set_walletyear(wallet) if not created: print( f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?" ) wallet.save() print(f"\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets") + + # Only the 1999 wallets have filenames which mean that the walletyear will be unset: wallets = Wallet.objects.filter(walletyear=None) for w in wallets: w.walletyear = datetime.date(1999, 1, 1) diff --git a/parsers/survex.py b/parsers/survex.py index d977296..8bde946 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -55,6 +55,33 @@ class SurvexLeg: compass = 0.0 clino = 0.0 +def datewallet(w, earliest): + """Gets the date of the youngest survexblock associated with the wallet + REFACTOR this to do the whole date-getting task + + Currently there is only one SurvexBlock, but this is in anticipation of + chnaging the schema to allow many. + """ + first = earliest + blocks = SurvexBlock.objects.filter(scanswallet=w) # only ONE I think ?! + for b in blocks: + if b.date: + if b.date < first: + first = b.date + if first == earliest: + # no date found + w.date = None + else: + w.date = first.isoformat() + return w.date + +def set_walletdate(w): + earliest = datetime.now().date() + if not w.date(): # sets .walletdate as a side-effect if it gets it from JSON + d = datewallet(w, earliest) # Not in JSON, so checks all the survex blocks + w.walletdate = d + w.save() + def stash_data_issue(parser=None, message=None, url=None, sb=None): """Avoid hitting the database for error messages until the end of the import""" global dataissues @@ -848,6 +875,7 @@ class LoadingSurvex: survexblock.save() # This is where we should check that the wallet JSON contains a link to the survexfile # and that the JSON date and walletdate are set correctly to the survexblock date. + set_walletdate(survexblock.scanswallet) else: perps = get_people_on_trip(survexblock) message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."