Setting wallet dates earlier in the import process

core/models/wallets.py

@@ -99,7 +99,7 @@ class Wallet(models.Model):
 
     # Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it
     def date(self):
-        """Reads all the JSON data just to get the JSNON date."""
+        """Reads all the JSON data just to get the JSON date."""
         if self.walletdate:
             return self.walletdate
         if not self.get_json():

core/views/scans.py

@@ -11,9 +11,9 @@ from troggle.core.models.survex import SingleScan, SurvexBlock
 from troggle.core.models.wallets import Wallet
 from troggle.core.models.troggle import DataIssue, Expedition, Person
 from troggle.core.views.expo import getmimetype
+from troggle.parsers.survex import set_walletdate
 
 # from troggle.parsers.people import GetPersonExpeditionNameLookup
-
 # import parsers.surveys
 
 """one of these views serves files as binary blobs, and simply set the mime type based on the file extension,
@@ -23,7 +23,7 @@ by looking inside the file before being served.
 need to check if inavlid query string is invalid, or produces multiple replies
 and render a user-friendly error page.
 
-Note that datewallet(), caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the
+Note that caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the
 manywallets dict.
 
 TODO 
@@ -47,23 +47,6 @@ def populatewallet(w):
         w.persons = list(set(survexpeople))
 
 
-def datewallet(w, earliest):
-    """Gets the date of the youngest survexblock associated with the wallet
-    REFACTOR this to do the whole date-getting task
-    """
-    first = earliest
-    blocks = SurvexBlock.objects.filter(scanswallet=w)
-    for b in blocks:
-        if b.date:
-            if b.date < first:
-                first = b.date
-    if first == earliest:
-        # no date found
-        w.date = None
-    else:
-        w.date = first.isoformat()
-    return w.date
-
         
 def caveifywallet(w):
     """Gets the cave from the list of survex files,
@@ -105,11 +88,7 @@ def fillblankpeople(w):
 
 def fillblankothers(w):
     if not w.walletdate:
-        earliest = datetime.datetime.now().date()
+        set_walletdate(w)
-        if not w.date():  # sets .walletdate as a side-effect, gets it from JSON
-            d = datewallet(w, earliest)  # if nothing in JASON, it looks at the survex blocks
-            w.walletdate = d
-            w.save()
 
     Gcavelookup = GetCaveLookup()
 
@@ -180,6 +159,8 @@ def walletslistperson(request, first_name, last_name):
 
 
 def setwalletsdates():
+    """This sets all the undated wallets, but they should already all be dated on
+    import or on edit"""
     wallets = Wallet.objects.filter(walletdate=None)
     print(f"undated wallets: {len(wallets)}")
     for w in wallets:

core/views/wallets_edit.py

@@ -842,6 +842,7 @@ def walletedit(request, path=None):
             },
         )
     else:  # no wallet data: should never happen as there should be default data in all cases
+        print(f"! - Should never get here ! Problem in wallet editing...")
         context = {
             "year": year,
             "prev": prev,

parsers/scans.py

@@ -6,7 +6,7 @@ from troggle.core.models.survex import SingleScan
 from troggle.core.models.troggle import DataIssue
 from troggle.core.models.wallets import Wallet
 
-"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
+"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. Loads all the wallets .
 """
 
 contentsjson = "contents.json"
@@ -16,9 +16,13 @@ git = settings.GIT
 # to do: Actually read all the JSON files and set the survex file field appropriately!
 
 
-def setwalletyear(wallet):
+def set_walletyear(wallet):
     _ = wallet.year()  # don't need return value. Just calling this saves it as w.walletyear
 
+def set_JSONwalletdate(wallet):
+    """At this point in the import process, the survex files have not been imported so
+    we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here."""
+    _ = wallet.date()  # don't need return value. Sets .walletdate as side effect
 
 def load_all_scans():
     """This iterates through the scans directories (either here or on the remote server)
@@ -104,7 +108,7 @@ def load_all_scans():
                     print("", flush=True, end="")
                     # Create the wallet object. But we don't have a date for it yet.
                     wallet = Wallet(fpath=fpath, walletname=walletname)
-                    setwalletyear(wallet)
+                    set_walletyear(wallet)
                     wallet.save()
                     wallets[walletname] = wallet
 
@@ -147,17 +151,18 @@ def load_all_scans():
                         # The wallets found from JSON should all have dates already
                         wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
                         wallets[walletname] = wallet
-                        # could now also load the json but we don't. Do later, on-demand
+                        # Now also load the json 
-                        # wallet.walletdate = wallet.date()
+                        # BUT can't check linked survex blocks as they haven't been imported yet
-                        # could check if link to svx file is valid too.. but do on-demand later
+                        set_JSONwalletdate(wallet)
-                        # But we *do* set the walletyear:
+                        set_walletyear(wallet)
-                        setwalletyear(wallet)
                         if not created:
                             print(
                                 f"\n  - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?"
                             )
                             wallet.save()
     print(f"\n  - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets")
+    
+    # Only the 1999 wallets have filenames which mean that the walletyear will be unset:
     wallets = Wallet.objects.filter(walletyear=None)
     for w in wallets:
         w.walletyear = datetime.date(1999, 1, 1)

parsers/survex.py

@@ -55,6 +55,33 @@ class SurvexLeg:
     compass = 0.0
     clino = 0.0
 
+def datewallet(w, earliest):
+    """Gets the date of the youngest survexblock associated with the wallet
+    REFACTOR this to do the whole date-getting task
+    
+    Currently there is only one SurvexBlock, but this is in anticipation of
+    chnaging the schema to allow many.
+    """
+    first = earliest
+    blocks = SurvexBlock.objects.filter(scanswallet=w) # only ONE I think ?!
+    for b in blocks:
+        if b.date:
+            if b.date < first:
+                first = b.date
+    if first == earliest:
+        # no date found
+        w.date = None
+    else:
+        w.date = first.isoformat()
+    return w.date
+
+def set_walletdate(w):
+    earliest = datetime.now().date()
+    if not w.date():  # sets .walletdate as a side-effect if it gets it from JSON
+        d = datewallet(w, earliest)  # Not in JSON, so checks all the survex blocks
+        w.walletdate = d
+        w.save()
+
 def stash_data_issue(parser=None, message=None, url=None, sb=None):
     """Avoid hitting the database for error messages until the end of the import"""
     global dataissues
@@ -848,6 +875,7 @@ class LoadingSurvex:
                 survexblock.save()
                 # This is where we should check that the wallet JSON contains a link to the survexfile
                 # and that the JSON date and walletdate are set correctly to the survexblock date.
+                set_walletdate(survexblock.scanswallet)
         else:
             perps = get_people_on_trip(survexblock)
             message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."