attempted speedup, explicit dates wallet objects

parsers/imports.py

@@ -41,7 +41,7 @@ def import_logbooks():
     with transaction.atomic():
         troggle.parsers.logbooks.LoadLogbooks()
 
-def import_logbook(year=2018):
+def import_logbook(year=2019):
     print(f"-- Importing Logbook {year}")
     with transaction.atomic():
         troggle.parsers.logbooks.LoadLogbook(year)

parsers/scans.py

@@ -26,7 +26,9 @@ git = settings.GIT
 
 # to do: Actually read all the JSON files and set the survex file field appropriately!
 
-    
+def setwalletyear(wallet):
+    _ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear
+
 def load_all_scans():
     '''This iterates through the scans directories (either here or on the remote server)
     and builds up the models we can access later.
@@ -85,7 +87,9 @@ def load_all_scans():
                     wallet = wallets[walletname]
                 else:
                     print("", flush=True, end='')
+                    # Create the wallet object. But we don't have a date for it yet.
                     wallet = Wallet(fpath=fpath, walletname=walletname)
+                    setwalletyear(wallet)
                     wallet.save()
                     wallets[walletname] = wallet
                 
@@ -110,7 +114,7 @@ def load_all_scans():
      
     # but we also need to check if JSON exists, even if there are no uploaded scan files.
     # Here we know there is a rigid folder structure, so no need to look for sub folders
-    print(f"\n  - Checking for wallets where only JSON exists, but there are no actual uploaded scan files:")
+    print(f"\n  - Checking for wallets where JSON exists, but there may be no uploaded scan files:")
     print('   ', end='')
     wjson = 0
     contents_path = Path(settings.DRAWINGS_DATA, "walletjson") 
@@ -127,10 +131,18 @@ def load_all_scans():
 
                         print(f"{walletname} ", end='')
                         fpath = Path(settings.SCANS_ROOT, str(yeardir.stem), walletname) 
+                        # The wallets found from JSON should all have dates already
                         wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
                         wallets[walletname] = wallet
-                        # could now also load the json and use it. check &ref is correct or missing too..
+                        # could now also load the json but we don't. Do later, on-demand
+                        # wallet.walletdate = wallet.date()
+                        # could check if link to svx file is valid too.. but do on-demand later
+                        # But we *do* set the walletyear:
+                        setwalletyear(wallet)
                         if not created:
                             print(f"\n  - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?")
                             wallet.save()
     print(f'\n  - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets')
+    wallets = Wallet.objects.filter(walletyear=None)
+    for w in wallets:
+        w.walletyear = datetime.date(1999, 1, 1)

parsers/survex.py

@@ -660,6 +660,8 @@ class LoadingSurvex():
             else:
                 survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
                 survexblock.save()
+                # This is where we should check that the wallet JSON contains a link to the survexfile
+                # and that the JSON date and walletdate are set correctly to the survexblock date.
         else:
             perps = get_people_on_trip(survexblock)
             message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."