attempted speedup, explicit dates wallet objects

core/models/survex.py

@@ -172,6 +172,7 @@ class Wallet(models.Model):
     fpath               = models.CharField(max_length=200)
     walletname          = models.CharField(max_length=200)
     walletdate          = models.DateField(blank=True, null=True)
+    walletyear          = models.DateField(blank=True, null=True)
    
     class Meta:
         ordering = ('walletname',)
@@ -213,6 +214,8 @@ class Wallet(models.Model):
                         print(f' - {datestr=} ')
                         try:
                             thisdate = datetime.date.fromisoformat(datestr)
+                            self.walletdate =  thisdate
+                            self.save()
                             try:
                                 waldata["date"] = thisdate.isoformat()
                             except:
@@ -220,20 +223,24 @@ class Wallet(models.Model):
                                 from troggle.core.models.troggle import DataIssue
                                 DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) 
                         except:
-                            message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load fro, {jsonfile} JSON file"
+                            message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
                             from troggle.core.models.troggle import DataIssue
                             DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) 
         return waldata
         
     def year(self): 
+        '''This gets the year syntactically without opening and reading the JSON
+        '''
         if len(self.walletname) < 5:
              return None       
         if self.walletname[4] != "#":
              return None
         year = int(self.walletname[0:4])
-        if year < 1976 or year > 2050:
+        if year < 1975 or year > 2050:
             return None  
         else:
+            self.walletyear =  datetime.date(year, 1, 1)
+            self.save()
             return str(year)
         
 
@@ -243,7 +250,7 @@ class Wallet(models.Model):
             return self.walletdate
         if not self.get_json():
             return None
-        jsondata = self.get_json()
+        jsondata = self.get_json() # use walrus operator?
 
         datestr = jsondata["date"]
         if not datestr:
@@ -252,12 +259,13 @@ class Wallet(models.Model):
             datestr = datestr.replace('.','-')    
             try:
                 samedate = datetime.date.fromisoformat(datestr)
+                self.walletdate = samedate.isoformat()
             except:
                 try:
                     samedate = datetime.date.fromisoformat(datestr[:10])
+                    self.walletdate = samedate.isoformat()
                 except:
                     samedate = None
-            self.walletdate = samedate.isoformat()
             self.save()
             return self.walletdate
         

core/views/scans.py

@@ -8,6 +8,7 @@ from urllib.request import urlopen
 from django.conf import settings
 from django.shortcuts import render
 from django.http import HttpResponse
+from django.db import transaction
 
 from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock
 from troggle.core.models.troggle import Person, Expedition
@@ -48,6 +49,9 @@ def populatewallet(w):
         w.persons = list(set(survexpeople)) 
 
 def datewallet(w, earliest):
+    '''Gets the date of the youngest survexblock associated with the wallet
+    REFACTOR this to do the whole date-getting task
+    '''
     first = earliest
     blocks = SurvexBlock.objects.filter(scanswallet = w)
     for b in blocks:
@@ -59,6 +63,7 @@ def datewallet(w, earliest):
         w.date = None
     else:
         w.date = first.isoformat()
+    return w.date
     
 def caveifywallet(w):
     '''Gets the cave from the list of survex files,
@@ -94,10 +99,13 @@ def fillblankpeople(w):
                     # print(f' - {wp=} {nobody=}')
                     populatewallet(w)
         
-def fillblankothers(w):       
+def fillblankothers(w):
-    earliest = datetime.datetime.now().date()
+    if not w.walletdate:
-    if not w.date():
+        earliest = datetime.datetime.now().date()
-        datewallet(w, earliest)
+        if not w.date(): # sets .walletdate as a side-effect, gets it from JSON
+            d =datewallet(w, earliest) # if nothing in JASON, it looks at the survex blocks
+            w.walletdate = d
+            w.save()
         
     Gcavelookup = GetCaveLookup()
 
@@ -140,6 +148,8 @@ def walletslistperson(request, first_name, last_name):
                     w.ticks = w.get_ticks() # the complaints in colour form
                     fixsurvextick(w, w.ticks)
         return manywallets
+        
+    print(f"-walletslistperson")    
     
     try:
         if last_name:
@@ -153,37 +163,43 @@ def walletslistperson(request, first_name, last_name):
     
     manywallets = tickspersonwallet(p)
     expeditions = Expedition.objects.all()
+    print(f"--")    
     return render(request, 'personwallets.html', { 'manywallets':manywallets, 'settings': settings, 'person': p, 'expeditions': expeditions})
 
+def setwalletsdates():
+    wallets = Wallet.objects.filter(walletdate=None) 
+    print(f"undated wallets: {len(wallets)}")
+    for w in wallets:
+        w.walletdate = w.date()
+        w.save()
+
 
 def walletslistyear(request, year):
-    '''Page which displays a list of all the wallets in a specific year 
+    '''Page which displays a list of all the wallets in a specific year.
+    We have a field .walletyear, which we set on import.
     '''
     def ticksyearwallet(year):
         manywallets = []
-        wallets = Wallet.objects.all() 
+        wallets = Wallet.objects.filter(walletyear__year=year) 
         for w in wallets:
-
+            manywallets.append(w)
-            if year == w.year():
+            fillblankpeople(w)
-                manywallets.append(w)
+            fillblankothers(w)
-                fillblankpeople(w)
+            w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
-                fillblankothers(w)
+            fixsurvextick(w, w.ticks)
-                w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
-                fixsurvextick(w, w.ticks)
-            else:
-                continue
 
         return manywallets
-        
+    print(f"-walletslistyear")    
     if year < 1976 or year > 2050:
         return render(request, 'errors/generic.html', {'message': 'Year out of range. Must be between 1976 and 2050'})   
-    else:
+    
-        year = str(year)
     #return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'})   
-
+    
+    year = str(year)
     manywallets = ticksyearwallet(year)
     expeditions = Expedition.objects.all()
-    expedition = Expedition.objects.filter(year=year)
+    expedition = expeditions.filter(year=year)
+    print(f"--")    
     return render(request, 'yearwallets.html', { 'manywallets':manywallets, 'settings': settings, 'year': year, 'expeditions': expeditions, 'expedition': expedition})
 
 
@@ -191,6 +207,8 @@ def walletslistyear(request, year):
 def cavewallets(request, caveid):
     '''Returns all the wallets for just one cave
     '''
+    print(f"-cavewalletsl")    
+
     Gcavelookup = GetCaveLookup()
     if caveid in Gcavelookup:
         cave = Gcavelookup[caveid]
@@ -224,6 +242,7 @@ def cavewallets(request, caveid):
         w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
         fixsurvextick(w, w.ticks)
     expeditions = Expedition.objects.all()
+    print(f"--")    
     return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave, 'expeditions': expeditions})
 
 

parsers/imports.py

@@ -41,7 +41,7 @@ def import_logbooks():
     with transaction.atomic():
         troggle.parsers.logbooks.LoadLogbooks()
 
-def import_logbook(year=2018):
+def import_logbook(year=2019):
     print(f"-- Importing Logbook {year}")
     with transaction.atomic():
         troggle.parsers.logbooks.LoadLogbook(year)

parsers/scans.py

@@ -26,7 +26,9 @@ git = settings.GIT
 
 # to do: Actually read all the JSON files and set the survex file field appropriately!
 
-    
+def setwalletyear(wallet):
+    _ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear
+
 def load_all_scans():
     '''This iterates through the scans directories (either here or on the remote server)
     and builds up the models we can access later.
@@ -85,7 +87,9 @@ def load_all_scans():
                     wallet = wallets[walletname]
                 else:
                     print("", flush=True, end='')
+                    # Create the wallet object. But we don't have a date for it yet.
                     wallet = Wallet(fpath=fpath, walletname=walletname)
+                    setwalletyear(wallet)
                     wallet.save()
                     wallets[walletname] = wallet
                 
@@ -110,7 +114,7 @@ def load_all_scans():
      
     # but we also need to check if JSON exists, even if there are no uploaded scan files.
     # Here we know there is a rigid folder structure, so no need to look for sub folders
-    print(f"\n  - Checking for wallets where only JSON exists, but there are no actual uploaded scan files:")
+    print(f"\n  - Checking for wallets where JSON exists, but there may be no uploaded scan files:")
     print('   ', end='')
     wjson = 0
     contents_path = Path(settings.DRAWINGS_DATA, "walletjson") 
@@ -127,10 +131,18 @@ def load_all_scans():
 
                         print(f"{walletname} ", end='')
                         fpath = Path(settings.SCANS_ROOT, str(yeardir.stem), walletname) 
+                        # The wallets found from JSON should all have dates already
                         wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
                         wallets[walletname] = wallet
-                        # could now also load the json and use it. check &ref is correct or missing too..
+                        # could now also load the json but we don't. Do later, on-demand
+                        # wallet.walletdate = wallet.date()
+                        # could check if link to svx file is valid too.. but do on-demand later
+                        # But we *do* set the walletyear:
+                        setwalletyear(wallet)
                         if not created:
                             print(f"\n  - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?")
                             wallet.save()
     print(f'\n  - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets')
+    wallets = Wallet.objects.filter(walletyear=None)
+    for w in wallets:
+        w.walletyear = datetime.date(1999, 1, 1)

parsers/survex.py

@@ -660,6 +660,8 @@ class LoadingSurvex():
             else:
                 survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
                 survexblock.save()
+                # This is where we should check that the wallet JSON contains a link to the survexfile
+                # and that the JSON date and walletdate are set correctly to the survexblock date.
         else:
             perps = get_people_on_trip(survexblock)
             message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."

templates/cavewallets.html

@@ -28,7 +28,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
   <tr>
     <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
     
-    <td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
+    <td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
     <td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
     <td style="padding:2px">{{wallet.persons }} </td>
     

templates/personwallets.html

@@ -27,7 +27,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
   <tr>
     <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
     
-    <td style="padding:2px" >{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
+    <td style="padding:2px" >{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
     <td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
     <td style="padding:2px">{{wallet.persons}}</td>
     <td style="padding:2px">

templates/wallet_table.html

@@ -21,7 +21,7 @@
   <tr>
     <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
     
-    <td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
+    <td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
     <td style="padding:2px">
     {% if wallet.cave %}
         {% if wallet.caveobj.slug %}

templates/yearwallets.html

@@ -35,7 +35,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
   <tr>
     <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
     
-    <td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
+    <td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
     <td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
     <td style="padding:2px">{{wallet.persons}}</td>
     <td style="padding:2px">