all working, queries improved, date-ordered.

core/views/logbooks.py

@@ -1,10 +1,10 @@
-
+from django.db.models import Q
 from django.shortcuts import render
 from django.views.generic.list import ListView
 
 import troggle.settings as settings
 from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry
-from troggle.core.models.survex import SurvexBlock
+from troggle.core.models.survex import SurvexBlock, SurvexFile
 from troggle.core.models.troggle import Expedition, Person
 from troggle.core.models.wallets import Wallet
 from troggle.core.utils import TROG
@@ -212,22 +212,13 @@ def logbookentry(request, date, slug):
 
     if this_logbookentry:
         if len(this_logbookentry) > 1:
+            # BUG
             return render(request, "object_list.html", {"object_list": this_logbookentry})
         else:
-            wallets = set()
-            allwallets = Wallet.objects.all()
-            refwallets = allwallets.filter(survexblock__date=date)
-            for r in refwallets:
-                wallets.add(r)
-
-            # Note that w.year() only works for wallets which have a valid JSON file existing
-            # This is very slow with a big lag as w.date() is a computed field
-            # Noticably slow with WSL2 and NTFS filesystem, even with caching as walletdate.
-            jwallets = allwallets.filter(walletdate=date)
-            for j in jwallets:
-                wallets.add(j)
-
-            svxothers = SurvexBlock.objects.filter(date=date)
+            # https://stackoverflow.com/questions/739776/how-do-i-do-an-or-filter-in-a-django-query
+            wallets = Wallet.objects.filter(Q(survexblock__date=date) | Q(walletdate=date)).distinct()
+            svxothers = SurvexFile.objects.filter(survexblock__date=date).distinct()
+            
             this_logbookentry = this_logbookentry[0]
             # This is the only page that uses next_.. and prev_..
             # and it is calculated on the fly in the model

core/views/survex.py

@@ -8,6 +8,7 @@ from collections import namedtuple
 
 from django import forms
 from django.db import models
+from django.db.models import Q
 
 from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
 from django.http import HttpResponse
@@ -257,7 +258,7 @@ def svx(request, survex_file):
         rform = SvxForm(request.POST)  #
         if rform.is_valid():  # All validation rules pass (how do we check it against the filename and users?)
             rcode = rform.cleaned_data["code"]
-            outputtype = rform.cleaned_data["outputtype"]
+            outputtype = rform.cleaned_data["outputtype"] # used by CodeMirror ajax I think
             difflist = form.DiffCode(rcode)
             # print(">>>> ", rform.data)
 
@@ -296,24 +297,16 @@ def svx(request, survex_file):
     svxincludes = re.findall(r"(?i)\*include\s+(\S+)", form.data["code"] or "")
 
     svxfile = form.survexfile # only valid once form.GetDiscCode() called
-    print(f"{svxfile=}")
     try:
         svxblocksall = svxfile.survexblock_set.all()
     except AttributeError: # some survexfiles just *include files and have no blocks themselves
         svxblocksall = []
-    svxblocks = []
-    for b in svxblocksall:
-        if b.date:
-            svxblocks.append(b)
-            print(f"- {b}")
     
-    # collect all the stuff that happens on the same dates as the survex blocks
+    # collect all the survex blocks which actually have a valid date
+    svxblocks = svxfile.survexblock_set.filter(date__isnull=False).order_by('date')
 
-    dates = set()
-    for b in svxblocks:
-        dates.add(b.date)
-        print(f"- {b.date}")
-    events = events_on_dates(dates)
+
+    events = events_on_dates(svxblocks)
 
     vmap = {
         "settings": settings,
@@ -329,30 +322,40 @@ def svx(request, survex_file):
     }
     # vmap.update(csrf(request)) # this now refreshes to the wrong value, now that we user render(request,
 
-    if outputtype == "ajax":
+    if outputtype == "ajax": # used by CodeMirror ajax I think
         return render(request, "svxfiledifflistonly.html", vmap)
 
     return render(request, "svxfile.html", vmap)
 
-SameDateEvents = namedtuple('SameDateEvents', ['trips', 'svxfiles', 'wallets'])
+SameDateEvents = namedtuple('SameDateEvents', ['trips', 'svxfiles', 'wallets', 'blocks'])
 
-def events_on_dates(dates):
+def events_on_dates(svxblocks):
     """Returns a dictionary of indexed by date. For each date there is a named tuple of 3 lists:
     logbookentries, survexfiles (NB files, not blocks), and wallets.
     """
+    # deduplicate but maintain date order
+    dates = [] 
+    for b in svxblocks:
+        if b.date not in dates:
+            dates.append(b.date)
+            # print(f"- {b.date}")
+            
     events = {}
-    for d in dates:
-        trips = LogbookEntry.objects.filter(date=d)
+    for date in dates:
+        trips = LogbookEntry.objects.filter(date=date)
         
-        svxfiles = False
+        svxfiles = SurvexFile.objects.filter(survexblock__date=date).distinct()
         
-        # Wallets needs to get those identified only from JSON too, 
-        # see logbookeentry() in views/logbooks.py 
-        allwallets = Wallet.objects.all()
-        refwallets = allwallets.filter(survexblock__date=d)
+        # https://stackoverflow.com/questions/739776/how-do-i-do-an-or-filter-in-a-django-query
+        wallets = Wallet.objects.filter(Q(survexblock__date=date) | Q(walletdate=date)).distinct()
         
-        events[d] = SameDateEvents(trips=trips, svxfiles=svxfiles, wallets=refwallets)
-    print(events)
+        blocks = []
+        for b in svxblocks:
+            if b.date == date:
+                blocks.append(b.name)
+        
+        events[date] = SameDateEvents(trips=trips, svxfiles=svxfiles, wallets=wallets, blocks=blocks)
+    # print(events)
     return events
 
 # The cavern running function. This is NOT where it is run inside the form! see SvxForm.Process() for that

core/views/wallets_edit.py

@@ -828,8 +828,6 @@ def walletedit(request, path=None):
     if 'notes not required' not in waldata: # cope with schema change
         waldata['notes not required'] = False
 
-    # for a in waldata:
-        # print(f"'{waldata[a]}' {a}")
     # find trips and survex files of the same date
     walletobject = make_wallet(wallet)
     if waldata["date"]:
@@ -845,7 +843,7 @@ def walletedit(request, path=None):
             print(message)
             return render(request, "errors/generic.html", {"message": message})
         if samedate:
-            svxothers = SurvexBlock.objects.filter(date=samedate)
+            svxothers = SurvexFile.objects.filter(survexblock__date=samedate).distinct()
             trips = LogbookEntry.objects.filter(date=samedate)
         else:
             svxothers = None