all working, queries improved, date-ordered.

core/views/logbooks.py

@@ -1,10 +1,10 @@
-
+from django.db.models import Q
 from django.shortcuts import render
 from django.views.generic.list import ListView
 
 import troggle.settings as settings
 from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry
-from troggle.core.models.survex import SurvexBlock
+from troggle.core.models.survex import SurvexBlock, SurvexFile
 from troggle.core.models.troggle import Expedition, Person
 from troggle.core.models.wallets import Wallet
 from troggle.core.utils import TROG
@@ -212,22 +212,13 @@ def logbookentry(request, date, slug):
 
     if this_logbookentry:
         if len(this_logbookentry) > 1:
+            # BUG
             return render(request, "object_list.html", {"object_list": this_logbookentry})
         else:
-            wallets = set()
+            # https://stackoverflow.com/questions/739776/how-do-i-do-an-or-filter-in-a-django-query
-            allwallets = Wallet.objects.all()
+            wallets = Wallet.objects.filter(Q(survexblock__date=date) | Q(walletdate=date)).distinct()
-            refwallets = allwallets.filter(survexblock__date=date)
+            svxothers = SurvexFile.objects.filter(survexblock__date=date).distinct()
-            for r in refwallets:
-                wallets.add(r)
             
-            # Note that w.year() only works for wallets which have a valid JSON file existing
-            # This is very slow with a big lag as w.date() is a computed field
-            # Noticably slow with WSL2 and NTFS filesystem, even with caching as walletdate.
-            jwallets = allwallets.filter(walletdate=date)
-            for j in jwallets:
-                wallets.add(j)
-
-            svxothers = SurvexBlock.objects.filter(date=date)
             this_logbookentry = this_logbookentry[0]
             # This is the only page that uses next_.. and prev_..
             # and it is calculated on the fly in the model

core/views/survex.py

@@ -8,6 +8,7 @@ from collections import namedtuple
 
 from django import forms
 from django.db import models
+from django.db.models import Q
 
 from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
 from django.http import HttpResponse
@@ -257,7 +258,7 @@ def svx(request, survex_file):
         rform = SvxForm(request.POST)  #
         if rform.is_valid():  # All validation rules pass (how do we check it against the filename and users?)
             rcode = rform.cleaned_data["code"]
-            outputtype = rform.cleaned_data["outputtype"]
+            outputtype = rform.cleaned_data["outputtype"] # used by CodeMirror ajax I think
             difflist = form.DiffCode(rcode)
             # print(">>>> ", rform.data)
 
@@ -296,24 +297,16 @@ def svx(request, survex_file):
     svxincludes = re.findall(r"(?i)\*include\s+(\S+)", form.data["code"] or "")
 
     svxfile = form.survexfile # only valid once form.GetDiscCode() called
-    print(f"{svxfile=}")
     try:
         svxblocksall = svxfile.survexblock_set.all()
     except AttributeError: # some survexfiles just *include files and have no blocks themselves
         svxblocksall = []
-    svxblocks = []
-    for b in svxblocksall:
-        if b.date:
-            svxblocks.append(b)
-            print(f"- {b}")
     
-    # collect all the stuff that happens on the same dates as the survex blocks
+    # collect all the survex blocks which actually have a valid date
+    svxblocks = svxfile.survexblock_set.filter(date__isnull=False).order_by('date')
 
-    dates = set()
+
-    for b in svxblocks:
+    events = events_on_dates(svxblocks)
-        dates.add(b.date)
-        print(f"- {b.date}")
-    events = events_on_dates(dates)
 
     vmap = {
         "settings": settings,
@@ -329,30 +322,40 @@ def svx(request, survex_file):
     }
     # vmap.update(csrf(request)) # this now refreshes to the wrong value, now that we user render(request,
 
-    if outputtype == "ajax":
+    if outputtype == "ajax": # used by CodeMirror ajax I think
         return render(request, "svxfiledifflistonly.html", vmap)
 
     return render(request, "svxfile.html", vmap)
 
-SameDateEvents = namedtuple('SameDateEvents', ['trips', 'svxfiles', 'wallets'])
+SameDateEvents = namedtuple('SameDateEvents', ['trips', 'svxfiles', 'wallets', 'blocks'])
 
-def events_on_dates(dates):
+def events_on_dates(svxblocks):
     """Returns a dictionary of indexed by date. For each date there is a named tuple of 3 lists:
     logbookentries, survexfiles (NB files, not blocks), and wallets.
     """
+    # deduplicate but maintain date order
+    dates = [] 
+    for b in svxblocks:
+        if b.date not in dates:
+            dates.append(b.date)
+            # print(f"- {b.date}")
+            
     events = {}
-    for d in dates:
+    for date in dates:
-        trips = LogbookEntry.objects.filter(date=d)
+        trips = LogbookEntry.objects.filter(date=date)
         
-        svxfiles = False
+        svxfiles = SurvexFile.objects.filter(survexblock__date=date).distinct()
         
-        # Wallets needs to get those identified only from JSON too, 
+        # https://stackoverflow.com/questions/739776/how-do-i-do-an-or-filter-in-a-django-query
-        # see logbookeentry() in views/logbooks.py 
+        wallets = Wallet.objects.filter(Q(survexblock__date=date) | Q(walletdate=date)).distinct()
-        allwallets = Wallet.objects.all()
-        refwallets = allwallets.filter(survexblock__date=d)
         
-        events[d] = SameDateEvents(trips=trips, svxfiles=svxfiles, wallets=refwallets)
+        blocks = []
-    print(events)
+        for b in svxblocks:
+            if b.date == date:
+                blocks.append(b.name)
+        
+        events[date] = SameDateEvents(trips=trips, svxfiles=svxfiles, wallets=wallets, blocks=blocks)
+    # print(events)
     return events
 
 # The cavern running function. This is NOT where it is run inside the form! see SvxForm.Process() for that

core/views/wallets_edit.py

@@ -828,8 +828,6 @@ def walletedit(request, path=None):
     if 'notes not required' not in waldata: # cope with schema change
         waldata['notes not required'] = False
 
-    # for a in waldata:
-        # print(f"'{waldata[a]}' {a}")
     # find trips and survex files of the same date
     walletobject = make_wallet(wallet)
     if waldata["date"]:
@@ -845,7 +843,7 @@ def walletedit(request, path=None):
             print(message)
             return render(request, "errors/generic.html", {"message": message})
         if samedate:
-            svxothers = SurvexBlock.objects.filter(date=samedate)
+            svxothers = SurvexFile.objects.filter(survexblock__date=samedate).distinct()
             trips = LogbookEntry.objects.filter(date=samedate)
         else:
             svxothers = None

templates/onthisdate.html

@@ -3,7 +3,7 @@
         {% if svxothers %}<u>Survex files</u> on this date:<br>
                 <span style="font-size: 70%; ">
                 {% for item in svxothers %}
-                {% if item.isSurvexBlock %}&nbsp;&nbsp;&nbsp;&nbsp;<a href="/survexfile/{{item.survexfile.path}}">{{item.survexfile.path|safe}}</a><br/>{% endif %}
+                &nbsp;&nbsp;&nbsp;&nbsp;<a href="/survexfile/{{item.path}}">{{item.path|safe}}</a><br/>
                 {% empty %}
                 <em>None found for this date.</em><br>
                 {% endfor %}
@@ -16,11 +16,8 @@
                 {% for item in wallets %}
                     &nbsp;&nbsp;&nbsp;&nbsp;<a href="/survey_scans/{{item.walletname|urlencode}}/">{{item.walletname|safe}}</a>
                     {% if item.name %}
-                        {{item.name|safe}} <br/>
+                        {{item.name|safe}}                        
-                    {% else %}
+                    {% endif %}{{item.get_fnames|truncatechars:80}}<br/>
-                        {{item.get_fnames|safe}}
-                        <br/>
-                    {% endif %}
                 {% empty %}
                     <em>None found for this date.</em><br>
                 {% endfor %}
@@ -28,7 +25,7 @@
         {% else %}
             <em>No wallets files found for this date.</em><br>
         {% endif %}
-         {% if trips %}<u>All logbook trips</u> on this date:<br>
+         {% if trips %}<u>Logbook trips</u> on this date:<br>
                 <span style="font-size: 70%; ">
                 {% for item in trips %}
                 {% if item.isLogbookEntry %}&nbsp;&nbsp;&nbsp;&nbsp;<a href="{{item.get_absolute_url}}">{{item.title|safe}}</a><br/>{% endif %}

templates/svxfile.html

@@ -83,16 +83,15 @@ LOGMESSAGES
 <span style="font-family: monospace;  font-size: 130%; ">
 {% for sb in svxblocks %}
 {% empty %} 
-Cannot find any survex blocks in this survex file (not looking at *include files). <br />
+Cannot find any <em>dated</em> survex blocks in this survex file (not looking at *include files). <br />
 Report this to a nerd if you think this is incorrect.
 <hr />
 {% endfor %}
 
 {% for key, value in events.items %}
 <details {% if forloop.first %} open{% endif %}>
-<summary><b><a href="/expedition/{{key|date:"Y"}}">{{key|date:"Y"}}</a>{{key|date:"-m-d"}}</b> </summary>
+<summary><b><a href="/expedition/{{key|date:"Y"}}">{{key|date:"Y"}}</a>{{key|date:"-m-d"}}</b> {{value.3}}</summary>
-<br />
+{% with trips=value.0 svxothers=value.1 wallets=value.2 %}
-{% with trips=value.0 svxfiles=value.1 wallets=value.2 %}
  {% include 'onthisdate.html' %}
 {% endwith %}
 </details>

templates/walletform.html

@@ -228,6 +228,7 @@
             </button>{% endif %}
         </form>
     </div>
+    
     <span style="font-family: monospace;  font-size: 150%; ">
 
         {% if trips %}<u>Logbook trips</u> on this date:<br>
@@ -241,17 +242,18 @@
     {% else %}
         <em>No Logbook trips found for this date.</em><br>
     {% endif %}
-    {% if svxothers %}<u>Other survex files</u> on this date:<br>
+    
-            <span style="font-size: 70%; ">
+        {% if svxothers %}<u>Survex files</u> on this date:<br>
-            {% for item in svxothers %}
+                <span style="font-size: 70%; ">
-            {% if item.isSurvexBlock %}&nbsp;&nbsp;&nbsp;&nbsp;<a href="/survexfile/{{item.survexfile.path}}">{{item.survexfile.path|safe}}</a><br/>{% endif %}
+                {% for item in svxothers %}
-            {% empty %}
+                &nbsp;&nbsp;&nbsp;&nbsp;<a href="/survexfile/{{item.path}}">{{item.path|safe}}</a><br/>
-            <em>None found for this date, but there should be..</em><br>
+                {% empty %}
-            {% endfor %}
+                <em>None found for this date.</em><br>
-            </span>
+                {% endfor %}
-    {% else %}
+                </span>
-        <em>No other survex files found for this date.</em><br>
+        {% else %}
-    {% endif %}
+            <em>No survex files found for this date.</em><br>
+        {% endif %}    
    
     {% if metadataurl %}<span style="font-size: 70%; "><details><summary>
     JSON  <br>