From 5c3927c25d51e907a176203431be3b23cdcfd8ae Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@gmail.com>
Date: Mon, 27 Feb 2023 22:23:24 +0000
Subject: [PATCH] all working, queries improved, date-ordered.
---
core/views/logbooks.py | 23 +++++------------
core/views/survex.py | 53 ++++++++++++++++++++------------------
core/views/wallets_edit.py | 4 +--
templates/onthisdate.html | 11 +++-----
templates/svxfile.html | 7 +++--
templates/walletform.html | 24 +++++++++--------
6 files changed, 56 insertions(+), 66 deletions(-)
diff --git a/core/views/logbooks.py b/core/views/logbooks.py
index 37d6aa9..626153c 100644
--- a/core/views/logbooks.py
+++ b/core/views/logbooks.py
@@ -1,10 +1,10 @@
-
+from django.db.models import Q
from django.shortcuts import render
from django.views.generic.list import ListView
import troggle.settings as settings
from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry
-from troggle.core.models.survex import SurvexBlock
+from troggle.core.models.survex import SurvexBlock, SurvexFile
from troggle.core.models.troggle import Expedition, Person
from troggle.core.models.wallets import Wallet
from troggle.core.utils import TROG
@@ -212,22 +212,13 @@ def logbookentry(request, date, slug):
if this_logbookentry:
if len(this_logbookentry) > 1:
+ # BUG
return render(request, "object_list.html", {"object_list": this_logbookentry})
else:
- wallets = set()
- allwallets = Wallet.objects.all()
- refwallets = allwallets.filter(survexblock__date=date)
- for r in refwallets:
- wallets.add(r)
-
- # Note that w.year() only works for wallets which have a valid JSON file existing
- # This is very slow with a big lag as w.date() is a computed field
- # Noticably slow with WSL2 and NTFS filesystem, even with caching as walletdate.
- jwallets = allwallets.filter(walletdate=date)
- for j in jwallets:
- wallets.add(j)
-
- svxothers = SurvexBlock.objects.filter(date=date)
+ # https://stackoverflow.com/questions/739776/how-do-i-do-an-or-filter-in-a-django-query
+ wallets = Wallet.objects.filter(Q(survexblock__date=date) | Q(walletdate=date)).distinct()
+ svxothers = SurvexFile.objects.filter(survexblock__date=date).distinct()
+
this_logbookentry = this_logbookentry[0]
# This is the only page that uses next_.. and prev_..
# and it is calculated on the fly in the model
diff --git a/core/views/survex.py b/core/views/survex.py
index 9507795..2a3c67d 100644
--- a/core/views/survex.py
+++ b/core/views/survex.py
@@ -8,6 +8,7 @@ from collections import namedtuple
from django import forms
from django.db import models
+from django.db.models import Q
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.http import HttpResponse
@@ -257,7 +258,7 @@ def svx(request, survex_file):
rform = SvxForm(request.POST) #
if rform.is_valid(): # All validation rules pass (how do we check it against the filename and users?)
rcode = rform.cleaned_data["code"]
- outputtype = rform.cleaned_data["outputtype"]
+ outputtype = rform.cleaned_data["outputtype"] # used by CodeMirror ajax I think
difflist = form.DiffCode(rcode)
# print(">>>> ", rform.data)
@@ -296,24 +297,16 @@ def svx(request, survex_file):
svxincludes = re.findall(r"(?i)\*include\s+(\S+)", form.data["code"] or "")
svxfile = form.survexfile # only valid once form.GetDiscCode() called
- print(f"{svxfile=}")
try:
svxblocksall = svxfile.survexblock_set.all()
except AttributeError: # some survexfiles just *include files and have no blocks themselves
svxblocksall = []
- svxblocks = []
- for b in svxblocksall:
- if b.date:
- svxblocks.append(b)
- print(f"- {b}")
- # collect all the stuff that happens on the same dates as the survex blocks
+ # collect all the survex blocks which actually have a valid date
+ svxblocks = svxfile.survexblock_set.filter(date__isnull=False).order_by('date')
- dates = set()
- for b in svxblocks:
- dates.add(b.date)
- print(f"- {b.date}")
- events = events_on_dates(dates)
+
+ events = events_on_dates(svxblocks)
vmap = {
"settings": settings,
@@ -329,30 +322,40 @@ def svx(request, survex_file):
}
# vmap.update(csrf(request)) # this now refreshes to the wrong value, now that we user render(request,
- if outputtype == "ajax":
+ if outputtype == "ajax": # used by CodeMirror ajax I think
return render(request, "svxfiledifflistonly.html", vmap)
return render(request, "svxfile.html", vmap)
-SameDateEvents = namedtuple('SameDateEvents', ['trips', 'svxfiles', 'wallets'])
+SameDateEvents = namedtuple('SameDateEvents', ['trips', 'svxfiles', 'wallets', 'blocks'])
-def events_on_dates(dates):
+def events_on_dates(svxblocks):
"""Returns a dictionary of indexed by date. For each date there is a named tuple of 3 lists:
logbookentries, survexfiles (NB files, not blocks), and wallets.
"""
+ # deduplicate but maintain date order
+ dates = []
+ for b in svxblocks:
+ if b.date not in dates:
+ dates.append(b.date)
+ # print(f"- {b.date}")
+
events = {}
- for d in dates:
- trips = LogbookEntry.objects.filter(date=d)
+ for date in dates:
+ trips = LogbookEntry.objects.filter(date=date)
- svxfiles = False
+ svxfiles = SurvexFile.objects.filter(survexblock__date=date).distinct()
- # Wallets needs to get those identified only from JSON too,
- # see logbookeentry() in views/logbooks.py
- allwallets = Wallet.objects.all()
- refwallets = allwallets.filter(survexblock__date=d)
+ # https://stackoverflow.com/questions/739776/how-do-i-do-an-or-filter-in-a-django-query
+ wallets = Wallet.objects.filter(Q(survexblock__date=date) | Q(walletdate=date)).distinct()
- events[d] = SameDateEvents(trips=trips, svxfiles=svxfiles, wallets=refwallets)
- print(events)
+ blocks = []
+ for b in svxblocks:
+ if b.date == date:
+ blocks.append(b.name)
+
+ events[date] = SameDateEvents(trips=trips, svxfiles=svxfiles, wallets=wallets, blocks=blocks)
+ # print(events)
return events
# The cavern running function. This is NOT where it is run inside the form! see SvxForm.Process() for that
diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py
index 7cc651a..9cb667e 100644
--- a/core/views/wallets_edit.py
+++ b/core/views/wallets_edit.py
@@ -828,8 +828,6 @@ def walletedit(request, path=None):
if 'notes not required' not in waldata: # cope with schema change
waldata['notes not required'] = False
- # for a in waldata:
- # print(f"'{waldata[a]}' {a}")
# find trips and survex files of the same date
walletobject = make_wallet(wallet)
if waldata["date"]:
@@ -845,7 +843,7 @@ def walletedit(request, path=None):
print(message)
return render(request, "errors/generic.html", {"message": message})
if samedate:
- svxothers = SurvexBlock.objects.filter(date=samedate)
+ svxothers = SurvexFile.objects.filter(survexblock__date=samedate).distinct()
trips = LogbookEntry.objects.filter(date=samedate)
else:
svxothers = None
diff --git a/templates/onthisdate.html b/templates/onthisdate.html
index dd75144..e2f44d1 100644
--- a/templates/onthisdate.html
+++ b/templates/onthisdate.html
@@ -3,7 +3,7 @@
{% if svxothers %}<u>Survex files</u> on this date:<br>
<span style="font-size: 70%; ">
{% for item in svxothers %}
- {% if item.isSurvexBlock %} <a href="/survexfile/{{item.survexfile.path}}">{{item.survexfile.path|safe}}</a><br/>{% endif %}
+ <a href="/survexfile/{{item.path}}">{{item.path|safe}}</a><br/>
{% empty %}
<em>None found for this date.</em><br>
{% endfor %}
@@ -16,11 +16,8 @@
{% for item in wallets %}
<a href="/survey_scans/{{item.walletname|urlencode}}/">{{item.walletname|safe}}</a>
{% if item.name %}
- {{item.name|safe}} <br/>
- {% else %}
- {{item.get_fnames|safe}}
- <br/>
- {% endif %}
+ {{item.name|safe}}
+ {% endif %}{{item.get_fnames|truncatechars:80}}<br/>
{% empty %}
<em>None found for this date.</em><br>
{% endfor %}
@@ -28,7 +25,7 @@
{% else %}
<em>No wallets files found for this date.</em><br>
{% endif %}
- {% if trips %}<u>All logbook trips</u> on this date:<br>
+ {% if trips %}<u>Logbook trips</u> on this date:<br>
<span style="font-size: 70%; ">
{% for item in trips %}
{% if item.isLogbookEntry %} <a href="{{item.get_absolute_url}}">{{item.title|safe}}</a><br/>{% endif %}
diff --git a/templates/svxfile.html b/templates/svxfile.html
index 5d493f8..e37c48f 100644
--- a/templates/svxfile.html
+++ b/templates/svxfile.html
@@ -83,16 +83,15 @@ LOGMESSAGES
<span style="font-family: monospace; font-size: 130%; ">
{% for sb in svxblocks %}
{% empty %}
-Cannot find any survex blocks in this survex file (not looking at *include files). <br />
+Cannot find any <em>dated</em> survex blocks in this survex file (not looking at *include files). <br />
Report this to a nerd if you think this is incorrect.
<hr />
{% endfor %}
{% for key, value in events.items %}
<details {% if forloop.first %} open{% endif %}>
-<summary><b><a href="/expedition/{{key|date:"Y"}}">{{key|date:"Y"}}</a>{{key|date:"-m-d"}}</b> </summary>
-<br />
-{% with trips=value.0 svxfiles=value.1 wallets=value.2 %}
+<summary><b><a href="/expedition/{{key|date:"Y"}}">{{key|date:"Y"}}</a>{{key|date:"-m-d"}}</b> {{value.3}}</summary>
+{% with trips=value.0 svxothers=value.1 wallets=value.2 %}
{% include 'onthisdate.html' %}
{% endwith %}
</details>
diff --git a/templates/walletform.html b/templates/walletform.html
index c35fa03..801b5a2 100644
--- a/templates/walletform.html
+++ b/templates/walletform.html
@@ -228,6 +228,7 @@
</button>{% endif %}
</form>
</div>
+
<span style="font-family: monospace; font-size: 150%; ">
{% if trips %}<u>Logbook trips</u> on this date:<br>
@@ -241,17 +242,18 @@
{% else %}
<em>No Logbook trips found for this date.</em><br>
{% endif %}
- {% if svxothers %}<u>Other survex files</u> on this date:<br>
- <span style="font-size: 70%; ">
- {% for item in svxothers %}
- {% if item.isSurvexBlock %} <a href="/survexfile/{{item.survexfile.path}}">{{item.survexfile.path|safe}}</a><br/>{% endif %}
- {% empty %}
- <em>None found for this date, but there should be..</em><br>
- {% endfor %}
- </span>
- {% else %}
- <em>No other survex files found for this date.</em><br>
- {% endif %}
+
+ {% if svxothers %}<u>Survex files</u> on this date:<br>
+ <span style="font-size: 70%; ">
+ {% for item in svxothers %}
+ <a href="/survexfile/{{item.path}}">{{item.path|safe}}</a><br/>
+ {% empty %}
+ <em>None found for this date.</em><br>
+ {% endfor %}
+ </span>
+ {% else %}
+ <em>No survex files found for this date.</em><br>
+ {% endif %}
{% if metadataurl %}<span style="font-size: 70%; "><details><summary>
JSON <br>