First attempts at better use of Django query optimisation

core/models/logbooks.py

@@ -25,9 +25,9 @@ class LogbookEntry(TroggleModel):
     date = (
         models.DateField()
     ) 
-    expedition = models.ForeignKey(Expedition, blank=True, null=True, on_delete=models.CASCADE)  
+    expedition = models.ForeignKey(Expedition, blank=True, null=True, on_delete=models.CASCADE, db_index=True)  
     title = models.CharField(max_length=200)
-    cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL)
+    cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL, db_index=True)
     place = models.CharField(
         max_length=100, blank=True, null=True, help_text="Only use this if you haven't chosen a cave"
     )
@@ -173,7 +173,7 @@ class PersonLogEntry(TroggleModel):
     then this PersonLogEntry is deleted too
     """
 
-    personexpedition = models.ForeignKey("PersonExpedition", null=True, on_delete=models.CASCADE)
+    personexpedition = models.ForeignKey("PersonExpedition", null=True, on_delete=models.CASCADE, db_index=True)
     time_underground = models.FloatField(help_text="In decimal hours")
     logbook_entry = models.ForeignKey(LogbookEntry, on_delete=models.CASCADE, db_index=True)
     is_logbook_entry_author = models.BooleanField(default=False)

core/models/survex.py

@@ -199,17 +199,17 @@ class SurvexBlock(models.Model):
     objects = SurvexBlockLookUpManager() # overwrites SurvexBlock.objects and enables lookup()
     name = models.CharField(max_length=100)
     title = models.CharField(max_length=200)
-    parent = models.ForeignKey("SurvexBlock", blank=True, null=True, on_delete=models.SET_NULL)
+    parent = models.ForeignKey("SurvexBlock", blank=True, null=True, on_delete=models.SET_NULL, db_index=True)
 
     date = models.DateField(blank=True, null=True)
-    expedition = models.ForeignKey("Expedition", blank=True, null=True, on_delete=models.SET_NULL)
+    expedition = models.ForeignKey("Expedition", blank=True, null=True, on_delete=models.SET_NULL, db_index=True)
     # if the survexfile object is deleted, then all the survex-blocks in it should be too,
     # though a block can span more than one file...
     survexfile = models.ForeignKey("SurvexFile", blank=True, null=True, on_delete=models.CASCADE, db_index=True)
     # survexpath = models.CharField(max_length=200, blank=True, null=True)  No need for this anymore
 
     scanswallet = models.ForeignKey(
-        "Wallet", null=True, on_delete=models.SET_NULL
+        "Wallet", null=True, on_delete=models.SET_NULL, db_index=True
     )  # only ONE wallet per block. The most recent seen overwites.. ugh.
 
     legsall = models.IntegerField(null=True)  # summary data for this block
@@ -245,11 +245,11 @@ class SurvexPersonRole(models.Model):
     """The CASCADE means that if a SurvexBlock or a Person is deleted, then the SurvexPersonRole 
     is deleted too
     """
-    survexblock = models.ForeignKey("SurvexBlock", on_delete=models.CASCADE)
+    survexblock = models.ForeignKey("SurvexBlock", on_delete=models.CASCADE, db_index=True)
     # increasing levels of precision, Surely we only need survexblock and (either person or personexpedition)?
     personname = models.CharField(max_length=100)
-    person = models.ForeignKey("Person", blank=True, null=True, on_delete=models.CASCADE) # not needed
-    personexpedition = models.ForeignKey("PersonExpedition", blank=True, null=True, on_delete=models.SET_NULL)
+    person = models.ForeignKey("Person", blank=True, null=True, on_delete=models.CASCADE, db_index=True) # not needed
+    personexpedition = models.ForeignKey("PersonExpedition", blank=True, null=True, on_delete=models.SET_NULL, db_index=True)
 
     def __str__(self):
         return str(self.personname) + " - " + str(self.survexblock)

core/models/troggle.py

@@ -154,8 +154,8 @@ class PersonExpedition(TroggleModel):
     is deleted too
     """
 
-    expedition = models.ForeignKey(Expedition, on_delete=models.CASCADE)
-    person = models.ForeignKey(Person, on_delete=models.CASCADE)
+    expedition = models.ForeignKey(Expedition, on_delete=models.CASCADE, db_index=True)
+    person = models.ForeignKey(Person, on_delete=models.CASCADE, db_index=True)
     slugfield = models.SlugField(max_length=50, blank=True, null=True)  # 2022 to be used in future
 
     # is_guest = models.BooleanField(default=False) # This is per-Person, not per-PersonExpedition
@@ -170,6 +170,7 @@ class PersonExpedition(TroggleModel):
 
     def get_absolute_url(self):
         # we do not use URL_ROOT any more.
+        # This is crackers, the whole point of get_absolute_url is to use the automatic reverse resolution, see below
         return(f"/personexpedition/{self.person.slug}/{self.expedition.year}")
         # why does this hang the system ?
         return reverse(
@@ -179,15 +180,6 @@ class PersonExpedition(TroggleModel):
                 "year": self.expedition.year,
             },
         )
-        # old style, no longer used
-        return reverse(
-                "personexpedition",
-                kwargs={
-                    "first_name": self.person.first_name,
-                    "last_name": self.person.last_name,
-                    "year": self.expedition.year,
-                },
-            )
 
     def surveyedleglength(self):
         """Survey length for this person on all survex trips on this expedition"""

core/views/logbooks.py

@@ -16,7 +16,7 @@ from troggle.parsers.imports import import_logbook
 """These views are for logbook items when they appear in an 'expedition' page
 and for persons: their individual pages and their perseonexpedition pages.
 
-It uses the global object TROG to hold some cached pages.
+It uses the global object TROG to hold some cached pages. USELESS as cache only works single-threaded, single-user.
 """
 
 todo = """- Fix the get_person_chronology() display bug.
@@ -91,17 +91,16 @@ def expedition(request, expeditionname):
             # print('! - expo {expeditionanme} using cached page')
             return render(request, "expedition.html", {**ts[expeditionname], "logged_in": logged_in})
 
-    expeditions = Expedition.objects.all()  # top menu only, evaluated only when template renders
 
-    entries = expo.logbookentry_set.all()
-    blocks = expo.survexblock_set.all()
+    entries = expo.logbookentry_set.only('date','title').filter(expedition=expo)
+    blocks = expo.survexblock_set.only('date','name').filter(expedition=expo).prefetch_related('scanswallet', 'survexfile')
     dateditems = list(entries) + list(blocks)  # evaluates the Django query and hits db
     dates = sorted(set([item.date for item in dateditems]))
 
-    allpersonlogentries = PersonLogEntry.objects.filter(personexpedition__expedition=expo)
+    allpersonlogentries = PersonLogEntry.objects.prefetch_related('logbook_entry').select_related('personexpedition__expedition').filter(personexpedition__expedition=expo)
 
     personexpodays = []
-    for personexpedition in expo.personexpedition_set.all():
+    for personexpedition in expo.personexpedition_set.all().prefetch_related('person'):
         expotrips = allpersonlogentries.filter(personexpedition=personexpedition)  # lazy
         expoblocks = blocks.filter(survexpersonrole__personexpedition=personexpedition)
 
@@ -118,6 +117,8 @@ def expedition(request, expeditionname):
             prow.append(pcell)
         personexpodays.append({"personexpedition": personexpedition, "personrow": prow, "sortname": personexpedition.person.last_name})
 
+    expeditions = Expedition.objects.only('year')  # top menu only, evaluated only when template renders, only need "year"
+
     ts[expeditionname] = {
         "year": int(expeditionname),
         "expedition": expo,

templates/expedition.html

@@ -13,7 +13,8 @@
   {% if otherexpedition == expedition %}
   | <b>{{otherexpedition.year}}</b>
   {% else %}
-  | <a href="{{otherexpedition.get_absolute_url}}">{{ otherexpedition.year }}</a>
+  <!-- speed up by removing function call in href {{otherexpedition.get_absolute_url}}-->
+  | <a href="/expedition/{{ otherexpedition.year }}">{{ otherexpedition.year }}</a>
   {% endif %}
 {% endfor %}
 </p>
@@ -101,8 +102,8 @@ an "<b>S</b>" for a survey trip.  The colours of the "<b>T</b>" and "<b>S</b>" a
 
 <table class="expeditionlogbooks">
 <tr><th>Date</th><th>Logged trips and diary entries</th><th>Surveys</th><th>Wallets</th></tr>
-{% regroup dateditems|dictsort:"date" by date as dates %}
-{% for date in dates %}
+{% regroup dateditems|dictsort:"date" by date as didates %}
+{% for date in didates %}
 <tr>
 <td>{{date.grouper|date:"D d M Y"}}</td>
 <td>{% for item in date.list %}