refactoring wallets and expos survey lengths code

core/views/statistics.py

@@ -15,21 +15,12 @@ from troggle.parsers.people import GetPersonExpeditionNameLookup, foreign_friend
 
 # from django.views.generic.list import ListView
 """Very simple report pages summarizing data about the whole set of expeditions and of
-the status of data inconsistencies
+the status of data inconsistencies. Also the wallets summary report.
 """
-
-def svxfilewild(request, year=None):
-    """Looks for survexfiles which do not have an associated
-    wallet, per year
-    """
+def legs_by_expo(expos):
     legsbyexpo = []
     addupsurvexlength = 0.0
     addupsurvexlegs = 0
- 
-    if not year:
-        expos = Expedition.objects.all()
-    else:
-        expos = Expedition.objects.filter(year=year)
     
     for expedition in expos:
         survexblocks = expedition.survexblock_set.all()
@@ -42,7 +33,9 @@ def svxfilewild(request, year=None):
         addupsurvexlegs += legsyear
         legsbyexpo.append((expedition, {"nsurvexlegs": legsyear, "survexleglength": survexleglength}))
     legsbyexpo.reverse()
-
+    return legsbyexpo, addupsurvexlegs, addupsurvexlength
+    
+def legs_and_lengths(expos):
     svxwild = []
     svxforeign = []
     wildlength = 0.0
@@ -66,6 +59,23 @@ def svxfilewild(request, year=None):
                     for p in people:
                         team.append(p.personname)
                     sb.team = team
+    return svxwild, svxforeign, wildlength, foreignlength
+    
+def svxfilewild(request, year=None):
+    """Looks for survexfiles which do not have an associated
+    wallet, per year. This should not be in this file, but with the other wallets reports.
+    """
+
+ 
+    if not year:
+        expos = Expedition.objects.all()
+    else:
+        expos = Expedition.objects.filter(year=year)
+    
+    legsbyexpo, addupsurvexlegs, addupsurvexlength = legs_by_expo(expos)
+
+    svxwild, svxforeign, wildlength, foreignlength = legs_and_lengths(expos)
+
     walletslength = 0.0
     if year:
         wallets = Wallet.objects.filter(walletyear__year=year)
@@ -73,7 +83,7 @@ def svxfilewild(request, year=None):
             for sb in w.survexblock_set.all():
                 walletslength += sb.legslength
     if not year:
-        year =  current_expo()
+        year =  " - all years"
     return render(request, "survexfilewild.html",
         {"addupsurvexlength": addupsurvexlength / 1000, 
         "legsbyexpo": legsbyexpo, 
@@ -86,7 +96,27 @@ def svxfilewild(request, year=None):
         "svxwild": svxwild,
         "svxforeign": svxforeign}
     )
-    
+
+def stats(request):
+    """Calculates number of survey blocks, the number of survey legs and the survey length for each year.
+    This is only underground survey legs, but includes ARGE as well as Expo survex files.
+    """
+    statsDict = {}
+    statsDict["expoCount"] = f"{Expedition.objects.count():,}"
+    statsDict["caveCount"] = f"{Cave.objects.count():,}"
+    statsDict["personCount"] = f"{Person.objects.count():,}"
+    statsDict["walletsCount"] = f"{Wallet.objects.count():,}"
+    statsDict["logbookEntryCount"] = f"{LogbookEntry.objects.count():,}"
+
+    expos = Expedition.objects.all()
+    legsbyexpo, addupsurvexlegs, addupsurvexlength = legs_by_expo(expos)
+
+    renderDict = {
+        **statsDict,
+        **{"addupsurvexlength": addupsurvexlength / 1000, "legsbyexpo": legsbyexpo, "nsurvexlegs": addupsurvexlegs, "year": current_expo()},
+    }  # new syntax
+    return render(request, "statistics.html", renderDict)
+   
 def therionissues(request):
     """Page displaying contents of a file produced during data import"""
     logname = "therionrefs.log"
@@ -229,36 +259,6 @@ def pathsreport(request):
     )
 
 
-def stats(request):
-    """Calculates number of survey blocks, the number of survey legs and the survey length for each year.
-    This is only underground survey legs, but includes ARGE as well as Expo survex files.
-    """
-    statsDict = {}
-    statsDict["expoCount"] = f"{Expedition.objects.count():,}"
-    statsDict["caveCount"] = f"{Cave.objects.count():,}"
-    statsDict["personCount"] = f"{Person.objects.count():,}"
-    statsDict["logbookEntryCount"] = f"{LogbookEntry.objects.count():,}"
-
-    legsbyexpo = []
-    addupsurvexlength = 0.0
-    addupsurvexlegs = 0
-    for expedition in Expedition.objects.all():
-        survexblocks = expedition.survexblock_set.all()
-        legsyear = 0
-        survexleglength = 0.0
-        for survexblock in survexblocks:
-            survexleglength += survexblock.legslength
-            legsyear += int(survexblock.legsall)
-        addupsurvexlength += survexleglength
-        addupsurvexlegs += legsyear
-        legsbyexpo.append((expedition, {"nsurvexlegs": legsyear, "survexleglength": survexleglength}))
-    legsbyexpo.reverse()
-
-    renderDict = {
-        **statsDict,
-        **{"addupsurvexlength": addupsurvexlength / 1000, "legsbyexpo": legsbyexpo, "nsurvexlegs": addupsurvexlegs, "year": current_expo()},
-    }  # new syntax
-    return render(request, "statistics.html", renderDict)
 
 
 def dataissues(request):

parsers/survex.py

@@ -306,7 +306,7 @@ class LoadingSurvex:
     rx_argsref = re.compile(r"(?i)^[\s.:]*((?:19[6789]\d)|(?:20[012345]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$")
     rx_badmerge = re.compile(r"(?i).*(\>\>\>\>\>)|(\=\=\=\=\=)|(\<\<\<\<\<).*$")
     rx_ref2 = re.compile(r"(?i)\s*ref[.;]?")
-    rx_commteam = re.compile(r"(?i)\s*(Messteam|Zeichner)\s*[:]?(.*)")
+    rx_commteam = re.compile(r"(?i)\s*(Messteam|Zeichner|LUSS Dead Mountains)\s*[:]?(.*)") # non-expo survex files
     rx_quotedtitle = re.compile(r'(?i)^"(.*)"$')
     
     """

templates/statistics.html

@@ -5,14 +5,14 @@
 
 <h1>Expedition Statistics</h1>
 
-<p>{{ expoCount }} expeditions: {{ personCount }} people,  {{ caveCount }} caves and {{ logbookEntryCount }} logbook entries. 
+<p>{{ expoCount }} expeditions: {{ personCount }} people,  {{ caveCount }} caves, {{ walletsCount }} wallets and {{ logbookEntryCount }} logbook entries. 
 
 <p>Number of survey legs: {{nsurvexlegs}} <br />
 Total length: {{addupsurvexlength|stringformat:".1f"}} km adding up the total for each year.</p>
 
 <p>These are uncorrected tape lengths which include pitches and duplicates but exclude splays or surface-surveys. 
 <p>
-This is work in progress (March 2023): the underground survey length does not match that in e.g.  
+This is work in progress June 2025): the underground survey length does not match that in e.g.  
 <a href="/wallets/year/2018">wallets for 2018</a> probably because ARGE surveys are not in any of our wallets.
 <p>This includes ARGE and other surveys currently. It will be changed to only include lengths surveyed by valid Expo-attendees.
 

templates/survexfilewild.html

@@ -6,7 +6,7 @@
 
 
 <h1>'Wild' Survex files {{year}}</h1>
-<p>
+<p> <a href="/survexfilewild">ALL YEARS</a>
 {% for otherexpedition in expeditions %}
   | <a <a href="/survexfilewild/{{ otherexpedition.year }}">{{otherexpedition.year}}</a>
 {% endfor %}
@@ -14,7 +14,7 @@
 <p>These are survex blocks within a survex file which have NO ATTACHED WALLET.
 <br />
 i.e. they have a survex block (begin..end) with no *REF line which refers to the wallet holding the raw data for that block of data.
-<p>These have "; Messteam" or "; Zeichner" in the survex block, so are non expo surveys.
+<p>These have "; Messteam", "; Zeichner" or "; "LUSS Dead Mountains" in the survex block, so are non expo surveys.
 <table>
 <tr><th>survex block with no *ref</th><th>date</th><th>parent block</th><th>within survex file</th><th>surveyed length</th></tr>
 {% for sb in svxforeign %}
@@ -28,7 +28,7 @@ i.e. they have a survex block (begin..end) with no *REF line which refers to the
 {% endfor %}
 </table>
 
-<p>These do not have "; Messteam" or "; Zeichner" in the survex block, so are <em>probably</em> expo surveys; definitely if they have identified team members (or have zero length).
+<p>These do not have "; Messteam", "; Zeichner" or "; "LUSS Dead Mountains" in the survex block, so are <em>probably</em> expo surveys; definitely if they have identified team members (or have zero length).
 <table>
 <tr><th>survex block with no *ref</th><th>date</th><th>parent block</th><th>within survex file</th><th>surveyed length</th><th>team</th></tr>
 {% for sb in svxwild %}