Merge branch 'python3-new' of ssh://expo.survex.com/home/expo/troggle into python3-new

_deploy/wsl/localsettingsWSL.py

@@ -178,7 +178,7 @@ EXPOWEB = REPOS_ROOT_PATH / "expoweb"
 CAVEDESCRIPTIONS        = EXPOWEB / "cave_data"
 ENTRANCEDESCRIPTIONS    = EXPOWEB / "entrance_data"
 EXPOWEB_URL = ''
-SCANS_URL = '/survey_scans/'
+# SCANS_URL = '/survey_scans/' # defunct, removed.
 
 # Sanitise these to be strings as all other code is expecting strings
 # and we have not made the change to pathlib Path type in the other localsettings-* variants yet.

core/forms.py

@@ -18,7 +18,7 @@ There are other, simpler, upload forms in view/uploads.py
 Some are not used and need renovating or destroying.
 '''
 
-todo = '''Re-enable TinyMCE
+todo = '''
 '''
 
 class CaveForm(ModelForm):

core/models/caves.py

@@ -26,6 +26,14 @@ from django.shortcuts import render
 from troggle.core.models.troggle import TroggleModel, Person, Expedition, DataIssue
 from troggle.core.models.survex import SurvexStation
 from troggle.core.utils import writetrogglefile
+from troggle.core.utils import TROG
+
+# Us ethe TROG global object to cache teh cave lookup list
+Gcavelookup = TROG['caves']['gcavelookup']
+Gcave_count = TROG['caves']['gcavecount']
+
+Gcavelookup = None
+Gcave_count = None
 
 '''The model declarations for Areas, Caves and Entrances. Also LogBookENtry, QM, PersonTrip
 '''
@@ -600,8 +608,6 @@ class PersonTrip(TroggleModel):
         return f'{self.personexpedition} ({self.logbook_entry.date})'
 
 
-Gcavelookup = None
-Gcave_count = None
 def GetCaveLookup():
     """A very relaxed way of finding probably the right cave given almost any string which might serve to identify it
     

core/models/survex.py

@@ -1,6 +1,10 @@
 import os
-from urllib.parse import urljoin
 import re
+import json
+import operator
+from urllib.parse import urljoin
+from pathlib import Path
+from functools import reduce
 
 from django.db import models
 from django.conf import settings
@@ -18,7 +22,7 @@ class SurvexDirectory(models.Model):
         verbose_name_plural = "Survex directories"
 
     def __str__(self):
-        return "[SurvexDirectory:"+str(self.path)  + "-" + str(self.primarysurvexfile.path) + "-" + str(self.cave)+"]" 
+        return "[SurvexDirectory:"+str(self.path)  + " |  Primary svx:" + str(self.primarysurvexfile.path) +".svx ]" 
 
 
 class SurvexFile(models.Model):
@@ -160,6 +164,9 @@ class SurvexPersonRole(models.Model):
         return str(self.person) + " - " + str(self.survexblock) 
 
 class Wallet(models.Model):
+    '''We do not keep the JSON values in the database, we query them afresh each time,
+    but we will change this when we need to do a Django query on e.g. personame
+    '''
     fpath               = models.CharField(max_length=200)
     walletname          = models.CharField(max_length=200)
     
@@ -169,8 +176,171 @@ class Wallet(models.Model):
     def get_absolute_url(self):
         return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)}))
 
+    def get_json(self):
+        jsonfile = Path(self.fpath, 'contents.json')
+        if not Path(jsonfile).is_file():
+            #print(f'{jsonfile} is not a file')
+            return None
+        else:
+            with open(jsonfile) as json_f:
+                try:
+                    waldata = json.load(json_f)
+                except:
+                    wurl = f"/scanupload/{self.walletname}" # .replace('#', ':')     
+                    message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
+                    #print(message)
+                    raise
+
+        return waldata
+        
+    def year(self): 
+        if self.walletname[4] != "#":
+             return None
+        year = int(self.walletname[0:4])
+        if year < 1976 or year > 2050:
+            return None  
+        else:
+            return str(year)
+        
+
+    # Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it
+    def date(self):
+        if not self.get_json():
+            return None
+        jsondata = self.get_json()
+        return jsondata["date"]
+        
+    def people(self):
+        if not self.get_json():
+            return None
+        jsondata = self.get_json()
+        return jsondata["people"]
+        
+    def cave(self):
+        if not self.get_json():
+            return None
+        jsondata = self.get_json()
+        return jsondata["cave"]
+
+    def name(self):
+        if not self.get_json():
+            return None
+        jsondata = self.get_json()
+        return jsondata["name"]
+
+    def get_fnames(self):
+        '''Filenames without the suffix, i.e. without the ".jpg"
+        '''
+        dirpath = Path(settings.SCANS_ROOT, self.fpath)
+        files = []
+        if dirpath.is_dir():
+            try:
+                for f in dirpath.iterdir():
+                    if f.is_file():
+                        if f.name != 'contents.json' and f.name != 'walletindex.html':
+                            files.append(Path(f.name).stem)
+            except FileNotFoundError:
+                pass
+        return files
+
+
+    def get_ticks(self):
+        waldata = self.get_json()
+        if not waldata:
+            return {}
+        ticks = {}
+        
+        # Initially, are there any required survex files present ?
+        survexok = "red"
+        ticks["S"] = "red"
+        if waldata["survex not required"]:
+            survexok = "green"
+            ticks["S"] = "green"
+        else:
+            if waldata["survex file"]:
+                if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
+                    waldata["survex file"] = [waldata["survex file"]]
+                ngood = 0
+                nbad = 0
+                ticks["S"] = "lightblue"
+                for svx in waldata["survex file"]:
+                    if svx !="":
+                        if (Path(settings.SURVEX_DATA) / svx).is_file():
+                            ngood += 1
+                        else:
+                            nbad += 1
+                if nbad == 0 and ngood >= 1:
+                    ticks["S"] = "green"
+                if nbad >= 1 and ngood >= 1:
+                    ticks["S"] = "orange"
+                if nbad >= 1 and ngood == 0:
+                    ticks["S"] = "red"
+        
+       # Cave Description
+        if waldata["description written"]: 
+            ticks["C"] = "green"
+        else:
+            ticks["C"] = survexok
+        # QMs
+        if waldata["qms written"]:
+            ticks["Q"] = "green"
+        else:
+            ticks["Q"] = survexok    
+                
+        # Notes, Plan, Elevation; Tunnel
+        if waldata["electronic survey"]:
+            ticks["N"] = "green"
+            ticks["P"] = "green"
+            ticks["E"] = "green"
+            ticks["T"] = "green"
+        else:
+        
+            files = self.get_fnames()
+            
+            # Notes required
+            notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False)
+            notes_scanned = reduce(operator.or_, [f.endswith("notes") for f in files], notes_scanned)
+            if notes_scanned:
+                ticks["N"] = "green"
+            else:
+                ticks["N"] = "red"
+
+            # Plan drawing required
+            plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False)
+            plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned)
+            plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"])
+            if plan_drawing_required:
+                ticks["P"] = "red"
+            else:
+                ticks["P"] = "green"
+
+            # Elev drawing required
+            elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False)
+            elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned)
+            elev_scanned = reduce(operator.or_, [f.endswith("elevation") for f in files], elev_scanned)
+            elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"])
+            if elev_drawing_required:
+                ticks["E"] = "red"
+            else:
+                ticks["E"] = "green"
+
+            # Tunnel / Therion
+            if elev_drawing_required or plan_drawing_required:
+                ticks["T"] = "red"
+            else:
+                ticks["T"] = "green"
+      
+
+        # Website
+        if waldata["website updated"]:
+            ticks["W"] = "green"
+        else:
+            ticks["W"] = "red"
+  
+        return ticks
+   
     def __str__(self):
-        return str(self.walletname) + " (Wallet)"
+        return "[" + str(self.walletname) + " (Wallet)]"
 
 class SingleScan(models.Model):
     ffile    = models.CharField(max_length=200)
@@ -189,7 +359,7 @@ class SingleScan(models.Model):
 class DrawingFile(models.Model):
     dwgpath          = models.CharField(max_length=200)
     dwgname          = models.CharField(max_length=200)
-    manywallets      = models.ManyToManyField("Wallet") # implicitly links via folders to scans to SVX files
+    dwgwallets       = models.ManyToManyField("Wallet") # implicitly links via folders to scans to SVX files
     scans            = models.ManyToManyField("SingleScan")  # implicitly links via scans to SVX files
     dwgcontains      = models.ManyToManyField("DrawingFile")  # case when its a frame type
     filesize         = models.IntegerField(default=0)

core/models/troggle.py

@@ -128,7 +128,7 @@ class Person(TroggleModel):
     
     #href        = models.CharField(max_length=200)
     orderref    = models.CharField(max_length=200)  # for alphabetic 
-    user        = models.OneToOneField(User, null=True, blank=True,on_delete=models.CASCADE)
+    user        = models.OneToOneField(User, null=True, blank=True,on_delete=models.CASCADE) # not used now
     def get_absolute_url(self):
         return urljoin(settings.URL_ROOT,reverse('person',kwargs={'first_name':self.first_name,'last_name':self.last_name}))
 

core/utils.py

@@ -42,6 +42,10 @@ TROG = {
     },
     'issues' : {
         'logdataissues' : {}
+    },
+    'caves' : {
+        'gcavelookup' : {},
+        'gcavecount' : {}
     }
 
 }

core/views/scans.py

@@ -1,5 +1,6 @@
 import os, stat
 import re
+import datetime
 from pathlib import Path
 from urllib.parse import urljoin, unquote as urlunquote
 from urllib.request import urlopen
@@ -8,9 +9,12 @@ from django.conf import settings
 from django.shortcuts import render
 from django.http import HttpResponse
 
-from troggle.core.models.survex import Wallet, SingleScan
+from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock
+from troggle.core.models.troggle import Person
 from troggle.core.models.caves import GetCaveLookup
 from troggle.core.views.expo import getmimetype
+#from troggle.parsers.people import GetPersonExpeditionNameLookup
+
 #import parsers.surveys
 
 '''one of these views serves files as binary blobs, and simply set the mime type based on the file extension,
@@ -19,8 +23,148 @@ by looking inside the file before being served.
 
 need to check if inavlid query string is invalid, or produces multiple replies
 and render a user-friendly error page.
+
+Note that datewallet(), caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the
+manywallets dict.
 '''
 
+def populatewallet(w):
+    '''Copy survex data here just for display, not permanently
+    '''
+    survexpeople = []
+    blocks = SurvexBlock.objects.filter(scanswallet = w)
+    for b in blocks:
+        for personrole in b.survexpersonrole_set.all(): 
+            survexpeople.append(personrole.personname)
+        w.persons = list(set(survexpeople)) 
+
+def datewallet(w, earliest):
+    first = earliest
+    blocks = SurvexBlock.objects.filter(scanswallet = w)
+    for b in blocks:
+        if b.date:
+            if b.date < first:
+                first = b.date    
+    if first == earliest:
+        # no date found
+        w.date = None
+    else:
+        w.date = first
+    
+def caveifywallet(w):
+    '''Gets the cave from the list of survex files,
+    only selects one of them though. Only used for display.
+    '''
+    blocks = SurvexBlock.objects.filter(scanswallet = w)
+    for b in blocks:    
+        # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath
+        if b.survexfile.cave:
+            w.cave  = b.survexfile.cave # just gets the last one, randomly. SHould make this a list or many:many ideally
+            
+def fillblankpeople(w):
+        wp = w.people()
+        if not wp: # an -empty list
+            populatewallet(w)
+        else:
+            if len(wp) == 1:
+                nobody = wp[0].lower()
+                if  nobody == 'unknown' or nobody == 'nobody' or nobody == ' ':
+                    populatewallet(w)
+        
+def fillblankothers(w):       
+    earliest = datetime.datetime.now().date()
+    if not w.date():
+        datewallet(w, earliest)
+        
+    c = w.cave()
+    if not c:
+        caveifywallet(w)
+        
+
+def walletslistperson(request, first_name, last_name):
+    '''Page which displays a list of all the wallets for a specific person
+    HORRIBLE linear search through everything. Index and do SQL query properly
+    '''
+    # This is where we face having to re-do everything to do with names properly, rather than the horrible series of hacks over 20 years..
+    #GetPersonExpeditionNameLookup
+    def tickspersonwallet(p):
+        manywallets = []
+        wallets = Wallet.objects.all()
+        for w in wallets:
+            w.persons = w.people() # ephemeral attribute for web page
+            fillblankpeople(w)
+            if w.persons:
+                if p.fullname in w.persons:
+                    manywallets.append(w)
+                    fillblankothers(w)
+                    w.ticks = w.get_ticks() # the complaints in colour form
+        return manywallets
+    
+    try:
+        if last_name:
+            p = Person.objects.get(fullname= f'{first_name} {last_name}')
+        else: 
+            # speciall Wookey-hack
+            p = Person.objects.get(first_name= f'{first_name}')
+    except:
+        #raise
+        return render(request, 'errors/generic.html', {'message': f'Unrecognised name of a expo person: "{first_name} {last_name}"'})   
+    
+    manywallets = tickspersonwallet(p)
+
+    return render(request, 'personwallets.html', { 'manywallets':manywallets, 'settings': settings, 'person': p})
+
+
+def walletslistyear(request, year):
+    '''Page which displays a list of all the wallets in a specific year 
+    '''
+    def ticksyearwallet(year):
+        manywallets = []
+        wallets = Wallet.objects.all() 
+        for w in wallets:
+
+            if year == w.year():
+                manywallets.append(w)
+                fillblankpeople(w)
+                fillblankothers(w)
+                w.ticks = w.get_ticks() # the complaints in colour form
+            else:
+                continue
+
+        return manywallets
+        
+    if year < 1976 or year > 2050:
+        return render(request, 'errors/generic.html', {'message': 'Year out of range. Must be between 1976 and 2050'})   
+    else:
+        year = str(year)
+    #return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'})   
+
+    manywallets = ticksyearwallet(year)
+ 
+    return render(request, 'yearwallets.html', { 'manywallets':manywallets, 'settings': settings, 'year': year})
+
+
+
+def cavewallets(request, caveid):
+    '''Returns all the wallets for just one cave
+    '''
+    Gcavelookup = GetCaveLookup()
+    if caveid in Gcavelookup:
+        cave = Gcavelookup[caveid]
+    else:
+        return render(request,'errors/badslug.html', {'badslug': caveid})
+
+    # remove duplication. SOrting is done in the template
+    wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave)) # NB a filtered set
+    manywallets = list(wallets)
+
+    for w in manywallets:
+        fillblankpeople(w)
+        fillblankothers(w)   
+        w.ticks = w.get_ticks() # the complaints in colour form
+    return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave})
+
+
 def oldwallet(request, path):
     '''Now called only for non-standard wallet structures for pre-2000 wallets
     '''
@@ -59,28 +203,13 @@ def scansingle(request, path, file):
         return render(request, 'errors/generic.html', {'message': message})   
 
 
-def allwallets(request):
+def allscans(request):
     '''Returns all the wallets in the system, we would like to use
     the Django queryset SQL optimisation https://docs.djangoproject.com/en/3.2/ref/models/querysets/#prefetch-related
     to get the related singlescan and survexblock objects but that requires rewriting this to do the query on those, not on
     the wallets
     '''
-    manywallets = Wallet.objects.all()
+    manywallets = Wallet.objects.all() # NB all of them
     # manywallets = Wallet.objects.all().prefetch_related('singlescan') fails as the link is defined on 'singlescan' not on 'wallet'
     return render(request, 'manywallets.html', { 'manywallets':manywallets, 'settings': settings })
     
-def cavewallets(request, cave_id):
-    '''Returns all the wallets for just one cave, 
-    '''
-        
-    Gcavelookup = GetCaveLookup()
-    if cave_id in Gcavelookup:
-        cave = Gcavelookup[cave_id]
-    else:
-        return render(request,'errors/badslug.html', {'badslug': cave_id})
-
-    # remove duplication. SOrting is done in the template
-    wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave))
-    manywallets = list(wallets)
-
-    return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave})

core/views/statistics.py

@@ -51,7 +51,7 @@ def pathsreport(request):
         "SURVEX_DATA" : str( settings.SURVEX_DATA),
         "SCANS_ROOT" : str( settings.SCANS_ROOT),
 #       "SURVEYS" : str( settings.SURVEYS),
-        "SCANS_URL" : str( settings.SCANS_URL),
+#       "SCANS_URL" : str( settings.SCANS_URL),
         "SURVEXPORT" : str( settings.SURVEXPORT),
         "DRAWINGS_DATA" : str( settings.DRAWINGS_DATA),
         "URL_ROOT" : str( settings.URL_ROOT)
@@ -88,7 +88,7 @@ def pathsreport(request):
         "SURVEX_DATA" : type(settings.SURVEX_DATA),
         "SCANS_ROOT" : type(settings.SCANS_ROOT),
 #       "SURVEYS" : type(settings.SURVEYS),
-        "SCANS_URL" : type(settings.SCANS_URL),
+#       "SCANS_URL" : type(settings.SCANS_URL),
         "SURVEXPORT" : type(settings.SURVEXPORT),
         "DRAWINGS_DATA" : type(settings.DRAWINGS_DATA),
         "URL_ROOT" : type(settings.URL_ROOT)

core/views/uploads.py

@@ -23,12 +23,12 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 #from troggle import settings
 from troggle.parsers.imports import import_caves, import_people, import_surveyscans
 from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
-from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml
+from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml, CopyWalletData
 # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
 from troggle.core.models.troggle import DataIssue
 from troggle.core.models.troggle import Expedition, Person, PersonExpedition
 from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
-from troggle.core.models.survex import DrawingFile
+from troggle.core.models.survex import DrawingFile, Wallet
 from troggle.core.views.scans import oldwallet, walletindex
 from troggle.core.views.caves import getCave
 
@@ -93,13 +93,16 @@ xlate = {"url": "description url",
         "electronic": "electronic survey",
         "pland": "plan drawn",
         "elevd": "elev drawn",
-        "psg": "name",
+        "psg": "name", # a name for this wallet
         "survex": "survex file",
         }
 
 def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
     '''Taken from old script wallets.py and edited to make more comprehensible
     Loads the survex files names and processes all complaints
+    
+    All needs to be restructred to use the get_ticks() function on the Wallets class in core/models/survex.py 
+    which does the same thing
     '''
     # Date
     if not waldata["date"]:
@@ -115,6 +118,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
         if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
             waldata["survex file"] = [waldata["survex file"]]       
         for svx in waldata["survex file"]:
+            if svx !="":
                 svxfiles.append(svx)
                 if not (Path(settings.SURVEX_DATA) / svx).is_file():
                     file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo"
@@ -133,20 +137,21 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
     # Notes required
     if not waldata["electronic survey"]:
         notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False)
-        notes_scanned = reduce(operator.or_, [f.endswith("note") for f in files], notes_scanned)
+        notes_scanned = reduce(operator.or_, [Path(f).stem.endswith("notes") for f in files], notes_scanned)
         if not notes_scanned:
             complaints.append("The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey.") 
 
     # Plan drawing required
     plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False)
-    plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned)
+    plan_scanned = reduce(operator.or_, [Path(f).stem.endswith("plan") for f in files], plan_scanned)
     plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"])
     if plan_drawing_required:
         complaints.append("The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found.") 
 
     # Elev drawing required
     elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False)
-    elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned)
+    elev_scanned = reduce(operator.or_, [Path(f).stem.endswith("elev") for f in files], elev_scanned)
+    elev_scanned = reduce(operator.or_, [Path(f).stem.endswith("elevation") for f in files], elev_scanned)
     elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"])
     if elev_drawing_required:
         complaints.append("The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found.") 
@@ -290,6 +295,21 @@ def scanupload(request, path=None):
                 with open(contents_path, "w") as jfile:
                     json.dump(wd, jfile, indent = 1)
                 # print(f'--- FINISHED saving to JSON\n')
+                
+                # This copies the new data to the drawings repo and commit it
+                # needs the troggle object wallet, not a string
+ 
+                try:
+                    w, created = Wallet.objects.get_or_create(walletname=wallet)
+                    print(f'wallet string {wallet}, wallet object {w} created new?: {created}')   
+                    if created:
+                        w.fpath = Path(settings.SCANS_ROOT, wallet[0:4], wallet)
+                        w.save()
+                    CopyWalletData(w)
+                except:
+                     print(f'wallet string {wallet}, FAIL TO GET WALLET OBJECT, maybe we need to create it ?')
+                     raise
+               
             else:
                 print(f'--- INVALID JSON Update form submitted')
                 print(formj.errors)

parsers/caves.py

@@ -13,23 +13,16 @@ from troggle.core.models.caves import Area, Cave, Entrance, CaveSlug, EntranceSl
 '''Reads all the cave description data by parsing the xml files (stored as e.g. :EXPOWEB:/cave_data/1623-161.html )
 and creating the various Cave, Entrance and necessary Area objects.
 
-This is the first import that happens after the dabase is reinitialised. 
+This is the first import that happens after the database is reinitialised. 
 So is the first thing that creates tables.
 
-BUT in Django 2.0 and later we cannot do any queries on data we have just entered 
-because this is all happening inside one transaction. Bummer.
-
-django.db.transaction.TransactionManagementError: 
-An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.
 '''
 
-todo='''- db Update does not work when a cave id is in the pending list but a proper cave description file exists
+todo='''  
-   and is being imported. It should work. But currently Django aborts and he file is not read in.
-   
  - Cannot use Edit This Page for pendingcaves.txt_edit as Edit This Page is expecting an html file.
    So we will need a separate file-editing capability just for this configuration file ?!
    
-- crashes on MariaDB on server when deleting Caves and complains Area needs a non null parent, But this is not true.
+- crashes on MariaDB in databasereset.py on server when deleting Caves and complains Area needs a non null parent, But this is not true.
   The only solution we have found is to let it crash, then stop and restart MariaDB (requires a logon able to sudo)
   and then restart the databasereset.py again. (status as of July 2022)
 '''
@@ -92,6 +85,15 @@ def do_pending_cave(k, url, area):
     '''
     slug = k
     
+    g = GetCaveLookup()
+    if slug in g:
+        message = f" ! {k} cave listed in pendingcaves.txt already exists."
+        DataIssue.objects.create(parser='caves', message=message, url=url)
+        print(message)
+        return
+
+    
+
     default_note = f"_Survex file found in loser repo but no description in expoweb <br><br><br>\n" 
     default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then " 
     default_note += f"<br><br>\n\n - (1) search in the survex file for the *ref to find a "  
@@ -118,7 +120,7 @@ def do_pending_cave(k, url, area):
     cave = Cave(
             unofficial_number = k, 
             underground_description = "Pending cave write-up - creating as empty object. No XML file available yet.",
-            survex_file = f"caves-{area.short_name}/{k}/{k}.svx",
+            survex_file = f"caves-{area.short_name}/{k[5:]}/{k[5:]}.svx",
             url = url,
             notes = default_note)
     if cave:
@@ -465,27 +467,6 @@ def readcaves():
         print(" - Saving Area 1626")
         area_1626.save()
 
-        print (" - Setting pending caves")
-        # Do this first, so that these empty entries are overwritten as they get properly created.
-
-        for k in pending:
-            
-            area = area_1623
-            areanum = k[0:4]
-            url = areanum + "/" +  k[5:] # Note we are not appending the .htm as we are modern folks now.
-            if areanum == "1623":
-                area = area_1623
-            if areanum == "1624":
-                area = area_1624
-            if areanum == "1626":
-                area = area_1626
-            try:    
-                do_pending_cave(k[5:], url, area)
-            except:
-                message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum}"
-                DataIssue.objects.create(parser='caves', message=message)
-                print(message)
-                raise
 
     with transaction.atomic():
         print(" - settings.CAVEDESCRIPTIONS: ", CAVEDESCRIPTIONS)
@@ -506,3 +487,26 @@ def readcaves():
     print (" - Setting up all the variously useful alias names")
     mycavelookup = GetCaveLookup()
     
+    print (" - Setting pending caves")
+    # Do this last, so we can detect if they are created and no longer 'pending'
+
+    for k in pending:
+        
+        area = area_1623
+        areanum = k[0:4]
+        url = areanum + "/" +  k[5:] # Note we are not appending the .htm as we are modern folks now.
+        if areanum == "1623":
+            area = area_1623
+        if areanum == "1624":
+            area = area_1624
+        if areanum == "1626":
+            area = area_1626
+        try:    
+            do_pending_cave(k, url, area)
+        except:
+            message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum}"
+            DataIssue.objects.create(parser='caves', message=message)
+            print(message)
+            raise
+
+

parsers/drawings.py

@@ -50,7 +50,7 @@ def find_dwg_file(dwgfile, path):
                 scansfile = scansfilel[0]
 
         if wallet:
-            dwgfile.manywallets.add(wallet)
+            dwgfile.dwgwallets.add(wallet)
         if scansfile:
             dwgfile.scans.add(scansfile)
     

parsers/scans.py

@@ -1,12 +1,15 @@
 import sys
 import os
+import subprocess
 import types
 import stat
 import csv
 import re
 import datetime
+import shutil, filecmp
 
 from functools import reduce
+from pathlib import Path
 
 import settings
 from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
@@ -18,7 +21,9 @@ from troggle.core.utils import save_carefully, GetListDir
 
 contentsjson = "contents.json"
 indexhtml = "walletindex.html"
+git = settings.GIT
 
+# to do: create a 'low priority' field, so that any such wallet does not appear in summary reports
 wallet_blank_json = {
  "cave": "", 
  "date": "", 
@@ -55,6 +60,22 @@ wallet_blank_html = '''<html><body><H1>Wallet WALLET</H1>
 </body></html>
 '''
                     
+def CheckEmptyDate(wallet):
+    '''If date is not set, get it from a linked survex file. If several, pick the earliest.
+    
+    Maybe also look at filedates for the scans in expofiles/surveyscans/ , but these can be re-set by copying.
+    '''
+    return
+    
+def CheckEmptyPeople(wallet):
+    '''If people list is empty, copy them from the survex files: all of them
+   
+    To be a Troggle model change; a many:many relationship between wallets and people,
+    as well as being a list in the JSON file (which is the permanent repository). We want the many:many
+    relationship so that we can filter wallets based on a person.
+    '''
+    return
+
 def LoadListScansFile(wallet):
     gld = [ ]
     # flatten out any directories in these wallet folders - should not be any
@@ -73,6 +94,44 @@ def LoadListScansFile(wallet):
             if c>=10:
                 print(".", end='')
                 c = 0
+def CopyWalletData(wallet):
+    '''Copies all the contents.json to a parallel set of folders in the drawings repo
+    refreshes everything during a ful import, but it shoudl all be up to date as every time
+    wallet data gets saved it should also be copied across and committed.
+    '''
+    year = wallet.walletname[0:4]
+    destfolder = Path(settings.DRAWINGS_DATA,'walletjson', year, wallet.walletname)
+    destjson = destfolder / contentsjson
+    sourcejson = Path(wallet.fpath, contentsjson)
+    if not os.path.exists(Path(destfolder)):
+        try:
+            os.makedirs(destfolder)
+            print(f' -  created folder {destfolder}..')
+        except PermissionError:
+            print(f"CANNOT  save this JSON file.\nPERMISSIONS incorrectly set on server for this folder {destfolder}. Ask a nerd to fix this.")
+    if os.path.isfile(sourcejson):
+        try:
+            if not os.path.isfile(destjson) or not filecmp.cmp(sourcejson, destjson):
+                shutil.copy(sourcejson, destjson)
+                print(f' -  Copied {sourcejson} to {destjson}')
+                dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
+                if dr_add.returncode != 0:
+                    msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout  + '\n\nreturn code: ' + str(dr_add.returncode)
+                    message = f'CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n' + msgdata
+                    print(message)
+                else:
+                    # ideally we would commit many chnages to many wallets just once. But most of the time only a couple of files will change.
+                    dr_commit = subprocess.run([git, "commit", "-m", f'Update of {contentsjson} in wallet'], cwd=destfolder, capture_output=True, text=True)
+                    # This produces return code = 1 if it commits OK
+                    if dr_commit.returncode != 0:
+                        msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout  + '\n\nreturn code: ' + str(dr_commit.returncode)
+                        message = f'Error code with git on server for this {contentsjson}. File is copied, added to git, but NOT committed.\n\n' + msgdata
+                        print(message)
+
+        except PermissionError:
+            print(f"CANNOT  copy this JSON file.\nPERMISSIONS incorrectly set on server for this file {destjson}. Ask a nerd to fix this.")
+
+
  
         
 # this iterates through the scans directories (either here or on the remote server)
@@ -109,17 +168,20 @@ def load_all_scans():
                 if fisdir:
                     wallet = Wallet(fpath=fpath, walletname=walletname)
                     # this is where we should load the contents.json for people so we can report on them later
-                    # this is where we shoudl record the year explicitly
+                    # this is where we should record the year explicitly
                     # line 347 of view/uploads.py and needs refactoring for loading contentsjson
                     wallet.save()
                     LoadListScansFile(wallet)
+                    CheckEmptyDate(wallet)
+                    CheckEmptyPeople(wallet)
+                    CopyWalletData(wallet)
         
         # what is this?
-        elif walletname != "thumbs":
+        # elif walletname != "thumbs":
-            print(f'\n - Wallet {walletname} - {fpath}')
+            # print(f'\n - Wallet {walletname} - {fpath}')
-            wallet = Wallet(fpath=fpath, walletname=walletname)
+            # wallet = Wallet(fpath=fpath, walletname=walletname)
-            wallet.save()
+            # wallet.save()
-            LoadListScansFile(wallet)
+            # LoadListScansFile(wallet)
         else:
             print(f'\n - IGNORE {walletname} - {fpath}')
         

parsers/survex.py

@@ -37,7 +37,6 @@ todo = '''Also walk the entire tree in the :loser: repo looking for unconnected
 - LoadSurvexFile() Creates a new current survexfile and valid .survexdirectory
         The survexblock passed-in is not necessarily the parent. FIX THIS.
         
-- rx_qm recognises only simple survey point ids. EXTEND to cover more naming formats and test fully for 2023
 '''
 survexblockroot = None
 ROOTBLOCK = "rootblock"
@@ -131,8 +130,8 @@ class LoadingSurvex():
 
     rx_cave    = re.compile(r'(?i)caves-(\d\d\d\d)/([-\d\w]+|\d\d\d\d-?\w+-\d+)')
     rx_comment = re.compile(r'([^;]*?)\s*(?:;\s*(.*))?\n?$')
-    rx_comminc = re.compile(r'(?i)^\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include
+    rx_comminc = re.compile(r'(?i)^\|\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include
-    rx_commcni = re.compile(r'(?i)^\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni
+    rx_commcni = re.compile(r'(?i)^\|\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni
     rx_include = re.compile(r'(?i)^\s*(\*include[\s].*)$')
     rx_commref = re.compile(r'(?i)^\s*ref(?:erence)?[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)')
     rx_wallet  = re.compile(r'(?i)^\s*wallet[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)')
@@ -178,13 +177,14 @@ class LoadingSurvex():
     callcount = 0
     caverncount = 0
     ignoreprefix = ["surface", "kataster", "fixedpts", "gpx"]
-    ignorenoncave = ["caves-1623", "caves-1623/2007-neu"]
+    ignorenoncave = ["caves-1623", "caves-1626", "caves-1623/2007-neu"]
     includedfilename =""
     currentsurvexblock = None
     currentsurvexfile = None
     currentcave = None
     caverndate = None
     currentpersonexped = []
+    pending = []
 
     def __init__(self):
         self.caveslist = GetCaveLookup()
@@ -690,9 +690,7 @@ class LoadingSurvex():
     def IdentifyCave(self, cavepath):
         if cavepath.lower() in self.caveslist:
             return self.caveslist[cavepath.lower()]
-        # TO DO - some of this is already done in generating self.caveslist so simplify this
+        # TO DO - this predates the big revision to Gcavelookup so look at this again carefully
-        # esp. as it is in a loop.
-        # TO DO recognise cave if different name, e.g. gruenstein == 281
         path_match = self.rx_cave.search(cavepath)
         if path_match:
             sluggy = '{}-{}'.format(path_match.group(1), path_match.group(2))
@@ -724,8 +722,17 @@ class LoadingSurvex():
     def ReportNonCaveIncludes(self, headpath, includelabel, depth):
         """Ignore surface, kataser and gpx *include survex files
         """
+        if not self.pending:
+            self.pending = set()
+            fpending = Path(settings.CAVEDESCRIPTIONS, "pendingcaves.txt")
+            if fpending.is_file():
+                with open(fpending, "r") as fo:
+                    cids = fo.readlines()
+                for cid in cids:
+                    self.pending.add(cid.rstrip('\n').upper())
+
         if headpath in self.ignorenoncave:
-            #message = f" - {headpath} is <ignorenoncave> (while creating '{includelabel}' sfile & sdirectory)"
+            message = f" - {headpath} is <ignorenoncave> (while creating '{includelabel}' sfile & sdirectory)"
             #print("\n"+message)
             #print("\n"+message,file=sys.stderr)
             return
@@ -735,20 +742,26 @@ class LoadingSurvex():
                 # print("\n"+message)
                 # print("\n"+message,file=sys.stderr)
                 return
-        message = f" ! Error: FAILURE '{headpath}' while creating '{includelabel}' at depth:[{depth}]. Not a cave or in the ignore list:'{self.ignoreprefix}'"
+        caveid = f'{headpath[6:10]}-{headpath[11:]}'.upper()
-        # getting this triggered for gpx/2018 (cavern error) but not for gpx/2017 (no content).
+        if caveid in self.pending:
+           # Yes we didn't find this cave, but we know it is a pending one. So not an error.
+           # print(f'! ALREADY PENDING {caveid}',file=sys.stderr)
+           return
+            
+        message = f" ! Error: not a cave nor ignorable. headpath:'{headpath}' while parsing '{includelabel=}.svx' at depth:[{len(depth)}].  ignore prefix list:'{self.ignoreprefix}'"
         print("\n"+message)
         print("\n"+message,file=sys.stderr)
         DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(headpath))
         print(f' # datastack in  LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr)
         for dict in self.datastack:
-            print(f'{dict["type"].upper()}   ', end="",file=sys.stderr)
+            print(f'<{dict["type"].upper()}   >', end="",file=sys.stderr)
         
 
     def LoadSurvexFile(self, svxid):
         """Creates SurvexFile in the database, and SurvexDirectory if needed
         with links to 'cave'
         Creates a new current survexfile and valid .survexdirectory
+        Inspects the parent folder of the survexfile and uses that to decide if this is a cave we know
         The survexblock passed-in is not necessarily the parent. FIX THIS.
         """
         if debugprint:
@@ -780,7 +793,7 @@ class LoadingSurvex():
         if cave:
             newdirectory.cave = cave
             newfile.cave   = cave
-            # print(f"\n - New directory {newdirectory} for cave {newdirectory.cave}",file=sys.stderr)
+            # print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr)
         else: # probably a surface survey, or a cave in a new area e.g. 1624 not previously managed, and not in the pending list
             self.ReportNonCaveIncludes(headpath, svxid, depth)
             
@@ -862,6 +875,7 @@ class LoadingSurvex():
             
         included = self.rx_comminc.match(comment)
         # ;*include means 'we have been included'; whereas *include means 'proceed to include' 
+        # bug, If the original survex file contians the line ;*include then we pick it up ! So fix our special code to be ;|*include
         if included:
             self.ProcessIncludeLine(included)
 
@@ -1211,7 +1225,7 @@ class LoadingSurvex():
                         #--------------------------------------------------------
                         self.depthinclude += 1
                         fininclude = open(fullpath,'r')
-                        fcollate.write(";*include {}\n".format(includepath))
+                        fcollate.write(";|*include {}\n".format(includepath))
                         flinear.write("{:2} {} *include {}\n".format(self.depthinclude, indent, includepath))
                         push = includepath.lower()
                         self.includestack.append(push)
@@ -1226,7 +1240,7 @@ class LoadingSurvex():
                             print(message,file=sys.stderr)
                             DataIssue.objects.create(parser='survex', message=message,  url=get_offending_filename(path))
                         flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop))
-                        fcollate.write(";*edulcni {}\n".format(pop))
+                        fcollate.write(";|*edulcni {}\n".format(pop))
                         fininclude.close()
                         self.depthinclude -= 1
                         #--------------------------------------------------------

templates/base.html

@@ -31,13 +31,13 @@
     <a href="{% url "survexcavessingle" "359" %}">359</a> |
     <a href="/survexfile/">Survex</a> |
     <a href="{% url "survexcaveslist" %}">All Survex</a> |
-    <a href="{% url "allwallets" %}">Scans</a> |
+    <a href="{% url "allscans" %}">Scans</a> |
     <a href="{% url "scanupload"  '2022:01' %}">Upload Scans</a> |
     <a href="{% url "dwgallfiles" %}">Drawings</a> |
     <a href="{% url "dwgupload" %}">Upload Drawings</a> |
     <a href="{% url "photoupload" %}">Upload Photos</a> |
-    <a href="/1623/290/290.html">290 (FGH)</a> |
+    <a href="/1623/290/290">290 (FGH)</a> |
-    <a href="/1626/359/359.html">359 (Homecoming)</a> |
+    <a href="/1626/359/359">359 (Homecoming)</a> |
     <br>
     
     <a href="{% url "dataissues" %}">Data Issues</a> |
@@ -48,15 +48,15 @@
     <a id="folklink" href="/folk">expoers</a>  |
     <a id="caversLink" href="{% url "notablepersons" %}">survey lengths</a>  |
     <a href="{% url "stats" %}">statistics</a> |
-    <a href="{% url "expedition" 2018 %}">Expo2018</a> |
+    <a href="/wallets/year/2019">Wallets(2019)</a> |
-    <a href="{% url "expedition" 2019 %}">Expo2019</a> |
+    <a href="{% url "expedition" 2019 %}">Expo(2019)</a> |
     <a href="{% url "controlpanel" %}">import/export</a>  |
     <a href="/admin/">Django admin</a>
 </div>
     
 <div id="nav">
   {% block nav %} 
-  <!-- Use id="nav" for the left side menu -->
+  <!-- Not used any more? -->
   {% endblock %}
 </div>
  
@@ -67,7 +67,6 @@
 
     <div id="related"> 
         {% block related %}
-
         {% endblock %}
     </div>
     {% block content %}

templates/cavewallets.html

@@ -4,29 +4,41 @@
 
 {% block content %}
 
-<h3>Survey scans folders (wallets) for <a href="/{{cave.url}}">{{cave}}</a></h3>
+<h3>Wallets for <a href="/{{cave.url}}">{{cave}}</a> {{cave.official_name|safe}}</h3>
 <p>Each wallet contains the scanned original in-cave survey notes and sketches of 
 plans and elevations. It also contains scans of centre-line survex output on which
 hand-drawn passage sections are drawn. These hand-drawn passages will eventually be 
 traced to produce Tunnel or Therion drawings and eventually the final complete cave survey.
-
+<p>This lists all the files in a wallet, some of which may not be for this specific cave.
+<p>See also wallets
+<ul>
+<li>per year, e.g. <a href="/wallets/year/2019">2019</a>
+<li>per person, e.g. <a href="/wallets/person/MichaelSargent">Michael Sargent</a>
+</ul>
+{% include 'wallet_table.html' %}
+<br />
 <table width=95%>
-<tr><th>Scans folder</th><th>Files</th><th>Survex blocks</th><th>Cave</th></tr>
+<tr><th>Wallet</th><th width=8%>Wallet Date</th><th>Wallet Name</th><th>People</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr>
-{% for scanswallet in manywallets|dictsort:"walletname" %}
+{% for wallet in manywallets|dictsort:"walletname" %}
   <tr>
-    <td style="padding:2px"><a href="{{scanswallet.get_absolute_url}}">{{scanswallet.walletname}}</a></td>
+    <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
-    <td align="right" style="padding:2px">{{scanswallet.singlescan_set.all|length}}</td>
+    
+    <td style="padding:2px">{{wallet.date}}</td>
+    <td style="padding:2px">{{wallet.name}}</td>
+    <td style="padding:2px">{{wallet.persons}}</td>
+    
+    <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td>
     <td style="padding:2px">
-    {% for survexblock in scanswallet.survexblock_set.all %}
+    {% for survexblock in wallet.survexblock_set.all %}
       <a href="{% url "svx" survexblock.survexfile.path %}">{{survexblock}}</a>
     {% endfor %}
     </td>
-    <td style="padding:2px">
-    {% for survexblock in scanswallet.survexblock_set.all %}
-        {% ifchanged survexblock.survexfile.cave %}
-        <a href="/{{survexblock.survexfile.cave.url}}">/{{survexblock.survexfile.cave.slug}}</a>
-        {% endifchanged %}
 
+    <td style="padding:2px; font-size: 70%;">
+    {% for drawing in wallet.drawingfile_set.all %}
+      <a href="{% url "dwgfilesingle" drawing.dwgpath %}">{{drawing.dwgpath}}</a><br>
+    {% empty %}
+        (no Tunnel drawings found: but there might be Therion drawings)
     {% endfor %}
     </td>
   </tr>

templates/dataissues.html

@@ -6,7 +6,7 @@
 <h1>Loading data from files: Issues arising that need attention</h1>
 
 <p>
-This is work in progress (June 2022).The URL links to the offending objects are enabled on only some types of fault as yet.
+This is work in progress.The URL links to the offending objects are enabled on only some types of fault as yet.
 <p>
 See the 
 <a href="/handbook/computing/todo-data.html">Data Management To Do list</a> as well as these import/parsing issues.

templates/dwgfiles.html

@@ -13,7 +13,7 @@
     <td align="right" style="padding:2px">{{dwgfile.npaths}}</td>   
 
     <td style="padding:2px">
-    {% for scanswallet in dwgfile.manywallets.all %}
+    {% for scanswallet in dwgfile.dwgwallets.all %}
       <a href="{{scanswallet.get_absolute_url}}">{{scanswallet.walletname}}</a>
     {% endfor %}
     </td>   

templates/manywallets.html

@@ -9,6 +9,12 @@
 plans and elevations. It also contains scans of centre-line survex output on which
 hand-drawn passage sections are drawn. These hand-drawn passages will eventually be 
 traced to produce Tunnel or Therion drawings and eventually the final complete cave survey.
+<p>See also wallets
+<ul>
+<li>per year, e.g. <a href="/wallets/year/2019">2019</a>
+<li>per cave, e.g. <a href="/cave/scans/1623-204">1623/204</a>
+<li>per person, e.g. <a href="/wallets/person/MichaelSargent">Michael Sargent</a>
+</ul>
 
 <!-- This should all be restructured to use .prefetch_related() and .select_related()
 see https://docs.djangoproject.com/en/3.2/ref/models/querysets/#prefetch-related

templates/person.html

@@ -27,6 +27,9 @@
 </ul>
 </p>
 
+<h3>Surveys done</h3>
+Wallets and surveys mentioning <a href="/wallets/person/{{person}}">{{person}}</a>
+
 {% if person.blurb %}
 {{person.blurb|safe}}
 {% else %}

templates/personwallets.html

@@ -0,0 +1,50 @@
+{% extends "base.html" %}
+
+{% block title %}One Person Survey scans folders (wallets){% endblock %}
+
+{% block content %}
+<h3>Wallets for <a href="{{person.get_absolute_url}}">{{person}}</a> </h3>
+<p>Each wallet contains the scanned original in-cave survey notes and sketches of 
+plans and elevations. It also contains scans of centre-line survex output on which
+hand-drawn passage sections are drawn. These hand-drawn passages will eventually be 
+traced to produce Tunnel or Therion drawings and eventually the final complete cave survey.
+
+<p>See also wallets
+<ul>
+<li>per year, e.g. <a href="/wallets/year/2019">2019</a>
+<li>per cave, e.g. <a href="/cave/scans/1623-161">1623/161</a>
+</ul>
+
+{% include 'wallet_table.html' %}
+<br />
+<table width=95%>
+<tr><th>Wallet</th><th width=8%>Wallet Date</th><th>Wallet Name</th><th  width=15%>Other People</th><th>Cave</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr>
+{% for wallet in manywallets|dictsort:"walletname" %}
+  <tr>
+    <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
+    
+    <td style="padding:2px" >{{wallet.date}}</td>
+    <td style="padding:2px">{{wallet.name}}</td>
+    <td style="padding:2px">{{wallet.persons}}</td>
+    <td style="padding:2px">{{wallet.cave}}</td>
+    
+    <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td>
+    <td style="padding:2px">
+    {% for survexblock in wallet.survexblock_set.all %}
+      <a href="{% url "svx" survexblock.survexfile.path %}">{{survexblock}}</a>
+    {% endfor %}
+    </td>
+
+    <td style="padding:2px; font-size: 70%;">
+    {% for drawing in wallet.drawingfile_set.all %}
+      <a href="{% url "dwgfilesingle" drawing.dwgpath %}">{{drawing.dwgpath}}</a><br>
+    {% empty %}
+        (no Tunnel drawings found: but there might be Therion drawings)
+    {% endfor %}
+    </td>
+  </tr>
+{% endfor %}
+</table>
+
+
+{% endblock %}

templates/svxcavesingle.html

@@ -10,6 +10,7 @@
 All the processing to extract the survex subdriectories and survex files is done in this template -->
 
 <p>Cave description: <a href="/{{cave.url}}">{{cave.url}}</a>
+<p>Wallets: <a href="/cave/scans/{{cave|safe}}">{{cave|safe}}</a>
 </p>
 <p>
 {% for survexdirectory in cave.survexdirectory_set.all %}

templates/wallet_table.html

@@ -0,0 +1,40 @@
+
+<table width=95%>
+<tr><th>Wallet</th><th width=8%>Wallet Date</th><th>Cave</th><th>Wallet Name</th>
+
+<!-- survex file-->
+<th style="font-family: monospace; font-size: 150%;" title="Survex data">S</th>
+<th style="font-family: monospace; font-size: 150%;" title="Survex Cave Description">C</th>
+<th style="font-family: monospace; font-size: 150%;" title="Survex QMs">Q</th>
+
+
+<!-- scanned-->
+<th style="font-family: monospace; font-size: 150%;" title="Notes">N</th>
+<th style="font-family: monospace; font-size: 150%;" title="Plan">P</th>
+<th style="font-family: monospace; font-size: 150%;" title="Elevation">E</th>
+
+<th style="font-family: monospace; font-size: 150%;" title="Tunnel or Therion">T</th>
+<th style="font-family: monospace; font-size: 150%;" title="Website updated">W</th>
+
+</tr>
+{% for wallet in manywallets|dictsort:"walletname" %}
+  <tr>
+    <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
+    
+    <td style="padding:2px" >{{wallet.date}}</td>
+    <td style="padding:2px">{{wallet.cave}}</td>
+    <td style="padding:2px">{{wallet.name}}</td>
+    
+    <td style="padding:1px; background-color:{{wallet.ticks.S}}">&nbsp;</td>
+    <td style="padding:1px; background-color:{{wallet.ticks.C}}">&nbsp;</td>
+    <td style="padding:1px; background-color:{{wallet.ticks.Q}}">&nbsp;</td>
+
+    <td style="padding:1px; background-color:{{wallet.ticks.N}}">&nbsp;</td>
+    <td style="padding:1px; background-color:{{wallet.ticks.P}}">&nbsp;</td>
+    <td style="padding:1px; background-color:{{wallet.ticks.E}}">&nbsp;</td>
+
+    <td style="padding:1px; background-color:{{wallet.ticks.T}}">&nbsp;</td>
+    <td style="padding:1px; background-color:{{wallet.ticks.W}}">&nbsp;</td>
+  </tr>
+{% endfor %}
+</table>

templates/walletform.html

@@ -129,7 +129,7 @@
            title="Date of the trip in ISO format: 2020-08-17"
            placeholder="{{date}}" value="{{date}}" required /> 
     <br>        
-           <label for="cave">Cave ID</label>
+           <label for="cave">Cave ID (only needed if no survex file yet)</label>
            <input 
            label = "Cave" name = "cave" size="12"
            title="Cave id e.g. 2017-DM-01 or 1623/256"
@@ -156,10 +156,10 @@
            <label for="elevd">Elevation drawn ?</label>
            <input type="checkbox" name="elevd" id="elevd" value="True" {% if "elev drawn" in checked %}checked{% endif %}>
     <br>
-           <label for="descriptionw">Cave description written ?</label>
+           <label for="descriptionw">Cave description written (or nothing recorded) ?</label>
            <input type="checkbox" name="descriptionw" id="descriptionw" value="True" {% if "description written" in checked %}checked{% endif %}>
     <br>
-           <label for="qmsw">QMs written ?</label>
+           <label for="qmsw">QMs written (or none seen) ?</label>
            <input type="checkbox" name="qmsw" id="qmsw" value="True" {% if "qms written" in checked %}checked{% endif %}>
     <br>
            <label for="websiteupt">Website updated ?</label>
@@ -174,7 +174,7 @@
            title="List of people on the survey trip"
            placeholder="{{people}}" value="{{people}}" /> 
     <br>
-           <label for="url">URL of cave description</label>
+           <label for="url">URL of survey area (only needed if not a cave)</label>
            <input 
            label = "URL" name = "url" size ="{{urlsize}}"
            title="URL of cave description, e.g. /1623/264/264.html"

templates/yearwallets.html

@@ -0,0 +1,49 @@
+{% extends "base.html" %}
+
+{% block title %}One Year Survey scans folders (wallets){% endblock %}
+
+{% block content %}
+<h3>Wallets for {{year}} </h3>
+<p>Each wallet contains the scanned original in-cave survey notes and sketches of 
+plans and elevations. It also contains scans of centre-line survex output on which
+hand-drawn passage sections are drawn. These hand-drawn passages will eventually be 
+traced to produce Tunnel or Therion drawings and eventually the final complete cave survey.
+
+<p>See also wallets
+<ul>
+<li>per cave, e.g. <a href="/cave/scans/1623-161">1623/161</a>
+<li>per person, e.g. <a href="/wallets/person/MichaelSargent">Michael Sargent</a>
+</ul>
+
+{% include 'wallet_table.html' %}
+<br />
+<table width=95%>
+<tr><th>Wallet</th><th width=8%>Wallet Date</th><th>Wallet Name</th><th>People</th><th>Cave</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr>
+{% for wallet in manywallets|dictsort:"walletname" %}
+  <tr>
+    <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
+    
+    <td style="padding:2px">{{wallet.date}}</td>
+    <td style="padding:2px">{{wallet.name}}</td>
+    <td style="padding:2px">{{wallet.persons}}</td>
+    <td style="padding:2px">{{wallet.cave}}</td>
+    
+    <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td>
+    <td style="padding:2px">
+    {% for survexblock in wallet.survexblock_set.all %}
+      <a href="{% url "svx" survexblock.survexfile.path %}">{{survexblock}}</a>
+    {% endfor %}
+    </td>
+
+    <td style="padding:2px;  font-size: 70%;">
+    {% for drawing in wallet.drawingfile_set.all %}
+      <a href="{% url "dwgfilesingle" drawing.dwgpath %}">{{drawing.dwgpath}}</a><br>
+    {% empty %}
+        (no Tunnel drawings found: but there might be Therion drawings)
+    {% endfor %}
+    </td>
+  </tr>
+{% endfor %}
+</table>
+
+{% endblock %}

urls.py

@@ -8,7 +8,7 @@ from django.contrib import auth
 from django.urls import path, reverse, resolve
 
 from troggle.core.views import statistics, survex
-from troggle.core.views.scans import scansingle, allwallets, cavewallets
+from troggle.core.views.scans import scansingle, allscans, cavewallets, walletslistyear, walletslistperson
 from troggle.core.views.drawings import dwgallfiles, dwgfilesingle
 from troggle.core.views.uploads import dwgupload, scanupload, photoupload
 from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage 
@@ -168,13 +168,18 @@ trogglepatterns = [
     path('survexfile/<path:survex_cave>',     survex.survexcavesingle,    name="survexcavessingle"),
 
 
-# The survey scans in the wallets. This short-cuts SCANS_URL which is not actually used anywhere!
+# The survey scans in the wallets. This short-cuts SCANS_URL which is not used anymore and is defunct
-    path('survey_scans/',               allwallets,   name="allwallets"), 
+    path('survey_scans/',                    allscans,    name="allscans"), # all the scans in all wallets
     path('survey_scans/<path:path>/',        scanupload,  name="singlewallet"), # replaced singlewallet()
     path('survey_scans/<path:path>/<file>',  scansingle,  name="scansingle"), # works, but html href goes direct to /expofiles/ too
-    re_path(r'^cave/scans/(?P<cave_id>[^/]+)$',    cavewallets,  name="cavewallets"), # like allwallets, but for just one cave
+    path('cave/scans/<slug:caveid>',         cavewallets, name="cavewallets"), # like allscans, but for just one cave
     
-# The tunnel and therion drawings files pages
+# The data about the wallets themselves, not the scans inside tehm
+    path('wallets/year/<int:year>',       walletslistyear,  name="walletslistyear"), # wallets that are for a specific year, as an integer '1985'
+    re_path('wallets/person/(?P<first_name>[A-Z]*[a-z\-\'&;]*)[^a-zA-Z]*(?P<last_name>[a-z\-\']*[^a-zA-Z]*[\-]*[A-Z]*[a-zA-Z\-&;]*)/?',   walletslistperson,  name="walletslistperson"), 
+
+
+# The tunnel and therion drawings files pageswalletslistcave
     path('dwgfiles',                      dwgallfiles,       name="dwgallfiles"), 
     path('dwgfiles/',                     dwgallfiles,       name="dwgallfiles"),
     path('dwgdataraw/<path:path>',        dwgfilesingle,     name="dwgfilesingle"),