Merge branch 'python3-new' of ssh://expo.survex.com/home/expo/troggle into python3-new

core/forms.py

@@ -18,7 +18,7 @@ There are other, simpler, upload forms in view/uploads.py
 Some are not used and need renovating or destroying.
 '''
 
-todo = '''Re-enable TinyMCE
+todo = '''
 '''
 
 class CaveForm(ModelForm):

core/models/caves.py

@@ -26,6 +26,14 @@ from django.shortcuts import render
 from troggle.core.models.troggle import TroggleModel, Person, Expedition, DataIssue
 from troggle.core.models.survex import SurvexStation
 from troggle.core.utils import writetrogglefile
+from troggle.core.utils import TROG
+
+# Us ethe TROG global object to cache teh cave lookup list
+Gcavelookup = TROG['caves']['gcavelookup']
+Gcave_count = TROG['caves']['gcavecount']
+
+Gcavelookup = None
+Gcave_count = None
 
 '''The model declarations for Areas, Caves and Entrances. Also LogBookENtry, QM, PersonTrip
 '''
@@ -600,8 +608,6 @@ class PersonTrip(TroggleModel):
         return f'{self.personexpedition} ({self.logbook_entry.date})'
 
 
-Gcavelookup = None
-Gcave_count = None
 def GetCaveLookup():
     """A very relaxed way of finding probably the right cave given almost any string which might serve to identify it
     

core/models/survex.py

@@ -1,6 +1,10 @@
 import os
-from urllib.parse import urljoin
 import re
+import json
+import operator
+from urllib.parse import urljoin
+from pathlib import Path
+from functools import reduce
 
 from django.db import models
 from django.conf import settings
@@ -18,7 +22,7 @@ class SurvexDirectory(models.Model):
         verbose_name_plural = "Survex directories"
 
     def __str__(self):
-        return "[SurvexDirectory:"+str(self.path)  + "-" + str(self.primarysurvexfile.path) + "-" + str(self.cave)+"]" 
+        return "[SurvexDirectory:"+str(self.path)  + " |  Primary svx:" + str(self.primarysurvexfile.path) +".svx ]" 
 
 
 class SurvexFile(models.Model):
@@ -160,6 +164,9 @@ class SurvexPersonRole(models.Model):
         return str(self.person) + " - " + str(self.survexblock) 
 
 class Wallet(models.Model):
+    '''We do not keep the JSON values in the database, we query them afresh each time,
+    but we will change this when we need to do a Django query on e.g. personame
+    '''
     fpath               = models.CharField(max_length=200)
     walletname          = models.CharField(max_length=200)
     
@@ -169,8 +176,171 @@ class Wallet(models.Model):
     def get_absolute_url(self):
         return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)}))
 
+    def get_json(self):
+        jsonfile = Path(self.fpath, 'contents.json')
+        if not Path(jsonfile).is_file():
+            #print(f'{jsonfile} is not a file')
+            return None
+        else:
+            with open(jsonfile) as json_f:
+                try:
+                    waldata = json.load(json_f)
+                except:
+                    wurl = f"/scanupload/{self.walletname}" # .replace('#', ':')     
+                    message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
+                    #print(message)
+                    raise
+
+        return waldata
+        
+    def year(self): 
+        if self.walletname[4] != "#":
+             return None
+        year = int(self.walletname[0:4])
+        if year < 1976 or year > 2050:
+            return None  
+        else:
+            return str(year)
+        
+
+    # Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it
+    def date(self):
+        if not self.get_json():
+            return None
+        jsondata = self.get_json()
+        return jsondata["date"]
+        
+    def people(self):
+        if not self.get_json():
+            return None
+        jsondata = self.get_json()
+        return jsondata["people"]
+        
+    def cave(self):
+        if not self.get_json():
+            return None
+        jsondata = self.get_json()
+        return jsondata["cave"]
+
+    def name(self):
+        if not self.get_json():
+            return None
+        jsondata = self.get_json()
+        return jsondata["name"]
+
+    def get_fnames(self):
+        '''Filenames without the suffix, i.e. without the ".jpg"
+        '''
+        dirpath = Path(settings.SCANS_ROOT, self.fpath)
+        files = []
+        if dirpath.is_dir():
+            try:
+                for f in dirpath.iterdir():
+                    if f.is_file():
+                        if f.name != 'contents.json' and f.name != 'walletindex.html':
+                            files.append(Path(f.name).stem)
+            except FileNotFoundError:
+                pass
+        return files
+
+
+    def get_ticks(self):
+        waldata = self.get_json()
+        if not waldata:
+            return {}
+        ticks = {}
+        
+        # Initially, are there any required survex files present ?
+        survexok = "red"
+        ticks["S"] = "red"
+        if waldata["survex not required"]:
+            survexok = "green"
+            ticks["S"] = "green"
+        else:
+            if waldata["survex file"]:
+                if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
+                    waldata["survex file"] = [waldata["survex file"]]
+                ngood = 0
+                nbad = 0
+                ticks["S"] = "lightblue"
+                for svx in waldata["survex file"]:
+                    if svx !="":
+                        if (Path(settings.SURVEX_DATA) / svx).is_file():
+                            ngood += 1
+                        else:
+                            nbad += 1
+                if nbad == 0 and ngood >= 1:
+                    ticks["S"] = "green"
+                if nbad >= 1 and ngood >= 1:
+                    ticks["S"] = "orange"
+                if nbad >= 1 and ngood == 0:
+                    ticks["S"] = "red"
+        
+       # Cave Description
+        if waldata["description written"]: 
+            ticks["C"] = "green"
+        else:
+            ticks["C"] = survexok
+        # QMs
+        if waldata["qms written"]:
+            ticks["Q"] = "green"
+        else:
+            ticks["Q"] = survexok    
+                
+        # Notes, Plan, Elevation; Tunnel
+        if waldata["electronic survey"]:
+            ticks["N"] = "green"
+            ticks["P"] = "green"
+            ticks["E"] = "green"
+            ticks["T"] = "green"
+        else:
+        
+            files = self.get_fnames()
+            
+            # Notes required
+            notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False)
+            notes_scanned = reduce(operator.or_, [f.endswith("notes") for f in files], notes_scanned)
+            if notes_scanned:
+                ticks["N"] = "green"
+            else:
+                ticks["N"] = "red"
+
+            # Plan drawing required
+            plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False)
+            plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned)
+            plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"])
+            if plan_drawing_required:
+                ticks["P"] = "red"
+            else:
+                ticks["P"] = "green"
+
+            # Elev drawing required
+            elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False)
+            elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned)
+            elev_scanned = reduce(operator.or_, [f.endswith("elevation") for f in files], elev_scanned)
+            elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"])
+            if elev_drawing_required:
+                ticks["E"] = "red"
+            else:
+                ticks["E"] = "green"
+
+            # Tunnel / Therion
+            if elev_drawing_required or plan_drawing_required:
+                ticks["T"] = "red"
+            else:
+                ticks["T"] = "green"
+      
+
+        # Website
+        if waldata["website updated"]:
+            ticks["W"] = "green"
+        else:
+            ticks["W"] = "red"
+  
+        return ticks
+   
     def __str__(self):
-        return str(self.walletname) + " (Wallet)"
+        return "[" + str(self.walletname) + " (Wallet)]"
 
 class SingleScan(models.Model):
     ffile    = models.CharField(max_length=200)
@@ -189,7 +359,7 @@ class SingleScan(models.Model):
 class DrawingFile(models.Model):
     dwgpath          = models.CharField(max_length=200)
     dwgname          = models.CharField(max_length=200)
-    manywallets      = models.ManyToManyField("Wallet") # implicitly links via folders to scans to SVX files
+    dwgwallets       = models.ManyToManyField("Wallet") # implicitly links via folders to scans to SVX files
     scans            = models.ManyToManyField("SingleScan")  # implicitly links via scans to SVX files
     dwgcontains      = models.ManyToManyField("DrawingFile")  # case when its a frame type
     filesize         = models.IntegerField(default=0)

core/models/troggle.py

@@ -124,11 +124,11 @@ class Person(TroggleModel):
     fullname    = models.CharField(max_length=200)
     is_vfho     = models.BooleanField(help_text="VFHO is the Vereines für Höhlenkunde in Obersteier, a nearby Austrian caving club.", default=False)
     mug_shot    = models.CharField(max_length=100, blank=True,null=True)
-    blurb = models.TextField(blank=True,null=True)
+    blurb       = models.TextField(blank=True,null=True)
     
     #href        = models.CharField(max_length=200)
     orderref    = models.CharField(max_length=200)  # for alphabetic 
-    user        = models.OneToOneField(User, null=True, blank=True,on_delete=models.CASCADE)
+    user        = models.OneToOneField(User, null=True, blank=True,on_delete=models.CASCADE) # not used now
     def get_absolute_url(self):
         return urljoin(settings.URL_ROOT,reverse('person',kwargs={'first_name':self.first_name,'last_name':self.last_name}))
 

core/utils.py

@@ -42,6 +42,10 @@ TROG = {
     },
     'issues' : {
         'logdataissues' : {}
+    },
+    'caves' : {
+        'gcavelookup' : {},
+        'gcavecount' : {}
     }
 
 }

core/views/scans.py

@@ -1,5 +1,6 @@
 import os, stat
 import re
+import datetime
 from pathlib import Path
 from urllib.parse import urljoin, unquote as urlunquote
 from urllib.request import urlopen
@@ -8,9 +9,12 @@ from django.conf import settings
 from django.shortcuts import render
 from django.http import HttpResponse
 
-from troggle.core.models.survex import Wallet, SingleScan
+from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock
+from troggle.core.models.troggle import Person
 from troggle.core.models.caves import GetCaveLookup
 from troggle.core.views.expo import getmimetype
+#from troggle.parsers.people import GetPersonExpeditionNameLookup
+
 #import parsers.surveys
 
 '''one of these views serves files as binary blobs, and simply set the mime type based on the file extension,
@@ -19,8 +23,148 @@ by looking inside the file before being served.
 
 need to check if inavlid query string is invalid, or produces multiple replies
 and render a user-friendly error page.
+
+Note that datewallet(), caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the
+manywallets dict.
 '''
+
+def populatewallet(w):
+    '''Copy survex data here just for display, not permanently
+    '''
+    survexpeople = []
+    blocks = SurvexBlock.objects.filter(scanswallet = w)
+    for b in blocks:
+        for personrole in b.survexpersonrole_set.all(): 
+            survexpeople.append(personrole.personname)
+        w.persons = list(set(survexpeople)) 
+
+def datewallet(w, earliest):
+    first = earliest
+    blocks = SurvexBlock.objects.filter(scanswallet = w)
+    for b in blocks:
+        if b.date:
+            if b.date < first:
+                first = b.date    
+    if first == earliest:
+        # no date found
+        w.date = None
+    else:
+        w.date = first
+    
+def caveifywallet(w):
+    '''Gets the cave from the list of survex files,
+    only selects one of them though. Only used for display.
+    '''
+    blocks = SurvexBlock.objects.filter(scanswallet = w)
+    for b in blocks:    
+        # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath
+        if b.survexfile.cave:
+            w.cave  = b.survexfile.cave # just gets the last one, randomly. SHould make this a list or many:many ideally
+            
+def fillblankpeople(w):
+        wp = w.people()
+        if not wp: # an -empty list
+            populatewallet(w)
+        else:
+            if len(wp) == 1:
+                nobody = wp[0].lower()
+                if  nobody == 'unknown' or nobody == 'nobody' or nobody == ' ':
+                    populatewallet(w)
+        
+def fillblankothers(w):       
+    earliest = datetime.datetime.now().date()
+    if not w.date():
+        datewallet(w, earliest)
+        
+    c = w.cave()
+    if not c:
+        caveifywallet(w)
+        
+
+def walletslistperson(request, first_name, last_name):
+    '''Page which displays a list of all the wallets for a specific person
+    HORRIBLE linear search through everything. Index and do SQL query properly
+    '''
+    # This is where we face having to re-do everything to do with names properly, rather than the horrible series of hacks over 20 years..
+    #GetPersonExpeditionNameLookup
+    def tickspersonwallet(p):
+        manywallets = []
+        wallets = Wallet.objects.all()
+        for w in wallets:
+            w.persons = w.people() # ephemeral attribute for web page
+            fillblankpeople(w)
+            if w.persons:
+                if p.fullname in w.persons:
+                    manywallets.append(w)
+                    fillblankothers(w)
+                    w.ticks = w.get_ticks() # the complaints in colour form
+        return manywallets
+    
+    try:
+        if last_name:
+            p = Person.objects.get(fullname= f'{first_name} {last_name}')
+        else: 
+            # speciall Wookey-hack
+            p = Person.objects.get(first_name= f'{first_name}')
+    except:
+        #raise
+        return render(request, 'errors/generic.html', {'message': f'Unrecognised name of a expo person: "{first_name} {last_name}"'})   
+    
+    manywallets = tickspersonwallet(p)
+
+    return render(request, 'personwallets.html', { 'manywallets':manywallets, 'settings': settings, 'person': p})
+
+
+def walletslistyear(request, year):
+    '''Page which displays a list of all the wallets in a specific year 
+    '''
+    def ticksyearwallet(year):
+        manywallets = []
+        wallets = Wallet.objects.all() 
+        for w in wallets:
+
+            if year == w.year():
+                manywallets.append(w)
+                fillblankpeople(w)
+                fillblankothers(w)
+                w.ticks = w.get_ticks() # the complaints in colour form
+            else:
+                continue
+
+        return manywallets
+        
+    if year < 1976 or year > 2050:
+        return render(request, 'errors/generic.html', {'message': 'Year out of range. Must be between 1976 and 2050'})   
+    else:
+        year = str(year)
+    #return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'})   
+
+    manywallets = ticksyearwallet(year)
  
+    return render(request, 'yearwallets.html', { 'manywallets':manywallets, 'settings': settings, 'year': year})
+
+
+
+def cavewallets(request, caveid):
+    '''Returns all the wallets for just one cave
+    '''
+    Gcavelookup = GetCaveLookup()
+    if caveid in Gcavelookup:
+        cave = Gcavelookup[caveid]
+    else:
+        return render(request,'errors/badslug.html', {'badslug': caveid})
+
+    # remove duplication. SOrting is done in the template
+    wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave)) # NB a filtered set
+    manywallets = list(wallets)
+
+    for w in manywallets:
+        fillblankpeople(w)
+        fillblankothers(w)   
+        w.ticks = w.get_ticks() # the complaints in colour form
+    return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave})
+
+
 def oldwallet(request, path):
     '''Now called only for non-standard wallet structures for pre-2000 wallets
     '''
@@ -59,28 +203,13 @@ def scansingle(request, path, file):
         return render(request, 'errors/generic.html', {'message': message})   
 
 
-def allwallets(request):
+def allscans(request):
     '''Returns all the wallets in the system, we would like to use
     the Django queryset SQL optimisation https://docs.djangoproject.com/en/3.2/ref/models/querysets/#prefetch-related
     to get the related singlescan and survexblock objects but that requires rewriting this to do the query on those, not on
     the wallets
     '''
-    manywallets = Wallet.objects.all()
+    manywallets = Wallet.objects.all() # NB all of them
     # manywallets = Wallet.objects.all().prefetch_related('singlescan') fails as the link is defined on 'singlescan' not on 'wallet'
     return render(request, 'manywallets.html', { 'manywallets':manywallets, 'settings': settings })
     
-def cavewallets(request, cave_id):
-    '''Returns all the wallets for just one cave, 
-    '''
-        
-    Gcavelookup = GetCaveLookup()
-    if cave_id in Gcavelookup:
-        cave = Gcavelookup[cave_id]
-    else:
-        return render(request,'errors/badslug.html', {'badslug': cave_id})
-
-    # remove duplication. SOrting is done in the template
-    wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave))
-    manywallets = list(wallets)
-
-    return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave})

core/views/statistics.py

@@ -51,7 +51,7 @@ def pathsreport(request):
         "SURVEX_DATA" : str( settings.SURVEX_DATA),
         "SCANS_ROOT" : str( settings.SCANS_ROOT),
 #       "SURVEYS" : str( settings.SURVEYS),
-        "SCANS_URL" : str( settings.SCANS_URL),
+#       "SCANS_URL" : str( settings.SCANS_URL),
         "SURVEXPORT" : str( settings.SURVEXPORT),
         "DRAWINGS_DATA" : str( settings.DRAWINGS_DATA),
         "URL_ROOT" : str( settings.URL_ROOT)
@@ -88,7 +88,7 @@ def pathsreport(request):
         "SURVEX_DATA" : type(settings.SURVEX_DATA),
         "SCANS_ROOT" : type(settings.SCANS_ROOT),
 #       "SURVEYS" : type(settings.SURVEYS),
-        "SCANS_URL" : type(settings.SCANS_URL),
+#       "SCANS_URL" : type(settings.SCANS_URL),
         "SURVEXPORT" : type(settings.SURVEXPORT),
         "DRAWINGS_DATA" : type(settings.DRAWINGS_DATA),
         "URL_ROOT" : type(settings.URL_ROOT)

core/views/uploads.py

@@ -23,12 +23,12 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 #from troggle import settings
 from troggle.parsers.imports import import_caves, import_people, import_surveyscans
 from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
-from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml
+from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml, CopyWalletData
 # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
 from troggle.core.models.troggle import DataIssue
 from troggle.core.models.troggle import Expedition, Person, PersonExpedition
 from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
-from troggle.core.models.survex import DrawingFile
+from troggle.core.models.survex import DrawingFile, Wallet
 from troggle.core.views.scans import oldwallet, walletindex
 from troggle.core.views.caves import getCave
 
@@ -93,13 +93,16 @@ xlate = {"url": "description url",
         "electronic": "electronic survey",
         "pland": "plan drawn",
         "elevd": "elev drawn",
-        "psg": "name",
+        "psg": "name", # a name for this wallet
         "survex": "survex file",
         }
 
 def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
     '''Taken from old script wallets.py and edited to make more comprehensible
     Loads the survex files names and processes all complaints
+    
+    All needs to be restructred to use the get_ticks() function on the Wallets class in core/models/survex.py 
+    which does the same thing
     '''
     # Date
     if not waldata["date"]:
@@ -115,13 +118,14 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
         if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
             waldata["survex file"] = [waldata["survex file"]]       
         for svx in waldata["survex file"]:
-            svxfiles.append(svx)
-            if not (Path(settings.SURVEX_DATA) / svx).is_file():
-                file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo"
-                complaints.append(file_complaint)
-                message = f"! {file_complaint}"
-                print(message)
-                DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
+            if svx !="":
+                svxfiles.append(svx)
+                if not (Path(settings.SURVEX_DATA) / svx).is_file():
+                    file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo"
+                    complaints.append(file_complaint)
+                    message = f"! {file_complaint}"
+                    print(message)
+                    DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
 
     if waldata["survex not required"] and waldata["survex file"] != "":
         survex_complaint = "Survex is stated as not required and yet there is a survex file!"    
@@ -133,20 +137,21 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
     # Notes required
     if not waldata["electronic survey"]:
         notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False)
-        notes_scanned = reduce(operator.or_, [f.endswith("note") for f in files], notes_scanned)
+        notes_scanned = reduce(operator.or_, [Path(f).stem.endswith("notes") for f in files], notes_scanned)
         if not notes_scanned:
             complaints.append("The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey.") 
 
     # Plan drawing required
     plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False)
-    plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned)
+    plan_scanned = reduce(operator.or_, [Path(f).stem.endswith("plan") for f in files], plan_scanned)
     plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"])
     if plan_drawing_required:
         complaints.append("The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found.") 
 
     # Elev drawing required
     elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False)
-    elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned)
+    elev_scanned = reduce(operator.or_, [Path(f).stem.endswith("elev") for f in files], elev_scanned)
+    elev_scanned = reduce(operator.or_, [Path(f).stem.endswith("elevation") for f in files], elev_scanned)
     elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"])
     if elev_drawing_required:
         complaints.append("The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found.") 
@@ -290,6 +295,21 @@ def scanupload(request, path=None):
                 with open(contents_path, "w") as jfile:
                     json.dump(wd, jfile, indent = 1)
                 # print(f'--- FINISHED saving to JSON\n')
+                
+                # This copies the new data to the drawings repo and commit it
+                # needs the troggle object wallet, not a string
+ 
+                try:
+                    w, created = Wallet.objects.get_or_create(walletname=wallet)
+                    print(f'wallet string {wallet}, wallet object {w} created new?: {created}')   
+                    if created:
+                        w.fpath = Path(settings.SCANS_ROOT, wallet[0:4], wallet)
+                        w.save()
+                    CopyWalletData(w)
+                except:
+                     print(f'wallet string {wallet}, FAIL TO GET WALLET OBJECT, maybe we need to create it ?')
+                     raise
+               
             else:
                 print(f'--- INVALID JSON Update form submitted')
                 print(formj.errors)