Complaints now on wallet page

core/TESTS/tests_logins.py

@@ -81,6 +81,9 @@ class PostTests(TestCase):
     def test_scan_upload(self):
         '''Expect scan upload to wallet to work on any file 
         Need to login first.
+        
+        This upload form looks for the cave, so the test fails if the database is not loaded with the cave
+        identified in the wallet
         '''
         c = self.client
         from django.contrib.auth.models import User
@@ -99,8 +102,8 @@ class PostTests(TestCase):
             for ph in [ r'test_upload_',
                     r'← 2020#00 →',
                     r'description written',
-                    r'plan not required',
+                    r'Plan not required',
-                    r'Upload scan into wallet']:
+                    r'edit settings or upload a file']:
                 phmatch    = re.search(ph, content)
                 self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")
                 

core/models/caves.py

@@ -574,6 +574,8 @@ def GetCaveLookup():
     """lookup function modelled on GetPersonExpeditionNameLookup
     repeated assignment each call, needs refactoring
     
+    Used when parsing wallets contents.json file too in views/uploads.py
+    
     Does NOT detect duplicates! Needs fixing.
     Needs to be a proper funciton that raises an exception if there is a duplicate.
     OR we could set it to return None if there are duplictes, and require the caller to 
@@ -609,6 +611,8 @@ def GetCaveLookup():
     # and Becka's email of 25 may 2020 on new kataster numbers
     
     # These might alse create more duplicate entries, so re-write it to check
+    # skip any missing keys as this gets called during tests when the database is not loaded
+    try:
         Gcavelookup["1987-02"] = Gcavelookup["267"]
         Gcavelookup["1990-01"] = Gcavelookup["171"]
         Gcavelookup["1990-02"] = Gcavelookup["172"]
@@ -733,6 +737,8 @@ def GetCaveLookup():
         Gcavelookup["homecoming"] = Gcavelookup["2018-dm-07"]
         Gcavelookup["heimkommen"] = Gcavelookup["2018-dm-07"]
         Gcavelookup["99ob02"] = Gcavelookup["1999-ob-02"]
+    except:
+        pass
     
     addmore = {}
     for id in Gcavelookup:

core/views/uploads.py

@@ -3,8 +3,10 @@ import subprocess
 import json
 import settings
 import urllib
+import operator
 
 from pathlib import Path
+from functools import reduce
 
 from django import forms
 
@@ -15,6 +17,7 @@ from django.http import HttpResponse, HttpResponseRedirect
 from django.shortcuts import render
 from django.template import Context, loader
 from django.core.files.storage import FileSystemStorage, default_storage
+from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 
 
 #from troggle import settings
@@ -27,6 +30,7 @@ from troggle.core.models.troggle import Expedition, Person, PersonExpedition
 from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
 from troggle.core.models.survex import DrawingFile
 from troggle.core.views.scans import oldwallet, walletindex
+from troggle.core.views.caves import getCave
 
 from .auth import login_required_if_public
 #from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt
@@ -35,6 +39,10 @@ from .auth import login_required_if_public
 '''
 
 todo = '''
+- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
+
+- Refactor scanupload() as it contains all the wallets 'complaints' code from the old script wallets.py
+
 - Need to validate uploaded file as being a valid image file, not a dubious script or hack
 
 - Write equivalent GPX upload form system, similar to scanupload() but in expofiles/gpslogs/
@@ -47,7 +55,6 @@ todo = '''
 
 - Enable folder creation in dwguploads or as a separate form
 
-- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
 '''
 
 class FilesForm(forms.Form):  # not a model-form, just a form-form
@@ -91,6 +98,85 @@ xlate = {"url": "description url",
         "survex": "survex file",
         }
 
+def get_complaints(complaints, waldata, svxfiles, files):
+    '''Taken from old script wallets.py and edited to make more comprehensible
+    Loads the survex files names and processes all complaints
+    '''
+    survex_complaint = ""
+
+    if waldata["survex file"]:
+        if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
+            waldata["survex file"] = [waldata["survex file"]]       
+        for svx in waldata["survex file"]:
+            svxfiles.append(svx)
+            if not (Path(settings.SURVEX_DATA) / svx).is_file():
+                file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo"
+                complaints.append(file_complaint)
+                message = f"! {file_complaint}"
+                print(message)
+                DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
+
+    if waldata["survex not required"] and waldata["survex file"] != "":
+        survex_complaint = "Survex is stated as not required and yet there is a survex file!"    
+    if not waldata["survex not required"] and waldata["survex file"] == "":
+        survex_complaint = "A survex file is required, but has not been specified!"    
+    if survex_complaint:
+        complaints.append(survex_complaint)
+    
+    # Notes required
+    if not waldata["electronic survey"]:
+        notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False)
+        notes_scanned = reduce(operator.or_, [f.endswith("note") for f in files], notes_scanned)
+        if not notes_scanned:
+            complaints.append("The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey.") 
+
+    # Plan drawing required
+    plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False)
+    plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned)
+    plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"])
+    if plan_drawing_required:
+        complaints.append("The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found.") 
+
+    # Elev drawing required
+    elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False)
+    elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned)
+    elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"])
+    if elev_drawing_required:
+        complaints.append("The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found.") 
+
+    # Electronic Surveys
+    if not waldata["electronic survey"]:
+        if elev_drawing_required or plan_drawing_required:
+            complaints.append("Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox.") 
+     
+    # Description
+    if not waldata["description written"]: 
+        complaints.append("The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done.") 
+    # QMs
+    if not waldata["qms written"]:
+        complaints.append("The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done.") 
+
+
+    # Website
+    if not waldata["website updated"]:
+        complaints.append("The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done.")
+
+    # FInd the cave, if it exists
+    try:
+        caveobject = getCave(waldata["cave"])   
+        print(f'getCave for id "{waldata["cave"]}" {caveobject}')
+        if not caveobject.url == waldata["description url"]:
+            complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.')
+    except Cave.MultipleObjectsReturned: 
+        complaints.append(f'The cave ID \'{waldata["cave"]}\' is AMBIGUOUS. Please fix it.')
+        caveobject = None
+    except ObjectDoesNotExist:
+        complaints.append(f'The cave ID \'{waldata["cave"]}\' is not recognised. Please fix it.')
+        caveobject = None
+     
+
+    return complaints, caveobject
+    
 # @login_required_if_public
 def scanupload(request, path=None):
     '''Upload scanned image files into a wallet on /expofiles
@@ -98,6 +184,9 @@ def scanupload(request, path=None):
     
     This does NOT use a Django model linked to a Django form. Just a simple Django form.
     You will find the Django documentation on forms very confusing, This is simpler.
+    
+    This subsumes much of the code which was in the old wallets.py script and so this function is very long
+    indeed and needs refactoring.
     '''
     filesaved = False
     actual_saved = []
@@ -270,22 +359,14 @@ def scanupload(request, path=None):
         if not waldata["description url"]:
             waldata["description url"]=""
         if waldata["cave"]:
-            cave =  waldata["cave"]
+            cave =  waldata["cave"] # text string
         if waldata["name"]:
             psg =  waldata["name"]
 
-        if waldata["survex file"]:
-            #print(f'+++ ${waldata["survex file"]}$ {type(waldata["survex file"])}')
-            if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
-                #print(f'+++ NOT A LIST: {waldata["survex file"]} {type(waldata["survex file"])}')
-                waldata["survex file"] = [waldata["survex file"]]
         
-        for svx in waldata["survex file"]:
+        #Survex and survex complaints
-            svxfiles.append(svx)
+        complaints, caveobject = get_complaints([], waldata, svxfiles, files)
-            if not (Path(settings.SURVEX_DATA) / svx).is_file():
+
-                    message = f"! {wallet} Incorrect survex file in wallet data: {svx} not found in LOSER repo"
-                    print(message)
-                    DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
      
         for f in checkboxes:
              if waldata[f]:
@@ -295,10 +376,12 @@ def scanupload(request, path=None):
                 'files': files, 'dirs': dirs, 'waldata': waldata, 'svxfiles': svxfiles,
                 'checked': checked,
                 'create': create, 
+                'complaints': complaints,
+                'caveobject': caveobject,
                 'people':  waldata["people"], 'peoplesize': str(len(str(waldata["people"]))),
                 'filesaved': filesaved, 'actual_saved': actual_saved }
 
-        return render(request, 'scanuploadform.html', 
+        return render(request, 'walletform.html', 
             {'form': form, 'wallet': wallet, **context, 
             'date': waldata["date"],
             'url':  waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))),
@@ -312,7 +395,7 @@ def scanupload(request, path=None):
                 'people':  "", 'peoplesize': 12,
                 'filesaved': filesaved, 'actual_saved': actual_saved }
         
-        return render(request, 'scanuploadform.html', 
+        return render(request, 'walletform.html', 
             {'form': form, 'wallet': wallet, **context, 
             'date': "",
             'url':  "", 'urlsize': 12,