Wallet editor into separate file

core/TESTS/test_imports.py

@@ -55,7 +55,10 @@ class SimpleTest(SimpleTestCase):
         from troggle.parsers.people import GetPersonExpeditionNameLookup
 
     def test_import_views_uploads(self):
-        from troggle.core.views.uploads import dwgupload, scanupload
+        from troggle.core.views.uploads import dwgupload
+
+    def test_import_views_walletedit(self):
+        from troggle.core.views.wallets import walletedit
 
     def test_import_parsers_QMs(self):
         from troggle.core.models.logbooks import QM

core/TESTS/tests_logins.py

@@ -73,7 +73,7 @@ class FixturePageTests(TestCase):
 
 
 class PostTests(TestCase):
-    """Tests scanupload form"""
+    """Tests walletedit form"""
 
     fixtures = ["auth_users"]
 
@@ -120,7 +120,7 @@ class PostTests(TestCase):
 
         with open("core/fixtures/test_upload_file.txt", "r") as testf:
             response = self.client.post(
-                f"/scanupload/{testyear}:00", data={"name": "test_upload_file.txt", "uploadfiles": testf}
+                f"/walletedit/{testyear}:00", data={"name": "test_upload_file.txt", "uploadfiles": testf}
             )
             content = response.content.decode()
             self.assertEqual(response.status_code, 200)

core/models/wallets.py

@@ -40,7 +40,7 @@ class Wallet(models.Model):
         fp = Path(self.fpath)
         wname = fp.name
         wyear = fp.parent.name
-        wurl = f"/scanupload/{self.walletname}"  # .replace('#', ':')
+        wurl = f"/walletedit/{self.walletname}"  # .replace('#', ':')
 
         jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
         if not Path(jsonfile).is_file():

core/views/scans.py

@@ -253,7 +253,7 @@ def cavewallets(request, caveid):
                     # print(f' - Found one ! {z.walletname=} {zcaveid=}')
                     wallets.add(z)
             else:
-                wurl = f"/scanupload/{z.walletname.replace('#',':')}"
+                wurl = f"/walletedit/{z.walletname.replace('#',':')}"
                 message = f" ! In {z.walletname} there is an unrecognised cave name '{zcaveid}' (out of {len(Gcavelookup):,} cave names and aliases)"
                 print(message)
                 DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)

core/views/uploads.py

@@ -1,31 +1,16 @@
-import datetime
-import json
-import os
-import re
-import socket
 import subprocess
-import urllib
 from pathlib import Path
 
 from django import forms
-from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
 from django.core.files.storage import FileSystemStorage
-from django.http import HttpResponseRedirect
 from django.shortcuts import render
 
 import settings
-from troggle.core.models.caves import Cave
+from troggle.core.models.survex import DrawingFile
-from troggle.core.models.logbooks import LogbookEntry  # , PersonLogEntry
-from troggle.core.models.survex import DrawingFile, SurvexBlock, SurvexFile, SurvexPersonRole
-from troggle.core.models.wallets import Wallet
 
 # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
-from troggle.core.models.troggle import DataIssue, Expedition
-from troggle.core.views.caves import getCave
-from troggle.core.views.scans import caveifywallet, oldwallet
 
 # from troggle import settings
-from troggle.parsers.scans import contentsjson
 
 from .auth import login_required_if_public
 
@@ -37,801 +22,30 @@ from .auth import login_required_if_public
 todo = """
 - Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
 
-- Refactor scanupload() as it contains all the wallets 'complaints' code from the old script wallets.py
-
 - Need to validate uploaded file as being a valid image file, not a dubious script or hack
 
-- Write equivalent GPX upload form system, similar to scanupload() but in expofiles/gpslogs/
+- Write equivalent GPX upload form system, similar to walletedit() but in expofiles/gpslogs/
   Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
   
 - Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says tunnel is only mostly correct XML
 
-- Validate image files using a magic recogniser in scanupload() https://pypi.org/project/reportlab/ or 
+- Validate image files using a magic recogniser in walletedit() https://pypi.org/project/reportlab/ or 
   https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file
 
 - Enable folder creation in dwguploads or as a separate form
 
 """
-WALLET_BLANK_JSON = {
-    "cave": "",
-    "date": "",
-    "free text": "",
-    # "description url": "1623/NNN",
-    "description written": False,
-    "electronic survey": False,
-    "elev drawn": False,
-    "elev not required": False,
-    "name": "",
-    "people": ["Unknown"],
-    "plan drawn": False,
-    "plan not required": False,
-    "qms written": False,
-    "survex file": [],
-    "survex not required": False,
-    "website updated": False,
-}
-
 
 class FilesForm(forms.Form):  # not a model-form, just a form-form
     uploadfiles = forms.FileField()
 
-
 class FilesRenameForm(forms.Form):  # not a model-form, just a form-form
     uploadfiles = forms.FileField()
     renameto = forms.CharField(strip=True, required=False)
 
-
-class WalletGotoForm(forms.Form):  # not a model-form, just a form-form
-    walletgoto = forms.CharField(strip=True, required=False)
-
-
 class TextForm(forms.Form):  # not a model-form, just a form-form
     photographer = forms.CharField(strip=True)
 
-
-class WalletForm(forms.Form):  # not a model-form, just a form-form
-    descriptionw = forms.CharField(strip=True, required=False)
-    people = forms.CharField(strip=True, required=False)
-    survexnr = forms.CharField(strip=True, required=False)
-    qmsw = forms.CharField(strip=True, required=False)
-    date = forms.CharField(strip=True, required=True)  # the only required field
-    websiteupt = forms.CharField(strip=True, required=False)
-    elevnr = forms.CharField(strip=True, required=False)
-    cave = forms.CharField(strip=True, required=False)
-    psg = forms.CharField(strip=True, required=False)
-    freetext = forms.CharField(strip=True, required=False)
-    plannr = forms.CharField(strip=True, required=False)
-    electronic = forms.CharField(strip=True, required=False)
-    pland = forms.CharField(strip=True, required=False)
-    elevd = forms.CharField(strip=True, required=False)
-    # url = forms.CharField(strip=True, required=False)
-    survex = forms.CharField(strip=True, required=False)
-
-
-xlate = {
-    # "url": "description url",
-    "descriptionw": "description written",
-    "people": "people",
-    "date": "date",
-    "cave": "cave",
-    "plannr": "plan not required",
-    "survexnr": "survex not required",
-    "qmsw": "qms written",
-    "elevnr": "elev not required",
-    "websiteupt": "website updated",
-    "electronic": "electronic survey",
-    "pland": "plan drawn",
-    "elevd": "elev drawn",
-    "psg": "name",  # a name for this wallet
-    "freetext": "free text",
-    "survex": "survex file",
-}
-
-
-def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
-    """Taken from old script wallets.py and edited to make more comprehensible
-    Loads the survex files names and processes all complaints
-
-    """
-    # If skipping through the wallets on the upload form, the wallet may not yet exist
-    try:
-        w = Wallet.objects.get(walletname=wallet)
-    except ObjectDoesNotExist:
-        return None, None
-
-    # Date
-    if not waldata["date"]:
-        complaints.append(
-            "A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one."
-        )
-
-    # People
-    if (
-        not waldata["people"]
-        or waldata["people"] == ["NOBODY"]
-        or waldata["people"] == ["Unknown"]
-        or waldata["people"] == [""]
-    ):
-        complaints.append(
-            "Someody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data."
-        )
-
-    # survex, but get_ticks has already done much of this ??
-    survex_complaint = ""
-
-    if waldata["survex file"]:
-        if not type(waldata["survex file"]) == list:  # a string also is a sequence type, so do it this way
-            waldata["survex file"] = [waldata["survex file"]]
-        for sx in waldata["survex file"]:
-            # this logic appears in several places, inc get_ticks(). Refactor.
-            if sx != "":
-                if Path(sx).suffix.lower() != ".svx":
-                    sx = sx + ".svx"
-                svxfiles.append(sx)
-                if not (Path(settings.SURVEX_DATA) / sx).is_file():
-                    file_complaint = f"{wallet} Incorrect survex file name. File {sx} was not found in LOSER repo"
-                    complaints.append(file_complaint)
-                    message = f"! {file_complaint}"
-                    print(message)
-                    DataIssue.objects.update_or_create(
-                        parser="scans", message=message, url=wurl
-                    )  # set URL to this wallet folder
-                else:
-                    try:
-                        sxpath = str(Path(sx).with_suffix(""))
-                        SurvexFile.objects.get(path=sxpath)
-                    except MultipleObjectsReturned:
-                        # can happen if connecting a wallet to a survex file.. i think..
-                        QSsvxfiles = SurvexFile.objects.filter(path=sxpath)
-                        for s in QSsvxfiles:
-                            print(s.path, s.cave, s.survexdirectory)
-                        # QSsvxfiles[0]  # dont' know how this happened, fix later..
-                    except:
-                        file_complaint = (
-                            f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.."
-                        )
-                        complaints.append(file_complaint)
-                        message = f"! {file_complaint}"
-                        print(message)
-                        DataIssue.objects.update_or_create(
-                            parser="scans", message=message, url=wurl
-                        )  # set URL to this wallet folder
-
-    if waldata["survex not required"] and waldata["survex file"] != [""]:
-        survex_complaint = (
-            f'Survex is stated as not required and yet there is a survex file! ({waldata["survex file"]})'
-        )
-    if not waldata["survex not required"] and waldata["survex file"] == [""]:
-        survex_complaint = "A survex file is required, but has not been specified!"
-    if survex_complaint:
-        complaints.append(survex_complaint)
-
-    ticks = w.get_ticks()
-
-    # Notes required
-    if ticks["N"] != "green":
-        complaints.append(
-            "The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey."
-        )
-
-    # Plan drawing required
-    if ticks["P"] != "green":
-        complaints.append(
-            "The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found."
-        )
-
-    # Elev drawing required
-    if ticks["E"] != "green":
-        complaints.append(
-            "The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found."
-        )
-
-    # ETherion
-    if ticks["T"] != "green":
-        complaints.append(
-            "Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox."
-        )
-
-    # Description
-    if not waldata["description written"]:
-        complaints.append(
-            "The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done."
-        )
-    # QMs
-    if not waldata["qms written"] and w.year() and int(w.year()) >= 2015:
-        complaints.append(
-            "The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done."
-        )
-
-    # Website
-    if not waldata["website updated"]:
-        complaints.append(
-            "The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done."
-        )
-
-    # Find the cave, if it exists
-    if waldata["cave"]:
-        try:
-            caveid = waldata["cave"]
-            if type(caveid) is list:
-                for i in caveid:
-                    i = i.replace("/", "-")
-                    caveobject = getCave(i)  # only the last one gets recorded.. ouch.
-            else:
-                caveid = caveid
-                caveobject = getCave(caveid)
-            print(f'getCave for id "{waldata["cave"]}" {caveobject}')
-            # if not caveobject.url == waldata["description url"]:
-            # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.')
-        except Cave.MultipleObjectsReturned:
-            complaints.append(f'The cave ID \'{waldata["cave"]}\' is AMBIGUOUS. Please fix it.')
-            caveobject = None
-        except ObjectDoesNotExist:
-            complaints.append(f'The cave ID \'{waldata["cave"]}\' is not recognised. Please fix it.')
-            caveobject = None
-    else:
-        complaints.append(
-            'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"'
-        )
-        caveobject = None
-
-    return complaints, caveobject
-
-
-# @login_required_if_public
-def scanupload(request, path=None):
-    """Upload scanned image files into a wallet on /expofiles
-    Also display AND EDIT the contents.json data in the wallet.
-
-    This is the main wallet display and edit page.
-
-    The Wallet object and the contents.json file are created when the user
-    first uploads files.
-
-    This does NOT use a Django model linked to a Django form. Just a simple Django form.
-    You will find the Django documentation on forms very confusing,
-    as it covers many very differnet things we do not need. This is simpler.
-
-    This subsumes much of the code which was in the old wallets.py script and so this function is very long
-    indeed and needs refactoring.
-
-    REWRITE bits using the ticklist, dateify, caveify, populate etc utility functions in core.view.scans.py
-    """
-    git = settings.GIT
-    filesaved = False
-    actual_saved = []
-
-    def read_json():
-        """Read JSON from the wallet metadata file in the repo
-        or fills with blank data if that files can't be read
-
-        Should sanitise to ensure no spurious backslashes e.g. in windows style paths"""
-        waldata = {}
-        if contents_path.is_file():
-            with open(contents_path) as json_file:
-                try:
-                    waldata = json.load(json_file)
-                except:
-                    message = f"! {wallet} Failed to load {contents_path} JSON file"
-                    print(message)
-                    DataIssue.objects.create(parser="scans", message=message, url=wurl)  # set URL to this wallet folder
-                    raise
-        else:  # no JSON file exists
-            print("--- No JSON exists, so creating blank copy")
-            waldata = WALLET_BLANK_JSON.copy()
-        if not waldata["survex file"]:
-            try:
-                w = Wallet.objects.get(walletname=wallet)
-                b = SurvexBlock.objects.filter(scanswallet=w)
-                waldata["survex file"] = []
-                for bsf in b:
-                    waldata["survex file"].append(bsf.survexfile.path)
-            except:
-                print(f"--- No wallet {wallet} exists in database")
-        return waldata
-
-    def save_json(jsondict):
-        # print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}')
-        if not os.path.exists(contents_path.parent):
-            print("--- No wallet directory in :drawings: repo, so creating it")
-            os.makedirs(contents_path.parent)
-
-        with open(contents_path, "w") as jfile:
-            json.dump(jsondict, jfile, indent=1)
-        # print(f'--- FINISHED saving to JSON at {contents_path}')
-
-    def make_wallet(walletname):
-        """We need a wallet Object so that the django template stuff can find the files"""
-        try:
-            w, created = Wallet.objects.get_or_create(walletname=walletname)
-            print(f"--- Wallet string {walletname}, wallet object {w} created new?: {created}")
-            if created:
-                w.fpath = Path(settings.SCANS_ROOT, walletname[0:4], walletname)
-                w.save()
-        except:
-            print(f"!-- Wallet string {walletname}, FAIL TO GET or create WALLET OBJECT")
-            raise
-        return w
-
-    def commit_json(waldata):
-        destfolder = contents_path.parent
-        dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
-        if dr_add.returncode != 0:
-            msgdata = (
-                "Ask a nerd to fix this.\n--"
-                + dr_add.stderr
-                + "\n--"
-                + dr_add.stdout
-                + "\n--return code: "
-                + str(dr_add.returncode)
-            )
-            message = (
-                f"CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n" + msgdata
-            )
-            print(message)
-            return render(request, "errors/generic.html", {"message": message})
-        else:
-            if socket.gethostname() != "expo":
-                comment = f"on dev machine '{socket.gethostname()}' "
-            else:
-                comment = ""
-            if "cave" in waldata:
-                label = waldata["cave"]
-            else:
-                if "name" in waldata:
-                    label = waldata["name"]
-                else:
-                    label = ""
-
-            dr_commit = subprocess.run(
-                [git, "commit", "-m", f"JSON update wallet {wallet} {label} {comment}"],
-                cwd=destfolder,
-                capture_output=True,
-                text=True,
-            )
-            # This produces return code = 1 if it commits OK
-            if dr_commit.returncode != 0:
-                msgdata = (
-                    "Ask a nerd to fix this.\n\n"
-                    + dr_commit.stderr
-                    + "\n\n"
-                    + dr_commit.stdout
-                    + "\n\nreturn code: "
-                    + str(dr_commit.returncode)
-                )
-                message = (
-                    f"Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n"
-                    + msgdata
-                )
-                print(message)
-                return render(request, "errors/generic.html", {"message": message})
-
-    def get_logbook_trips():
-        return None
-
-    checkboxes = [
-        "description written",
-        "survex not required",
-        "qms written",
-        "website updated",
-        "plan not required",
-        "plan drawn",
-        "elev not required",
-        "elev drawn",
-        "electronic survey",
-    ]
-    if path:
-        wallet = urllib.parse.unquote(path)
-    else:
-        wallet = "2022#00"  # improve this later
-
-    year = wallet[:4]
-    try:
-        if wallet[4] != "#" and wallet[4] != ":":
-            # print(f'! - FORM scanupload - {wallet[4]} unurlencoded {unquote(wallet)[4]}')
-            #  print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET')
-            return oldwallet(request, path)
-    except:
-        # if nonumeric wallet name for example
-        return oldwallet(request, path)
-
-    if str(wallet).lower().endswith("indexpages"):
-        # print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET')
-        return walletindex(request, path)
-
-    if not re.match("(19|20)\d\d[:#]\d\d", wallet):
-        wallet = "2022:00"  # improve this later
-    # print(f'! - FORM scanupload - start {wallet}')
-
-    if path:
-        pass
-        # print(f'! - FORM scanupload - start wallet:{wallet}: path:{path}:')
-    if int(year) < 1977:
-        year = "1977"
-    if int(year) > 2050:
-        year = "2050"
-    nexty = f"{int(year)+1}"
-    prevy = f"{int(year)-1}"
-
-    wnumber = wallet[5:]
-    next = f"{int(wnumber)+1:02d}"
-    prev = f"{int(wnumber)-1:02d}"
-
-    if int(wnumber) == 0:
-        prev = f"{int(wnumber):02d}"
-
-    wurl = f"/scanupload/{wallet}".replace("#", ":")
-    wallet = wallet.replace(":", "#")
-    dirpath = Path(settings.SCANS_ROOT, year, wallet)
-    contents_path = Path(settings.DRAWINGS_DATA, "walletjson") / year / wallet / contentsjson
-
-    form = FilesForm()
-
-    if request.method == "POST":
-        if "psg" in request.POST:  # handle metadata form
-            formj = WalletForm(request.POST)
-            # Beware. All fields returned as strings. Must re-type them as lists etc. before using or re-saving
-            # Also lots of hassle with lists of strings interpreted as a single string
-            # Unset checkboxes do not return any value, checked ones return "True". So need initialising to False
-            if formj.is_valid():
-                posted = request.POST.copy()
-                posted.pop("csrfmiddlewaretoken")  # discard this
-                wd = WALLET_BLANK_JSON.copy()
-                for f in checkboxes:
-                    wd[f] = False
-                    # print(f'--- wd ${f}$  - {wd[f]}')
-                for f in posted:
-                    wd[xlate[f]] = posted[f].replace("'", '"')
-
-                    if posted[f] == "True":
-                        wd[xlate[f]] = True
-
-                wd["people"] = wd["people"][1:-1].replace('"', "").split(",")
-                for i, elem in enumerate(wd["people"]):
-                    wd["people"][i] = elem.strip()
-
-                # print(f'--- ${wd["survex file"]}$  - {type(wd["survex file"])}')
-                if wd["survex file"]:  # allow for no survex file at all
-                    if wd["survex file"][0] == "[":
-                        wd["survex file"] = wd["survex file"][1:-1]
-                    wd["survex file"] = wd["survex file"].replace('"', "").split(",")
-                    for i, elem in enumerate(wd["survex file"]):
-                        wd["survex file"][i] = elem.strip()
-                    # print(f'--- {wd["survex file"]} - {type(wd["survex file"])}')
-
-                save_json(wd)
-                walletobject = make_wallet(wallet)
-                commit_json(wd)
-
-            else:
-                print("--- INVALID JSON Update form submitted")
-                print(formj.errors)
-                return render(request, "errors/generic.html", {"message": formj.errors})
-
-        elif (
-            "walletgoto" in request.POST
-        ):  # not editing wallet data or uploading a file.. going direct to a named wallet
-            formg = WalletGotoForm(request.POST, request.FILES)
-            if formg.is_valid():
-                walletgoto = request.POST["walletgoto"]
-
-                return HttpResponseRedirect(f'/scanupload/{walletgoto.replace("#",":")}')
-
-        else:  # not editing wallet data, uploading a file. But should not overwrite metadata at all.
-            form = FilesForm(request.POST, request.FILES)
-
-            if form.is_valid():
-                # print(f'--- FORM scanupload multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}')
-                multiple = request.FILES.getlist("uploadfiles")
-                fs = FileSystemStorage(os.path.join(dirpath))  # creates wallet folder if necessary
-
-                waldata = read_json()
-                actual_saved = []
-                if multiple:
-                    for f in multiple:
-                        try:  # crashes in Django os.chmod call if on WSL, but does save file!
-                            saved_filename = fs.save(f.name, content=f)
-                        except:
-                            print(f"\n !! Permissions failure ?! on attempting to save scanfile {f.name}")
-                            if "saved_filename" in locals():
-                                if saved_filename.is_file():
-                                    actual_saved.append(saved_filename)
-                                    # print(f'! - FORM scanupload multiple {actual_saved}')
-                                    filesaved = True
-                    # print(f'--- FORM scanupload multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}')
-                    save_json(waldata)
-                    walletobject = make_wallet(wallet)
-                    commit_json(waldata)
-    #
-    # Not a POST, so a GET starts here. And also control gets here after a POST is processed.
-    #
-    files = []
-    dirs = []
-    # print(f'! - FORM scanupload - start {wallet} {dirpath}')
-    if dirpath.is_dir():
-        create = False  # wallet exists because folder exists, even if nothing in it
-        try:
-            for f in dirpath.iterdir():
-                if f.is_dir():
-                    for d in f.iterdir():
-                        dirs.append(f"{f.name}/{d.name}")
-                if f.is_file():
-                    files.append(f.name)
-        except FileNotFoundError:
-            files.append(
-                "(No wallet yet. It would be created if you upload a scan and then save the form with a date.)"
-            )
-    else:
-        create = True
-
-    if len(files) > 0:
-        files = sorted(files)
-
-    if dirs:
-        dirs = sorted(dirs)
-    try:
-        waldata = read_json()
-    except:
-        message = f"Nasty failure in parsing wallets metadata in {contents_path}. Probably backslash not forward slash in filename path"
-        return render(request, "errors/generic.html", {"message": message})
-
-    jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wallet[0:4] / wallet / "contents.json"
-    # print(f'! - FORM scanupload - jsonfile {jsonfile}')
-    if not Path(jsonfile).is_file():
-        metadataurl = ""
-    else:
-        metadataurl = Path("/dwgdataraw", "walletjson") / wallet[0:4] / wallet.replace("#", ":") / "contents.json"
-    psg = ""
-    freetext = ""
-    chkplannr = ""
-    chkpland = ""
-    svxfiles = []
-    trips = []
-    checked = {}
-    context = {}
-    if waldata:  # should always be true as populated by blank data if json file doesn't exist
-
-        # if not type(waldata["people"])==list:
-        # if waldata["people"][0] == '"':
-        # waldata["people"] = waldata["people"][1:-1]
-        # waldata["people"] = list(waldata["people"])
-
-        if (
-            not waldata["date"]
-            or not waldata["people"]
-            or waldata["people"] == ["Unknown"]
-            or waldata["people"] == [""]
-            or waldata["cave"] == ""
-        ):  # json file does not exist, blank data, or people not typed into JSON file
-            # refactor into separate functions for no date set or no people set or no cave set
-            # print(f'No date set')
-            print(f"\n - Incomplete, empty or default wallet data {wallet} {waldata=}")
-            refs = []
-            dates = []
-            team = []
-            caverefs = []
-            caves = []
-            names = []
-            svxf = ""
-            if waldata["survex file"]:
-                if not type(waldata["survex file"]) == list:  # a string also is a sequence type, so do it this way
-                    waldata["survex file"] = [waldata["survex file"]]
-            for svxf in waldata["survex file"]:
-                print(f" - {svxf=}")
-                if svxf:
-                    svx = Path(svxf)
-                    if svx.suffix.lower() != ".svx":
-                        svx = svx.with_suffix(".svx")
-                    f = Path(settings.SURVEX_DATA) / svx
-                    if f.is_file():
-                        path = svx.parent / svx.stem
-                        # print(f' - {path=}')
-                        try:
-                            svxfile = SurvexFile.objects.get(path=path)
-
-                            print(f" - {svxfile=}")
-                            if svxfile.cave:
-                                caves.append(svxfile.cave)
-                                caverefs.append(svxfile.cave.reference())
-                            blocks = SurvexBlock.objects.filter(survexfile=svxfile)
-                            for b in blocks:
-                                print(f" - - {b=} {b.scanswallet=} {b.date=}")
-                                if b.scanswallet:
-                                    refs.append(b.scanswallet)
-                                    if b.scanswallet.walletname == wallet:
-                                        if b.date:
-                                            dates.append(b.date)
-                                        if b.name != b.title:
-                                            names.append(str(b.name) + "|" + str(b.title))
-                                        else:
-                                            names.append(str(b.name))
-                                        # we can use the people, across all blocks that have this *ref
-                                        QSpeople = SurvexPersonRole.objects.filter(survexblock=b)
-                                        print(f" - - {QSpeople=}")
-                                        for p in QSpeople:
-                                            print(f" - - {p.personname} ")
-                                            team.append(p.personname)
-                                    # else:
-                                    # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}')
-                        except:
-                            message = "Specified survex file not found - database may be empty, or this survex file is not *included anywhere."
-                            # return render(request, 'errors/generic.html', {'message': message})
-                            pass
-
-                        if dates:
-                            waldata["date"] = min(dates).isoformat()
-                        print(f" - - {team=} ")
-                        team = list(set(team))
-                        waldata["people"] = team
-
-                        caverefs = list(set(caverefs))
-                        caves = list(set(caves))
-
-                        if len(caverefs) == 1:
-                            waldata["cave"] = caverefs[0]
-                            print(f" - Setting wallet cave to {caverefs[0]}")
-                            # waldata["description url"] = caves[0]
-                        elif len(caverefs) == 0:
-                            waldata["cave"] = ""
-                            # waldata["description url"] = ""
-                            print(f" - No caves in this wallet {wallet}. ")
-                        else:
-                            waldata["cave"] = "several caves"
-                            # waldata["description url"] = "several.."
-                            print(
-                                f" - More than one Cave {caves} in this wallet {wallet}. Not managed in this troggle release."
-                            )
-                        if len(names) == 1:
-                            if waldata["name"] == "":
-                                waldata["name"] = names[0]
-                            print(f" - Setting wallet name to {names[0]}")
-                        elif len(names) == 0:
-                            waldata["name"] = ""
-                            print(" - Setting wallet name blank")
-                        else:
-                            waldata["name"] = f"several, please edit: {names}"
-                            print(
-                                f" - More than one block name is relevant {names} in this wallet {wallet}. Not managed in this troggle release."
-                            )
-
-        if "cave" in waldata:
-            cave = waldata["cave"]  # text string
-        else:
-            cave = ""
-        if waldata["name"]:
-            psg = waldata["name"]
-        if "free text" in waldata:
-            freetext = waldata["free text"]
-
-        # find trips and survex files of the same date
-        if waldata["date"]:
-            datestr = waldata["date"].replace(".", "-")
-            try:
-                samedate = datetime.date.fromisoformat(datestr)
-            except ValueError:
-                # probably a single digit day number. HACKUS MAXIMUS.
-                # clearly we need to fix this when we first import date strings..
-                datestr = datestr[:-1] + "0" + datestr[-1]
-                print(f" - {datestr=} ")
-                try:
-                    samedate = datetime.date.fromisoformat(datestr)
-                except:
-                    try:
-                        samedate = datetime.date.fromisoformat(datestr[:10])
-                    except:
-                        samedate = None
-
-            thisexpo = Expedition.objects.get(year=int(year))
-            if samedate:
-                svxothers = SurvexBlock.objects.filter(date=samedate)
-                trips = LogbookEntry.objects.filter(date=samedate)
-            else:
-                svxothers = None
-                trips = None
-
-        else:
-            svxothers = None
-            trips = None
-
-        # Survex and survex complaints, comes from json file on disc, not as pre-populated as above
-        complaints, caveobject = get_complaints([], waldata, svxfiles, files, wallet, wurl)
-        # print(f' - {caveobject=}')
-
-        for f in checkboxes:
-            if waldata[f]:
-                checked[f] = "checked"
-
-        survexsize = str(min(len(str(waldata["survex file"])), 46))
-
-        try:
-            thiswallet = Wallet.objects.get(walletname=wallet)
-            caveifywallet(thiswallet)
-            thiswallet.ticks = thiswallet.get_ticks()  # the complaints in colour form
-            # fixsurvextick(thiswallet, thiswallet.ticks)
-            print(thiswallet)
-        except:
-            thiswallet = None
-        context = {
-            "year": year,
-            "prev": prev,
-            "next": next,
-            "prevy": prevy,
-            "nexty": nexty,
-            "files": files,
-            "dirs": dirs,
-            "waldata": waldata,
-            "svxfiles": svxfiles,
-            "checked": checked,
-            "trips": trips,
-            "manywallets": [thiswallet],
-            "svxothers": svxothers,
-            "create": create,
-            "metadataurl": metadataurl,
-            "complaints": complaints,
-            "caveobject": caveobject,
-            "people": waldata["people"],
-            "peoplesize": str(len(str(waldata["people"]))),
-            "filesaved": filesaved,
-            "actual_saved": actual_saved,
-        }
-
-        return render(
-            request,
-            "walletform.html",
-            {
-                "form": form,
-                "wallet": wallet,
-                **context,
-                "date": waldata["date"],
-                #'url':  waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))),
-                "survex": waldata["survex file"],
-                "survexsize": survexsize,
-                "cave": cave,
-                "psg": psg,
-                "freetext": freetext,
-                "psgsize": str(max(12, len(str(psg)))),
-                "freetextsize": str(max(60, len(str(freetext)))),
-            },
-        )
-    else:  # no wallet data: should never happen as there should be default data in all cases
-        context = {
-            "year": year,
-            "prev": prev,
-            "next": next,
-            "prevy": prevy,
-            "nexty": nexty,
-            "files": files,
-            "dirs": dirs,
-            "waldata": waldata,
-            "svxfiles": svxfiles,
-            "checked": checked,
-            "create": create,
-            "people": "",
-            "peoplesize": 12,
-            "filesaved": filesaved,
-            "actual_saved": actual_saved,
-        }
-
-        return render(
-            request,
-            "walletform.html",
-            {
-                "form": form,
-                "wallet": wallet,
-                **context,
-                "date": "",
-                #'url':  "", 'urlsize': 12,
-                "survex": "",
-                "survexsize": 46,
-                "cave": cave,
-                "psg": psg,
-                "freetext": freetext,
-                "psgsize": 12,
-                "freetextsize": 20,
-            },
-        )
-
-
 @login_required_if_public
 def photoupload(request, folder=None):
     """Upload photo image files into /expofiles/photos/<year>/<photographer>/

core/views/wallets.py

@@ -0,0 +1,817 @@
+import datetime
+import json
+import os
+import re
+import socket
+import subprocess
+import urllib
+from pathlib import Path
+
+from django import forms
+from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
+from django.core.files.storage import FileSystemStorage
+from django.http import HttpResponseRedirect
+from django.shortcuts import render
+
+import settings
+from troggle.core.models.caves import Cave
+from troggle.core.models.logbooks import LogbookEntry  # , PersonLogEntry
+from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
+from troggle.core.models.troggle import DataIssue, Expedition
+from troggle.core.models.wallets import Wallet
+
+from troggle.core.views.caves import getCave
+from troggle.core.views.scans import caveifywallet, oldwallet
+from troggle.core.views.uploads import FilesForm
+
+from troggle.parsers.scans import contentsjson
+
+
+# from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt
+
+"""File upload 'views' 
+"""
+
+todo = """
+- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
+
+- Refactor walletedit() as it contains all the wallets 'complaints' code from the old script wallets.py
+
+- Need to validate uploaded file as being a valid image file, not a dubious script or hack
+
+- Write equivalent GPX upload form system, similar to walletedit() but in expofiles/gpslogs/
+  Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
+  
+- Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says tunnel is only mostly correct XML
+
+- Validate image files using a magic recogniser in walletedit() https://pypi.org/project/reportlab/ or 
+  https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file
+
+- Enable folder creation in dwguploads or as a separate form
+
+"""
+WALLET_BLANK_JSON = {
+    "cave": "",
+    "date": "",
+    "free text": "",
+    # "description url": "1623/NNN",
+    "description written": False,
+    "electronic survey": False,
+    "elev drawn": False,
+    "elev not required": False,
+    "name": "",
+    "people": ["Unknown"],
+    "plan drawn": False,
+    "plan not required": False,
+    "qms written": False,
+    "survex file": [],
+    "survex not required": False,
+    "website updated": False,
+}
+
+
+class WalletGotoForm(forms.Form):  # not a model-form, just a form-form
+    walletgoto = forms.CharField(strip=True, required=False)
+
+
+class WalletForm(forms.Form):  # not a model-form, just a form-form
+    descriptionw = forms.CharField(strip=True, required=False)
+    people = forms.CharField(strip=True, required=False)
+    survexnr = forms.CharField(strip=True, required=False)
+    qmsw = forms.CharField(strip=True, required=False)
+    date = forms.CharField(strip=True, required=True)  # the only required field
+    websiteupt = forms.CharField(strip=True, required=False)
+    elevnr = forms.CharField(strip=True, required=False)
+    cave = forms.CharField(strip=True, required=False)
+    psg = forms.CharField(strip=True, required=False)
+    freetext = forms.CharField(strip=True, required=False)
+    plannr = forms.CharField(strip=True, required=False)
+    electronic = forms.CharField(strip=True, required=False)
+    pland = forms.CharField(strip=True, required=False)
+    elevd = forms.CharField(strip=True, required=False)
+    # url = forms.CharField(strip=True, required=False)
+    survex = forms.CharField(strip=True, required=False)
+
+
+xlate = {
+    # "url": "description url",
+    "descriptionw": "description written",
+    "people": "people",
+    "date": "date",
+    "cave": "cave",
+    "plannr": "plan not required",
+    "survexnr": "survex not required",
+    "qmsw": "qms written",
+    "elevnr": "elev not required",
+    "websiteupt": "website updated",
+    "electronic": "electronic survey",
+    "pland": "plan drawn",
+    "elevd": "elev drawn",
+    "psg": "name",  # a name for this wallet
+    "freetext": "free text",
+    "survex": "survex file",
+}
+
+
+def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
+    """Taken from old script wallets.py and edited to make more comprehensible
+    Loads the survex files names and processes all complaints
+
+    """
+    # If skipping through the wallets on the upload form, the wallet may not yet exist
+    try:
+        w = Wallet.objects.get(walletname=wallet)
+    except ObjectDoesNotExist:
+        return None, None
+
+    # Date
+    if not waldata["date"]:
+        complaints.append(
+            "A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one."
+        )
+
+    # People
+    if (
+        not waldata["people"]
+        or waldata["people"] == ["NOBODY"]
+        or waldata["people"] == ["Unknown"]
+        or waldata["people"] == [""]
+    ):
+        complaints.append(
+            "Someody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data."
+        )
+
+    # survex, but get_ticks has already done much of this ??
+    survex_complaint = ""
+
+    if waldata["survex file"]:
+        if not type(waldata["survex file"]) == list:  # a string also is a sequence type, so do it this way
+            waldata["survex file"] = [waldata["survex file"]]
+        for sx in waldata["survex file"]:
+            # this logic appears in several places, inc get_ticks(). Refactor.
+            if sx != "":
+                if Path(sx).suffix.lower() != ".svx":
+                    sx = sx + ".svx"
+                svxfiles.append(sx)
+                if not (Path(settings.SURVEX_DATA) / sx).is_file():
+                    file_complaint = f"{wallet} Incorrect survex file name. File {sx} was not found in LOSER repo"
+                    complaints.append(file_complaint)
+                    message = f"! {file_complaint}"
+                    print(message)
+                    DataIssue.objects.update_or_create(
+                        parser="scans", message=message, url=wurl
+                    )  # set URL to this wallet folder
+                else:
+                    try:
+                        sxpath = str(Path(sx).with_suffix(""))
+                        SurvexFile.objects.get(path=sxpath)
+                    except MultipleObjectsReturned:
+                        # can happen if connecting a wallet to a survex file.. i think..
+                        QSsvxfiles = SurvexFile.objects.filter(path=sxpath)
+                        for s in QSsvxfiles:
+                            print(s.path, s.cave, s.survexdirectory)
+                        # QSsvxfiles[0]  # dont' know how this happened, fix later..
+                    except:
+                        file_complaint = (
+                            f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.."
+                        )
+                        complaints.append(file_complaint)
+                        message = f"! {file_complaint}"
+                        print(message)
+                        DataIssue.objects.update_or_create(
+                            parser="scans", message=message, url=wurl
+                        )  # set URL to this wallet folder
+
+    if waldata["survex not required"] and waldata["survex file"] != [""]:
+        survex_complaint = (
+            f'Survex is stated as not required and yet there is a survex file! ({waldata["survex file"]})'
+        )
+    if not waldata["survex not required"] and waldata["survex file"] == [""]:
+        survex_complaint = "A survex file is required, but has not been specified!"
+    if survex_complaint:
+        complaints.append(survex_complaint)
+
+    ticks = w.get_ticks()
+
+    # Notes required
+    if ticks["N"] != "green":
+        complaints.append(
+            "The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey."
+        )
+
+    # Plan drawing required
+    if ticks["P"] != "green":
+        complaints.append(
+            "The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found."
+        )
+
+    # Elev drawing required
+    if ticks["E"] != "green":
+        complaints.append(
+            "The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found."
+        )
+
+    # ETherion
+    if ticks["T"] != "green":
+        complaints.append(
+            "Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox."
+        )
+
+    # Description
+    if not waldata["description written"]:
+        complaints.append(
+            "The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done."
+        )
+    # QMs
+    if not waldata["qms written"] and w.year() and int(w.year()) >= 2015:
+        complaints.append(
+            "The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done."
+        )
+
+    # Website
+    if not waldata["website updated"]:
+        complaints.append(
+            "The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done."
+        )
+
+    # Find the cave, if it exists
+    if waldata["cave"]:
+        try:
+            caveid = waldata["cave"]
+            if type(caveid) is list:
+                for i in caveid:
+                    i = i.replace("/", "-")
+                    caveobject = getCave(i)  # only the last one gets recorded.. ouch.
+            else:
+                caveid = caveid
+                caveobject = getCave(caveid)
+            print(f'getCave for id "{waldata["cave"]}" {caveobject}')
+            # if not caveobject.url == waldata["description url"]:
+            # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.')
+        except Cave.MultipleObjectsReturned:
+            complaints.append(f'The cave ID \'{waldata["cave"]}\' is AMBIGUOUS. Please fix it.')
+            caveobject = None
+        except ObjectDoesNotExist:
+            complaints.append(f'The cave ID \'{waldata["cave"]}\' is not recognised. Please fix it.')
+            caveobject = None
+    else:
+        complaints.append(
+            'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"'
+        )
+        caveobject = None
+
+    return complaints, caveobject
+
+
+# @login_required_if_public
+def walletedit(request, path=None):
+    """Upload scanned image files into a wallet on /expofiles
+    Also display AND EDIT the contents.json data in the wallet.
+
+    This is the main wallet display and edit page.
+
+    The Wallet object and the contents.json file are created when the user
+    first uploads files.
+
+    This does NOT use a Django model linked to a Django form. Just a simple Django form.
+    You will find the Django documentation on forms very confusing,
+    as it covers many very differnet things we do not need. This is simpler.
+
+    This subsumes much of the code which was in the old wallets.py script and so this function is very long
+    indeed and needs refactoring.
+
+    REWRITE bits using the ticklist, dateify, caveify, populate etc utility functions in core.view.scans.py
+    """
+    git = settings.GIT
+    filesaved = False
+    actual_saved = []
+
+    def read_json():
+        """Read JSON from the wallet metadata file in the repo
+        or fills with blank data if that files can't be read
+
+        Should sanitise to ensure no spurious backslashes e.g. in windows style paths"""
+        waldata = {}
+        if contents_path.is_file():
+            with open(contents_path) as json_file:
+                try:
+                    waldata = json.load(json_file)
+                except:
+                    message = f"! {wallet} Failed to load {contents_path} JSON file"
+                    print(message)
+                    DataIssue.objects.create(parser="scans", message=message, url=wurl)  # set URL to this wallet folder
+                    raise
+        else:  # no JSON file exists
+            print("--- No JSON exists, so creating blank copy")
+            waldata = WALLET_BLANK_JSON.copy()
+        if not waldata["survex file"]:
+            try:
+                w = Wallet.objects.get(walletname=wallet)
+                b = SurvexBlock.objects.filter(scanswallet=w)
+                waldata["survex file"] = []
+                for bsf in b:
+                    waldata["survex file"].append(bsf.survexfile.path)
+            except:
+                print(f"--- No wallet {wallet} exists in database")
+        return waldata
+
+    def save_json(jsondict):
+        # print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}')
+        if not os.path.exists(contents_path.parent):
+            print("--- No wallet directory in :drawings: repo, so creating it")
+            os.makedirs(contents_path.parent)
+
+        with open(contents_path, "w") as jfile:
+            json.dump(jsondict, jfile, indent=1)
+        # print(f'--- FINISHED saving to JSON at {contents_path}')
+
+    def make_wallet(walletname):
+        """We need a wallet Object so that the django template stuff can find the files"""
+        try:
+            w, created = Wallet.objects.get_or_create(walletname=walletname)
+            print(f"--- Wallet string {walletname}, wallet object {w} created new?: {created}")
+            if created:
+                w.fpath = Path(settings.SCANS_ROOT, walletname[0:4], walletname)
+                w.save()
+        except:
+            print(f"!-- Wallet string {walletname}, FAIL TO GET or create WALLET OBJECT")
+            raise
+        return w
+
+    def commit_json(waldata):
+        destfolder = contents_path.parent
+        dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
+        if dr_add.returncode != 0:
+            msgdata = (
+                "Ask a nerd to fix this.\n--"
+                + dr_add.stderr
+                + "\n--"
+                + dr_add.stdout
+                + "\n--return code: "
+                + str(dr_add.returncode)
+            )
+            message = (
+                f"CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n" + msgdata
+            )
+            print(message)
+            return render(request, "errors/generic.html", {"message": message})
+        else:
+            if socket.gethostname() != "expo":
+                comment = f"on dev machine '{socket.gethostname()}' "
+            else:
+                comment = ""
+            if "cave" in waldata:
+                label = waldata["cave"]
+            else:
+                if "name" in waldata:
+                    label = waldata["name"]
+                else:
+                    label = ""
+
+            dr_commit = subprocess.run(
+                [git, "commit", "-m", f"JSON update wallet {wallet} {label} {comment}"],
+                cwd=destfolder,
+                capture_output=True,
+                text=True,
+            )
+            # This produces return code = 1 if it commits OK
+            if dr_commit.returncode != 0:
+                msgdata = (
+                    "Ask a nerd to fix this.\n\n"
+                    + dr_commit.stderr
+                    + "\n\n"
+                    + dr_commit.stdout
+                    + "\n\nreturn code: "
+                    + str(dr_commit.returncode)
+                )
+                message = (
+                    f"Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n"
+                    + msgdata
+                )
+                print(message)
+                return render(request, "errors/generic.html", {"message": message})
+
+    def get_logbook_trips():
+        return None
+
+    checkboxes = [
+        "description written",
+        "survex not required",
+        "qms written",
+        "website updated",
+        "plan not required",
+        "plan drawn",
+        "elev not required",
+        "elev drawn",
+        "electronic survey",
+    ]
+    if path:
+        wallet = urllib.parse.unquote(path)
+    else:
+        wallet = "2022#00"  # improve this later
+
+    year = wallet[:4]
+    try:
+        if wallet[4] != "#" and wallet[4] != ":":
+            # print(f'! - FORM walletedit - {wallet[4]} unurlencoded {unquote(wallet)[4]}')
+            #  print(f'! - FORM walletedit - start {wallet} REDIRECT TO OLDWALLET')
+            return oldwallet(request, path)
+    except:
+        # if nonumeric wallet name for example
+        return oldwallet(request, path)
+
+    if str(wallet).lower().endswith("indexpages"):
+        # print(f'! - FORM walletedit - start {wallet} REDIRECT TO OLDWALLET')
+        return walletindex(request, path)
+
+    if not re.match("(19|20)\d\d[:#]\d\d", wallet):
+        wallet = "2022:00"  # improve this later
+    # print(f'! - FORM walletedit - start {wallet}')
+
+    if path:
+        pass
+        # print(f'! - FORM walletedit - start wallet:{wallet}: path:{path}:')
+    if int(year) < 1977:
+        year = "1977"
+    if int(year) > 2050:
+        year = "2050"
+    nexty = f"{int(year)+1}"
+    prevy = f"{int(year)-1}"
+
+    wnumber = wallet[5:]
+    next = f"{int(wnumber)+1:02d}"
+    prev = f"{int(wnumber)-1:02d}"
+
+    if int(wnumber) == 0:
+        prev = f"{int(wnumber):02d}"
+
+    wurl = f"/walletedit/{wallet}".replace("#", ":")
+    wallet = wallet.replace(":", "#")
+    dirpath = Path(settings.SCANS_ROOT, year, wallet)
+    contents_path = Path(settings.DRAWINGS_DATA, "walletjson") / year / wallet / contentsjson
+
+    form = FilesForm()
+
+    if request.method == "POST":
+        if "psg" in request.POST:  # handle metadata form
+            formj = WalletForm(request.POST)
+            # Beware. All fields returned as strings. Must re-type them as lists etc. before using or re-saving
+            # Also lots of hassle with lists of strings interpreted as a single string
+            # Unset checkboxes do not return any value, checked ones return "True". So need initialising to False
+            if formj.is_valid():
+                posted = request.POST.copy()
+                posted.pop("csrfmiddlewaretoken")  # discard this
+                wd = WALLET_BLANK_JSON.copy()
+                for f in checkboxes:
+                    wd[f] = False
+                    # print(f'--- wd ${f}$  - {wd[f]}')
+                for f in posted:
+                    wd[xlate[f]] = posted[f].replace("'", '"')
+
+                    if posted[f] == "True":
+                        wd[xlate[f]] = True
+
+                wd["people"] = wd["people"][1:-1].replace('"', "").split(",")
+                for i, elem in enumerate(wd["people"]):
+                    wd["people"][i] = elem.strip()
+
+                # print(f'--- ${wd["survex file"]}$  - {type(wd["survex file"])}')
+                if wd["survex file"]:  # allow for no survex file at all
+                    if wd["survex file"][0] == "[":
+                        wd["survex file"] = wd["survex file"][1:-1]
+                    wd["survex file"] = wd["survex file"].replace('"', "").split(",")
+                    for i, elem in enumerate(wd["survex file"]):
+                        wd["survex file"][i] = elem.strip()
+                    # print(f'--- {wd["survex file"]} - {type(wd["survex file"])}')
+
+                save_json(wd)
+                walletobject = make_wallet(wallet)
+                commit_json(wd)
+
+            else:
+                print("--- INVALID JSON Update form submitted")
+                print(formj.errors)
+                return render(request, "errors/generic.html", {"message": formj.errors})
+
+        elif (
+            "walletgoto" in request.POST
+        ):  # not editing wallet data or uploading a file.. going direct to a named wallet
+            formg = WalletGotoForm(request.POST, request.FILES)
+            if formg.is_valid():
+                walletgoto = request.POST["walletgoto"]
+
+                return HttpResponseRedirect(f'/walletedit/{walletgoto.replace("#",":")}')
+
+        else:  # not editing wallet data, uploading a file. But should not overwrite metadata at all.
+            form = FilesForm(request.POST, request.FILES)
+
+            if form.is_valid():
+                # print(f'--- FORM walletedit multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}')
+                multiple = request.FILES.getlist("uploadfiles")
+                fs = FileSystemStorage(os.path.join(dirpath))  # creates wallet folder if necessary
+
+                waldata = read_json()
+                actual_saved = []
+                if multiple:
+                    for f in multiple:
+                        try:  # crashes in Django os.chmod call if on WSL, but does save file!
+                            saved_filename = fs.save(f.name, content=f)
+                        except:
+                            print(f"\n !! Permissions failure ?! on attempting to save scanfile {f.name}")
+                            if "saved_filename" in locals():
+                                if saved_filename.is_file():
+                                    actual_saved.append(saved_filename)
+                                    # print(f'! - FORM walletedit multiple {actual_saved}')
+                                    filesaved = True
+                    # print(f'--- FORM walletedit multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}')
+                    save_json(waldata)
+                    walletobject = make_wallet(wallet)
+                    commit_json(waldata)
+    #
+    # Not a POST, so a GET starts here. And also control gets here after a POST is processed.
+    #
+    files = []
+    dirs = []
+    # print(f'! - FORM walletedit - start {wallet} {dirpath}')
+    if dirpath.is_dir():
+        create = False  # wallet exists because folder exists, even if nothing in it
+        try:
+            for f in dirpath.iterdir():
+                if f.is_dir():
+                    for d in f.iterdir():
+                        dirs.append(f"{f.name}/{d.name}")
+                if f.is_file():
+                    files.append(f.name)
+        except FileNotFoundError:
+            files.append(
+                "(No wallet yet. It would be created if you upload a scan and then save the form with a date.)"
+            )
+    else:
+        create = True
+
+    if len(files) > 0:
+        files = sorted(files)
+
+    if dirs:
+        dirs = sorted(dirs)
+    try:
+        waldata = read_json()
+    except:
+        message = f"Nasty failure in parsing wallets metadata in {contents_path}. Probably backslash not forward slash in filename path"
+        return render(request, "errors/generic.html", {"message": message})
+
+    jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wallet[0:4] / wallet / "contents.json"
+    # print(f'! - FORM walletedit - jsonfile {jsonfile}')
+    if not Path(jsonfile).is_file():
+        metadataurl = ""
+    else:
+        metadataurl = Path("/dwgdataraw", "walletjson") / wallet[0:4] / wallet.replace("#", ":") / "contents.json"
+    psg = ""
+    freetext = ""
+    chkplannr = ""
+    chkpland = ""
+    svxfiles = []
+    trips = []
+    checked = {}
+    context = {}
+    if waldata:  # should always be true as populated by blank data if json file doesn't exist
+
+        # if not type(waldata["people"])==list:
+        # if waldata["people"][0] == '"':
+        # waldata["people"] = waldata["people"][1:-1]
+        # waldata["people"] = list(waldata["people"])
+
+        if (
+            not waldata["date"]
+            or not waldata["people"]
+            or waldata["people"] == ["Unknown"]
+            or waldata["people"] == [""]
+            or waldata["cave"] == ""
+        ):  # json file does not exist, blank data, or people not typed into JSON file
+            # refactor into separate functions for no date set or no people set or no cave set
+            # print(f'No date set')
+            print(f"\n - Incomplete, empty or default wallet data {wallet} {waldata=}")
+            refs = []
+            dates = []
+            team = []
+            caverefs = []
+            caves = []
+            names = []
+            svxf = ""
+            if waldata["survex file"]:
+                if not type(waldata["survex file"]) == list:  # a string also is a sequence type, so do it this way
+                    waldata["survex file"] = [waldata["survex file"]]
+            for svxf in waldata["survex file"]:
+                print(f" - {svxf=}")
+                if svxf:
+                    svx = Path(svxf)
+                    if svx.suffix.lower() != ".svx":
+                        svx = svx.with_suffix(".svx")
+                    f = Path(settings.SURVEX_DATA) / svx
+                    if f.is_file():
+                        path = svx.parent / svx.stem
+                        # print(f' - {path=}')
+                        try:
+                            svxfile = SurvexFile.objects.get(path=path)
+
+                            print(f" - {svxfile=}")
+                            if svxfile.cave:
+                                caves.append(svxfile.cave)
+                                caverefs.append(svxfile.cave.reference())
+                            blocks = SurvexBlock.objects.filter(survexfile=svxfile)
+                            for b in blocks:
+                                print(f" - - {b=} {b.scanswallet=} {b.date=}")
+                                if b.scanswallet:
+                                    refs.append(b.scanswallet)
+                                    if b.scanswallet.walletname == wallet:
+                                        if b.date:
+                                            dates.append(b.date)
+                                        if b.name != b.title:
+                                            names.append(str(b.name) + "|" + str(b.title))
+                                        else:
+                                            names.append(str(b.name))
+                                        # we can use the people, across all blocks that have this *ref
+                                        QSpeople = SurvexPersonRole.objects.filter(survexblock=b)
+                                        print(f" - - {QSpeople=}")
+                                        for p in QSpeople:
+                                            print(f" - - {p.personname} ")
+                                            team.append(p.personname)
+                                    # else:
+                                    # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}')
+                        except:
+                            message = "Specified survex file not found - database may be empty, or this survex file is not *included anywhere."
+                            # return render(request, 'errors/generic.html', {'message': message})
+                            pass
+
+                        if dates:
+                            waldata["date"] = min(dates).isoformat()
+                        print(f" - - {team=} ")
+                        team = list(set(team))
+                        waldata["people"] = team
+
+                        caverefs = list(set(caverefs))
+                        caves = list(set(caves))
+
+                        if len(caverefs) == 1:
+                            waldata["cave"] = caverefs[0]
+                            print(f" - Setting wallet cave to {caverefs[0]}")
+                            # waldata["description url"] = caves[0]
+                        elif len(caverefs) == 0:
+                            waldata["cave"] = ""
+                            # waldata["description url"] = ""
+                            print(f" - No caves in this wallet {wallet}. ")
+                        else:
+                            waldata["cave"] = "several caves"
+                            # waldata["description url"] = "several.."
+                            print(
+                                f" - More than one Cave {caves} in this wallet {wallet}. Not managed in this troggle release."
+                            )
+                        if len(names) == 1:
+                            if waldata["name"] == "":
+                                waldata["name"] = names[0]
+                            print(f" - Setting wallet name to {names[0]}")
+                        elif len(names) == 0:
+                            waldata["name"] = ""
+                            print(" - Setting wallet name blank")
+                        else:
+                            waldata["name"] = f"several, please edit: {names}"
+                            print(
+                                f" - More than one block name is relevant {names} in this wallet {wallet}. Not managed in this troggle release."
+                            )
+
+        if "cave" in waldata:
+            cave = waldata["cave"]  # text string
+        else:
+            cave = ""
+        if waldata["name"]:
+            psg = waldata["name"]
+        if "free text" in waldata:
+            freetext = waldata["free text"]
+
+        # find trips and survex files of the same date
+        if waldata["date"]:
+            datestr = waldata["date"].replace(".", "-")
+            try:
+                samedate = datetime.date.fromisoformat(datestr)
+            except ValueError:
+                # probably a single digit day number. HACKUS MAXIMUS.
+                # clearly we need to fix this when we first import date strings..
+                datestr = datestr[:-1] + "0" + datestr[-1]
+                print(f" - {datestr=} ")
+                try:
+                    samedate = datetime.date.fromisoformat(datestr)
+                except:
+                    try:
+                        samedate = datetime.date.fromisoformat(datestr[:10])
+                    except:
+                        samedate = None
+
+            thisexpo = Expedition.objects.get(year=int(year))
+            if samedate:
+                svxothers = SurvexBlock.objects.filter(date=samedate)
+                trips = LogbookEntry.objects.filter(date=samedate)
+            else:
+                svxothers = None
+                trips = None
+
+        else:
+            svxothers = None
+            trips = None
+
+        # Survex and survex complaints, comes from json file on disc, not as pre-populated as above
+        complaints, caveobject = get_complaints([], waldata, svxfiles, files, wallet, wurl)
+        # print(f' - {caveobject=}')
+
+        for f in checkboxes:
+            if waldata[f]:
+                checked[f] = "checked"
+
+        survexsize = str(min(len(str(waldata["survex file"])), 46))
+
+        try:
+            thiswallet = Wallet.objects.get(walletname=wallet)
+            caveifywallet(thiswallet)
+            thiswallet.ticks = thiswallet.get_ticks()  # the complaints in colour form
+            # fixsurvextick(thiswallet, thiswallet.ticks)
+            print(thiswallet)
+        except:
+            thiswallet = None
+        context = {
+            "year": year,
+            "prev": prev,
+            "next": next,
+            "prevy": prevy,
+            "nexty": nexty,
+            "files": files,
+            "dirs": dirs,
+            "waldata": waldata,
+            "svxfiles": svxfiles,
+            "checked": checked,
+            "trips": trips,
+            "manywallets": [thiswallet],
+            "svxothers": svxothers,
+            "create": create,
+            "metadataurl": metadataurl,
+            "complaints": complaints,
+            "caveobject": caveobject,
+            "people": waldata["people"],
+            "peoplesize": str(len(str(waldata["people"]))),
+            "filesaved": filesaved,
+            "actual_saved": actual_saved,
+        }
+
+        return render(
+            request,
+            "walletform.html",
+            {
+                "form": form,
+                "wallet": wallet,
+                **context,
+                "date": waldata["date"],
+                #'url':  waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))),
+                "survex": waldata["survex file"],
+                "survexsize": survexsize,
+                "cave": cave,
+                "psg": psg,
+                "freetext": freetext,
+                "psgsize": str(max(12, len(str(psg)))),
+                "freetextsize": str(max(60, len(str(freetext)))),
+            },
+        )
+    else:  # no wallet data: should never happen as there should be default data in all cases
+        context = {
+            "year": year,
+            "prev": prev,
+            "next": next,
+            "prevy": prevy,
+            "nexty": nexty,
+            "files": files,
+            "dirs": dirs,
+            "waldata": waldata,
+            "svxfiles": svxfiles,
+            "checked": checked,
+            "create": create,
+            "people": "",
+            "peoplesize": 12,
+            "filesaved": filesaved,
+            "actual_saved": actual_saved,
+        }
+
+        return render(
+            request,
+            "walletform.html",
+            {
+                "form": form,
+                "wallet": wallet,
+                **context,
+                "date": "",
+                #'url':  "", 'urlsize': 12,
+                "survex": "",
+                "survexsize": 46,
+                "cave": cave,
+                "psg": psg,
+                "freetext": freetext,
+                "psgsize": 12,
+                "freetextsize": 20,
+            },
+        )

templates/base.html

@@ -32,7 +32,7 @@
     <a href="/survexfile/">Survex</a> |
     <a href="{% url "survexcaveslist" %}">All Survex</a> |
     <a href="{% url "allscans" %}">Scans</a> |
-    <a href="{% url "scanupload"  '2022:01' %}">Upload Scans</a> |
+    <a href="{% url "walletedit"  '2022:01' %}">Upload Scans</a> |
     <a href="{% url "dwgallfiles" %}">Drawings</a> |
     <a href="{% url "dwgupload" %}">Upload Drawings</a> |
     <a href="{% url "photoupload" %}">Upload Photos</a> |

templates/tasks.html

@@ -66,7 +66,7 @@ ul {list-style: disc}
 <h3>Upload new data</h3>
 <ul>
 <li><a style="color:grey"href="/dwgupload/">Upload a Tunnel or Therion drawing</a></li>
-<li><a style="color:grey"href="/scanupload/2022:00">Upload a scan into a survey wallet</a></li>
+<li><a style="color:grey"href="/walletedit/2022:00">Upload a scan into a survey wallet</a></li>
 
 <li><a href="/survexfile/caves-1623/999/999.svx">Upload or type in a Survex file</a> - rewrite the area/number data in the browser title bar, 
 from 'caves-1623/999/999.svx' to the new number

templates/wallet_new.html

@@ -1,6 +1,6 @@
 
 <p>To create a wallet, if you have only a survex file and nothing to upload into the wallet,
-you currently need to got to <a href="/scanupload/">Upload Scans</a>, navigate to the new wallet number 
+you currently need to got to <a href="/walletedit/">Upload Scans</a>, navigate to the new wallet number 
 and then upload a dummy file, e.g. 'nothingyet.txt'. You can then edit the wallet details: the date,
 the people involved, and the url of the survexfile which you will already have uploaded using e.g.  
 <a href="/survexfile/caves-1623/290/mynewsurvex.svx">/survexfile/caves-1623/290/mynewsurvex.svx</a> 

templates/walletform.html

@@ -23,13 +23,13 @@
                 
     </form>
 <p style="font-family: monospace; font-weight: bold; font-size: 130%; text-align: center">
-<a style="font-weight: normal;" href="/scanupload/{{prevy}}:01">{{prevy}}</a>
+<a style="font-weight: normal;" href="/walletedit/{{prevy}}:01">{{prevy}}</a>
 &nbsp;...&nbsp;
-<a href="/scanupload/{{year}}:{{prev}}">{{year}}:{{prev}}</a> 
+<a href="/walletedit/{{year}}:{{prev}}">{{year}}:{{prev}}</a> 
 &larr; {{wallet}} &rarr; 
-<a href="/scanupload/{{year}}:{{next}}">{{year}}:{{next}}</a>
+<a href="/walletedit/{{year}}:{{next}}">{{year}}:{{next}}</a>
 &nbsp;...&nbsp;
-<a style="font-weight: normal;" href="/scanupload/{{nexty}}:01">{{nexty}}</a>
+<a style="font-weight: normal;" href="/walletedit/{{nexty}}:01">{{nexty}}</a>
 </p>  
 
  

urls.py

@@ -22,8 +22,8 @@ from troggle.core.views.other import (controlpanel, exportlogbook, frontpage,
 from troggle.core.views.prospect import prospecting
 from troggle.core.views.scans import (allscans, cavewallets, scansingle,
                                       walletslistperson, walletslistyear)
-from troggle.core.views.uploads import dwgupload, photoupload, scanupload
+from troggle.core.views.uploads import dwgupload, photoupload
-
+from troggle.core.views.wallets import walletedit
 """This sets the actualurlpatterns[] and urlpatterns[] lists which django uses 
 to resolve urls - in both directions as these are declarative. 
 
@@ -87,8 +87,8 @@ trogglepatterns = [
     re_path(r'^admin/',      admin.site.urls), # includes admin login & logout urls
     
 # Uploads - uploading a file
-    path('scanupload/',                scanupload,  name='scanupload'), # path=2020#01
+    path('walletedit/',                walletedit,  name='walletedit'), # path=2020#01
-    path('scanupload/<path:path>',     scanupload,  name='scanupload'), # path=2020#01
+    path('walletedit/<path:path>',     walletedit,  name='walletedit'), # path=2020#01
     path('photoupload/',                 photoupload, name='photoupload'), # restricted to current year
     path('photoupload/<path:folder>',    photoupload, name='photoupload'), # restricted to current year
     path('dwgupload/<path:folder>',      dwgupload,   name='dwgupload'), 
@@ -172,7 +172,7 @@ trogglepatterns = [
 
 # The survey scans in the wallets. This short-cuts SCANS_URL which is not used anymore and is defunct
     path('survey_scans/',                    allscans,    name="allscans"), # all the scans in all wallets
-    path('survey_scans/<path:path>/',        scanupload,  name="singlewallet"), # replaced singlewallet()
+    path('survey_scans/<path:path>/',        walletedit,  name="singlewallet"), # replaced singlewallet()
     path('survey_scans/<path:path>/<file>',  scansingle,  name="scansingle"), # works, but html href goes direct to /expofiles/ too
     path('cave/scans/<slug:caveid>',         cavewallets, name="cavewallets"), # like allscans, but for just one cave