troggle-unchained/core/views/wallets.py

import datetime
import json
import os
import re
import socket
import subprocess
import urllib
from pathlib import Path

from django import forms
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.core.files.storage import FileSystemStorage
from django.http import HttpResponseRedirect
from django.shortcuts import render

import settings
from troggle.core.models.caves import Cave
from troggle.core.models.logbooks import LogbookEntry  # , PersonLogEntry
from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.models.wallets import Wallet, YEAR_RANGE

from troggle.core.views.caves import getCave
from troggle.core.views.scans import caveifywallet, oldwallet
from troggle.core.views.uploads import FilesForm

from troggle.parsers.scans import contentsjson


"""Main wallet editing form, which includes scan file upload into the wallet 
"""

todo = """
- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import

- Refactor walletedit() as it contains all the wallets 'complaints' code from the old script wallets.py

- Need to validate uploaded file as being a valid image file, not a dubious script or hack
"""
WALLET_BLANK_JSON = {
    "cave": "",
    "date": "",
    "free text": "",
    # "description url": "1623/NNN",
    "description written": False,
    "electronic survey": False,
    "elev drawn": False,
    "elev not required": False,
    "name": "",
    "people": ["Unknown"],
    "plan drawn": False,
    "plan not required": False,
    "qms written": False,
    "survex file": [],
    "survex not required": False,
    "website updated": False,
}


class WalletGotoForm(forms.Form):  # not a model-form, just a form-form
    walletgoto = forms.CharField(strip=True, required=False)


class WalletForm(forms.Form):  # not a model-form, just a form-form
    descriptionw = forms.CharField(strip=True, required=False)
    people = forms.CharField(strip=True, required=False)
    survexnr = forms.CharField(strip=True, required=False)
    qmsw = forms.CharField(strip=True, required=False)
    date = forms.CharField(strip=True, required=True)  # the only required field
    websiteupt = forms.CharField(strip=True, required=False)
    elevnr = forms.CharField(strip=True, required=False)
    cave = forms.CharField(strip=True, required=False)
    psg = forms.CharField(strip=True, required=False)
    freetext = forms.CharField(strip=True, required=False)
    plannr = forms.CharField(strip=True, required=False)
    electronic = forms.CharField(strip=True, required=False)
    pland = forms.CharField(strip=True, required=False)
    elevd = forms.CharField(strip=True, required=False)
    # url = forms.CharField(strip=True, required=False)
    survex = forms.CharField(strip=True, required=False)


xlate = {
    # "url": "description url",
    "descriptionw": "description written",
    "people": "people",
    "date": "date",
    "cave": "cave",
    "plannr": "plan not required",
    "survexnr": "survex not required",
    "qmsw": "qms written",
    "elevnr": "elev not required",
    "websiteupt": "website updated",
    "electronic": "electronic survey",
    "pland": "plan drawn",
    "elevd": "elev drawn",
    "psg": "name",  # a name for this wallet
    "freetext": "free text",
    "survex": "survex file",
}


def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
    """Taken from old script wallets.py and edited to make more comprehensible
    Loads the survex files names and processes all complaints

    """
    # If skipping through the wallets on the upload form, the wallet may not yet exist
    try:
        w = Wallet.objects.get(walletname=wallet)
    except ObjectDoesNotExist:
        return None, None

    # Date
    if not waldata["date"]:
        complaints.append(
            "A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one."
        )

    # People
    if (
        not waldata["people"]
        or waldata["people"] == ["NOBODY"]
        or waldata["people"] == ["Unknown"]
        or waldata["people"] == [""]
    ):
        complaints.append(
            "Somebody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data."
        )

    # survex, but get_ticks has already done much of this ??
    survex_complaint = ""

    if waldata["survex file"]:
        if not type(waldata["survex file"]) == list:  # a string also is a sequence type, so do it this way
            waldata["survex file"] = [waldata["survex file"]]
        for sx in waldata["survex file"]:
            # this logic appears in several places, inc get_ticks(). Refactor.
            if sx != "":
                if Path(sx).suffix.lower() != ".svx":
                    sx = sx + ".svx"
                svxfiles.append(sx)
                if not (Path(settings.SURVEX_DATA) / sx).is_file():
                    file_complaint = f"{wallet} Incorrect survex file name. File {sx} was not found in LOSER repo"
                    complaints.append(file_complaint)
                    message = f"! {file_complaint}"
                    print(message)
                    DataIssue.objects.update_or_create(
                        parser="scans", message=message, url=wurl
                    )  # set URL to this wallet folder
                else:
                    try:
                        sxpath = str(Path(sx).with_suffix(""))
                        SurvexFile.objects.get(path=sxpath)
                    except MultipleObjectsReturned:
                        # can happen if connecting a wallet to a survex file.. i think..
                        QSsvxfiles = SurvexFile.objects.filter(path=sxpath)
                        for s in QSsvxfiles:
                            print(s.path, s.cave, s.survexdirectory)
                        # QSsvxfiles[0]  # dont' know how this happened, fix later..
                    except:
                        file_complaint = (
                            f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.."
                        )
                        complaints.append(file_complaint)
                        message = f"! {file_complaint}"
                        print(message)
                        DataIssue.objects.update_or_create(
                            parser="scans", message=message, url=wurl
                        )  # set URL to this wallet folder

    if waldata["survex not required"] and waldata["survex file"] != [""]:
        survex_complaint = (
            f'Survex is stated as not required and yet there is a survex file! ({waldata["survex file"]})'
        )
    if not waldata["survex not required"] and waldata["survex file"] == [""]:
        survex_complaint = "A survex file is required, but has not been specified!"
    if survex_complaint:
        complaints.append(survex_complaint)

    ticks = w.get_ticks()

    # Notes required
    if ticks["N"] != "green":
        complaints.append(
            "The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey."
        )

    # Plan drawing required
    if ticks["P"] != "green":
        complaints.append(
            "The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found."
        )

    # Elev drawing required
    if ticks["E"] != "green":
        complaints.append(
            "The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found."
        )

    # ETherion
    if ticks["T"] != "green":
        complaints.append(
            "Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox."
        )

    # Description
    if not waldata["description written"]:
        complaints.append(
            "The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done."
        )
    # QMs
    if not waldata["qms written"] and w.year() and int(w.year()) >= 2015:
        complaints.append(
            "The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done."
        )

    # Website
    if not waldata["website updated"]:
        complaints.append(
            "The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done."
        )

    # Find the cave, if it exists
    if waldata["cave"]:
        try:
            caveid = waldata["cave"]
            if type(caveid) is list:
                for i in caveid:
                    i = i.replace("/", "-")
                    caveobject = getCave(i)  # only the last one gets recorded.. ouch.
            else:
                caveid = caveid
                caveobject = getCave(caveid)
            print(f'getCave for id "{waldata["cave"]}" {caveobject}')
            # if not caveobject.url == waldata["description url"]:
            # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.')
        except Cave.MultipleObjectsReturned:
            complaints.append(f'The cave ID \'{waldata["cave"]}\' is AMBIGUOUS. Please fix it.')
            caveobject = None
        except ObjectDoesNotExist:
            complaints.append(f'The cave ID \'{waldata["cave"]}\' is not recognised. Please fix it.')
            caveobject = None
    else:
        complaints.append(
            'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"'
        )
        caveobject = None

    return complaints, caveobject


# @login_required_if_public
def walletedit(request, path=None):
    """Create a new wallet or upload scanned image files into a wallet on /expofiles
    Also display AND EDIT the contents.json data in the wallet.

    This is the main wallet display and edit page.

    The Wallet object and the contents.json file are created when the user
    creates the wallet AND THEN SAVES IT WITH A DATE.

    This does NOT use a Django model linked to a Django form. Just a simple Django form.
    You will find the Django documentation on forms very confusing,
    as it covers many very differnet things we do not need. This is simpler.

    This subsumes much of the code which was in the pre-2022 non-troggle wallets.py script 
    and so this function is very long indeed and needs refactoring.

    REWRITE bits using the ticklist, dateify, caveify, populate etc utility functions in core.view.scans.py
    """
    git = settings.GIT
    filesaved = False
    actual_saved = []
    
    def get_next_empty():
        """Gets the next most number for a new wallet just after the most recent one in the 
        db. But if it has no date set, then ignore it as it was only just created"""
        latest = Wallet.objects.filter(walletname__startswith="20",walletdate__isnull=False).latest('walletname')
        next = int(latest.walletname[5:]) + 1
        return f"{latest.walletname[:4]}:{next:02d}"
    
    def preprocess_path(path):
        if path:
            wpath = urllib.parse.unquote(path)
        else:
            return (None, get_next_empty() )

        try:
            year = wpath[:4] # if path too short, exception catches it
            sepr = wpath[4]
            y = int(year) # exception catches non-integer [:4]
            wnumber = int(wpath[5:]) # exception catches nonumeric wallet number
            if sepr != "#" and sepr != ":":
                return (oldwallet(request, path), None)
        except:
            # if nonumeric wpath name for example
            return (oldwallet(request, path), None)

        if not re.match("(19|20)\d\d[:#]\d\d\d?", wpath):
            return (None, get_next_empty() )
            
        ymin, ymax = YEAR_RANGE
        if int(year) < ymin:
            year = str(ymin+2)
        if int(year) > ymax:
            return (None, get_next_empty() )
            
        wallet = f"{year}:{wnumber:02d}"
        return (None, wallet)
        
    def create_nav_links(wallet):
        print(f"--- {wallet} now current")
        y = wallet[:4]
        n = wallet[5:]
        nexty = f"{int(y)+1}"
        prevy = f"{int(y)-1}"
        next = f"{int(n)+1:02d}"
        prev = f"{int(n)-1:02d}"

        if int(n) == 0:
            prev = f"{int(n):02d}"
            
        return next, nexty, prev, prevy, y

    def read_json():
        """Read JSON from the wallet metadata file in the repo
        or fills with blank data if that files can't be read

        Should sanitise to ensure no spurious backslashes e.g. in windows style paths"""
        waldata = {}
        if contents_path.is_file():
            with open(contents_path) as json_file:
                try:
                    waldata = json.load(json_file)
                except:
                    message = f"! {wallet} Failed to load {contents_path} JSON file"
                    print(message)
                    DataIssue.objects.create(parser="scans", message=message, url=wurl)  # set URL to this wallet folder
                    raise
        else:  # no JSON file exists
            print("--- No JSON exists, so creating blank copy")
            waldata = WALLET_BLANK_JSON.copy()
        if not waldata["survex file"]:
            try:
                w = Wallet.objects.get(walletname=wallet)
                b = SurvexBlock.objects.filter(scanswallet=w)
                waldata["survex file"] = []
                for bsf in b:
                    waldata["survex file"].append(bsf.survexfile.path)
            except:
                print(f"--- No wallet {wallet} exists in database")
        return waldata

    def save_json(jsondict):
        # print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}')
        if not os.path.exists(contents_path.parent):
            print("--- No wallet directory in :drawings: repo, so creating it")
            os.makedirs(contents_path.parent)

        with open(contents_path, "w") as jfile:
            json.dump(jsondict, jfile, indent=1)
        # print(f'--- FINISHED saving to JSON at {contents_path}')

    def make_wallet(walletname):
        """We need a wallet Object so that the django template stuff can find the files"""
        try:
            w, created = Wallet.objects.get_or_create(walletname=walletname)
            print(f"--- Wallet string {walletname}, wallet object {w} created new?: {created}")
            if created:
                w.fpath = Path(settings.SCANS_ROOT, walletname[0:4], walletname)
                w.save()
        except:
            print(f"!-- Wallet string {walletname}, FAIL TO GET or create WALLET OBJECT")
            raise
        return w

    def commit_json(waldata):
        destfolder = contents_path.parent
        dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
        if dr_add.returncode != 0:
            msgdata = (
                "Ask a nerd to fix this.\n--"
                + dr_add.stderr
                + "\n--"
                + dr_add.stdout
                + "\n--return code: "
                + str(dr_add.returncode)
            )
            message = (
                f"CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n" + msgdata
            )
            print(message)
            return render(request, "errors/generic.html", {"message": message})
        else:
            if socket.gethostname() != "expo":
                comment = f"on dev machine '{socket.gethostname()}' "
            else:
                comment = ""
            if "cave" in waldata:
                label = waldata["cave"]
            else:
                if "name" in waldata:
                    label = waldata["name"]
                else:
                    label = ""

            dr_commit = subprocess.run(
                [git, "commit", "-m", f"JSON update wallet {wallet} {label} {comment}"],
                cwd=destfolder,
                capture_output=True,
                text=True,
            )
            # This produces return code = 1 if it commits OK
            if dr_commit.returncode != 0:
                msgdata = (
                    "Ask a nerd to fix this.\n\n"
                    + dr_commit.stderr
                    + "\n\n"
                    + dr_commit.stdout
                    + "\n\nreturn code: "
                    + str(dr_commit.returncode)
                )
                message = (
                    f"Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n"
                    + msgdata
                )
                print(message)
                return render(request, "errors/generic.html", {"message": message})

    def get_logbook_trips():
        return None

    checkboxes = [
        "description written",
        "survex not required",
        "qms written",
        "website updated",
        "plan not required",
        "plan drawn",
        "elev not required",
        "elev drawn",
        "electronic survey",
    ]
    
    redirect, wallet = preprocess_path(path)
    if redirect:
        return redirect
    next, nexty, prev, prevy, year = create_nav_links(wallet)    

    wurl = f"/walletedit/{wallet}".replace("#", ":")
    wallet = wallet.replace(":", "#")
    dirpath = Path(settings.SCANS_ROOT, year, wallet)
    contents_path = Path(settings.DRAWINGS_DATA, "walletjson") / year / wallet / contentsjson
    
    fresh_wallet = False

    form = FilesForm()

    if request.method == "POST":
        # print(f'--- POST processing starts {wallet=} {path=}')
        if "psg" in request.POST:  # handle metadata form
            formj = WalletForm(request.POST)
            # Beware. All fields returned as strings. Must re-type them as lists etc. before using or re-saving
            # Also lots of hassle with lists of strings interpreted as a single string
            # Unset checkboxes do not return any value, checked ones return "True". So need initialising to False
            if formj.is_valid():
                posted = request.POST.copy()
                posted.pop("csrfmiddlewaretoken")  # discard this
                wd = WALLET_BLANK_JSON.copy()
                for f in checkboxes:
                    wd[f] = False
                    # print(f'--- wd ${f}$  - {wd[f]}')
                for f in posted:
                    wd[xlate[f]] = posted[f].replace("'", '"')

                    if posted[f] == "True":
                        wd[xlate[f]] = True

                wd["people"] = wd["people"][1:-1].replace('"', "").split(",")
                for i, elem in enumerate(wd["people"]):
                    wd["people"][i] = elem.strip()

                # print(f'--- ${wd["survex file"]}$  - {type(wd["survex file"])}')
                if wd["survex file"]:  # allow for no survex file at all
                    if wd["survex file"][0] == "[":
                        wd["survex file"] = wd["survex file"][1:-1]
                    wd["survex file"] = wd["survex file"].replace('"', "").split(",")
                    for i, elem in enumerate(wd["survex file"]):
                        wd["survex file"][i] = elem.strip()
                    # print(f'--- {wd["survex file"]} - {type(wd["survex file"])}')

                save_json(wd)
                walletobject = make_wallet(wallet) # will already exist
                commit_json(wd)

            else:
                print("--- INVALID JSON Update form submitted")
                print(formj.errors)
                return render(request, "errors/generic.html", {"message": formj.errors})

        elif (
            "walletgoto" in request.POST
        ):  # not editing wallet data or uploading a file.. going direct to a named wallet
            formg = WalletGotoForm(request.POST, request.FILES)
            if formg.is_valid():
                walletgoto = request.POST["walletgoto"]

                return HttpResponseRedirect(f'/walletedit/{walletgoto.replace("#",":")}')

        else:  # not editing wallet data, creating a wallet or uploading a file. Should not overwrite metadata at all.
            if "submitbutton" in request.POST:
                print(f"--- Submit button value {request.POST['submitbutton']}")
                if request.POST['submitbutton']=="Create":
                    w = WALLET_BLANK_JSON.copy()
                    save_json(w)
                    walletobject = make_wallet(wallet) # no date set yet
                    commit_json(w)
         
            form = FilesForm(request.POST, request.FILES)
            if form.is_valid():
                # print(f'--- FORM walletedit multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}')
                multiple = request.FILES.getlist("uploadfiles")
                fs = FileSystemStorage(os.path.join(dirpath))  # creates wallet folder if necessary

                waldata = read_json()
                actual_saved = []
                if multiple:
                    for f in multiple:
                        try:  # crashes in Django os.chmod call if on WSL, but does save file!
                            saved_filename = fs.save(f.name, content=f)
                        except:
                            print(f"\n !! Permissions failure ?! on attempting to save scanfile {f.name}")
                            if "saved_filename" in locals():
                                if saved_filename.is_file():
                                    actual_saved.append(saved_filename)
                                    # print(f'! - FORM walletedit multiple {actual_saved}')
                                    filesaved = True
                    # print(f'--- FORM walletedit multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}')
                    save_json(waldata)
                    walletobject = make_wallet(wallet)
                    commit_json(waldata)
            else:
                print("--- Upload files form invalid, which is correct if just created.")
    #
    # Not a POST, so a GET starts here. And also control gets here after a POST is processed.
    #
    files = []
    dirs = []
    # print(f'! - FORM walletedit - start {wallet} {dirpath}')
    if dirpath.is_dir():
        create = False  # wallet exists because folder exists, even if nothing in it
        try:
            for f in dirpath.iterdir():
                if f.is_dir():
                    for d in f.iterdir():
                        dirs.append(f"{f.name}/{d.name}")
                if f.is_file():
                    files.append(f.name)
        except FileNotFoundError:
            files.append(
                "(No wallet yet. It would be created if you upload a scan and then save the form with a date.)"
            )
    else:
        # either on GET or on dropping-through after the POST creating a new wallet object:
        if Wallet.objects.filter(walletname=wallet).exists():
            create = False
        else:
            create = True

    if len(files) > 0:
        files = sorted(files)

    if dirs:
        dirs = sorted(dirs)
    try:
        waldata = read_json()
    except:
        message = f"Nasty failure in parsing wallets metadata in {contents_path}. Probably backslash not forward slash in filename path"
        return render(request, "errors/generic.html", {"message": message})

    jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wallet[0:4] / wallet / "contents.json"
    # print(f'! - FORM walletedit - jsonfile {jsonfile}')
    if not Path(jsonfile).is_file():
        metadataurl = ""
    else:
        metadataurl = Path("/dwgdataraw", "walletjson") / wallet[0:4] / wallet.replace("#", ":") / "contents.json"
    psg = ""
    freetext = ""
    chkplannr = ""
    chkpland = ""
    svxfiles = []
    trips = []
    checked = {}
    context = {}
    if waldata:  # should always be true as populated by blank data if json file doesn't exist

        if (
            not waldata["date"]
            or not waldata["people"]
            or waldata["people"] == ["Unknown"]
            or waldata["people"] == [""]
            or waldata["cave"] == ""
        ):  # json file does not exist, blank data, or people not typed into JSON file
            # refactor into separate functions for no date set or no people set or no cave set
            # print(f'No date set')
            print(f"\n - Incomplete, empty or default wallet data {wallet} {waldata=}")
            refs = []
            dates = []
            team = []
            caverefs = []
            caves = []
            names = []
            svxf = ""
            if waldata["survex file"]:
                if not type(waldata["survex file"]) == list:  # a string also is a sequence type, so do it this way
                    waldata["survex file"] = [waldata["survex file"]]
            for svxf in waldata["survex file"]:
                if svxf:
                    svx = Path(svxf)
                    if svx.suffix.lower() != ".svx":
                        svx = svx.with_suffix(".svx")
                    f = Path(settings.SURVEX_DATA) / svx
                    if f.is_file():
                        fpath = svx.parent / svx.stem
                        # print(f' - {fpath=}')
                        try:
                            svxfile = SurvexFile.objects.get(path=fpath)

                            print(f" - {svxfile=}")
                            if svxfile.cave:
                                caves.append(svxfile.cave)
                                caverefs.append(svxfile.cave.reference())
                            blocks = SurvexBlock.objects.filter(survexfile=svxfile)
                            for b in blocks:
                                print(f" - - {b=} {b.scanswallet=} {b.date=}")
                                if b.scanswallet:
                                    refs.append(b.scanswallet)
                                    if b.scanswallet.walletname == wallet:
                                        if b.date:
                                            dates.append(b.date)
                                        if b.name != b.title:
                                            names.append(str(b.name) + "|" + str(b.title))
                                        else:
                                            names.append(str(b.name))
                                        # we can use the people, across all blocks that have this *ref
                                        QSpeople = SurvexPersonRole.objects.filter(survexblock=b)
                                        print(f" - - {QSpeople=}")
                                        for p in QSpeople:
                                            print(f" - - {p.personname} ")
                                            team.append(p.personname)
                                    # else:
                                    # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}')
                        except:
                            message = "Specified survex file not found - database may be empty, or this survex file is not *included anywhere."
                            # return render(request, 'errors/generic.html', {'message': message})
                            pass

                        if dates:
                            waldata["date"] = min(dates).isoformat()
                        print(f" - - {team=} ")
                        team = list(set(team))
                        waldata["people"] = team

                        caverefs = list(set(caverefs))
                        caves = list(set(caves))

                        if len(caverefs) == 1:
                            waldata["cave"] = caverefs[0]
                            print(f" - Setting wallet cave to {caverefs[0]}")
                            # waldata["description url"] = caves[0]
                        elif len(caverefs) == 0:
                            waldata["cave"] = ""
                            # waldata["description url"] = ""
                            print(f" - No caves in this wallet {wallet}. ")
                        else:
                            waldata["cave"] = "several caves"
                            # waldata["description url"] = "several.."
                            print(
                                f" - More than one Cave {caves} in this wallet {wallet}. Not managed in this troggle release."
                            )
                        if len(names) == 1:
                            if waldata["name"] == "":
                                waldata["name"] = names[0]
                            print(f" - Setting wallet name to {names[0]}")
                        elif len(names) == 0:
                            waldata["name"] = ""
                            print(" - Setting wallet name blank")
                        else:
                            waldata["name"] = f"several, please edit: {names}"
                            print(
                                f" - More than one block name is relevant {names} in this wallet {wallet}. Not managed in this troggle release."
                            )

        if "cave" in waldata:
            cave = waldata["cave"]  # text string
        else:
            cave = ""
        if waldata["name"]:
            psg = waldata["name"]
        if "free text" in waldata:
            freetext = waldata["free text"]

        # find trips and survex files of the same date
        if waldata["date"]:
            datestr = waldata["date"].replace(".", "-")
            try:
                samedate = datetime.date.fromisoformat(datestr)
            except ValueError:
                # probably a single digit day number. HACKUS MAXIMUS.
                # clearly we need to fix this when we first import date strings..
                datestr = datestr[:-1] + "0" + datestr[-1]
                print(f" - {datestr=} ")
                try:
                    samedate = datetime.date.fromisoformat(datestr)
                except:
                    try:
                        samedate = datetime.date.fromisoformat(datestr[:10])
                    except:
                        samedate = None
            walletobject = make_wallet(wallet)
            walletobject.walletdate = samedate
            walletobject.save()

            try:
                thisexpo = Expedition.objects.get(year=int(year))
            except: # creating a wallet for an expo that does not exist perhaps
                message = f"Trying to access an Expo for '{year}' which does not exist (yet)." 
                message += " See /handbook/computing/newyear.html"
                print(message)
                return render(request, "errors/generic.html", {"message": message})
            if samedate:
                svxothers = SurvexBlock.objects.filter(date=samedate)
                trips = LogbookEntry.objects.filter(date=samedate)
            else:
                svxothers = None
                trips = None

        else:
            svxothers = None
            trips = None

        # Survex and survex complaints, comes from json file on disc, not as pre-populated as above
        complaints, caveobject = get_complaints([], waldata, svxfiles, files, wallet, wurl)
        # print(f' - {caveobject=}')

        for f in checkboxes:
            if waldata[f]:
                checked[f] = "checked"

        survexsize = str(min(len(str(waldata["survex file"])), 46))

        try:
            thiswallet = Wallet.objects.get(walletname=wallet)
            caveifywallet(thiswallet)
            thiswallet.ticks = thiswallet.get_ticks()  # the complaints in colour form
            # fixsurvextick(thiswallet, thiswallet.ticks)
            print(f"--- {wallet}  {thiswallet} walletdate={thiswallet.walletdate} immediately before form render")
        except:
            thiswallet = None
        context = {
            "year": year,
            "prev": prev,
            "next": next,
            "prevy": prevy,
            "nexty": nexty,
            "files": files,
            "dirs": dirs,
            "waldata": waldata,
            "svxfiles": svxfiles,
            "checked": checked,
            "trips": trips,
            "manywallets": [thiswallet],
            "svxothers": svxothers,
            "create": create,
            "metadataurl": metadataurl,
            "complaints": complaints,
            "caveobject": caveobject,
            "people": waldata["people"],
            "peoplesize": str(len(str(waldata["people"]))),
            "filesaved": filesaved,
            "actual_saved": actual_saved,
        }

        return render(
            request,
            "walletform.html",
            {
                "form": form,
                "wallet": wallet,
                **context,
                "date": waldata["date"],
                #'url':  waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))),
                "survex": waldata["survex file"],
                "survexsize": survexsize,
                "cave": cave,
                "psg": psg,
                "freetext": freetext,
                "psgsize": str(max(12, len(str(psg)))),
                "freetextsize": str(max(60, len(str(freetext)))),
            },
        )
    else:  # no wallet data: should never happen as there should be default data in all cases
        context = {
            "year": year,
            "prev": prev,
            "next": next,
            "prevy": prevy,
            "nexty": nexty,
            "files": files,
            "dirs": dirs,
            "waldata": waldata,
            "svxfiles": svxfiles,
            "checked": checked,
            "create": create,
            "people": "",
            "peoplesize": 12,
            "filesaved": filesaved,
            "actual_saved": actual_saved,
        }

        return render(
            request,
            "walletform.html",
            {
                "form": form,
                "wallet": wallet,
                **context,
                "date": "",
                #'url':  "", 'urlsize': 12,
                "survex": "",
                "survexsize": 46,
                "cave": cave,
                "psg": psg,
                "freetext": freetext,
                "psgsize": 12,
                "freetextsize": 20,
            },
        )