From 1f5b56a5931666b1a6627a2ae83d3f846badbaf9 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Tue, 31 Jan 2023 17:13:41 +0000 Subject: [PATCH] Wallet editor into separate file --- core/TESTS/test_imports.py | 5 +- core/TESTS/tests_logins.py | 4 +- core/models/wallets.py | 2 +- core/views/scans.py | 2 +- core/views/uploads.py | 792 +---------------------------------- core/views/wallets.py | 817 +++++++++++++++++++++++++++++++++++++ templates/base.html | 2 +- templates/tasks.html | 2 +- templates/wallet_new.html | 2 +- templates/walletform.html | 8 +- urls.py | 10 +- 11 files changed, 840 insertions(+), 806 deletions(-) create mode 100644 core/views/wallets.py diff --git a/core/TESTS/test_imports.py b/core/TESTS/test_imports.py index 99d9a30..636f0b8 100644 --- a/core/TESTS/test_imports.py +++ b/core/TESTS/test_imports.py @@ -55,7 +55,10 @@ class SimpleTest(SimpleTestCase): from troggle.parsers.people import GetPersonExpeditionNameLookup def test_import_views_uploads(self): - from troggle.core.views.uploads import dwgupload, scanupload + from troggle.core.views.uploads import dwgupload + + def test_import_views_walletedit(self): + from troggle.core.views.wallets import walletedit def test_import_parsers_QMs(self): from troggle.core.models.logbooks import QM diff --git a/core/TESTS/tests_logins.py b/core/TESTS/tests_logins.py index 0dae63e..0e6da96 100644 --- a/core/TESTS/tests_logins.py +++ b/core/TESTS/tests_logins.py @@ -73,7 +73,7 @@ class FixturePageTests(TestCase): class PostTests(TestCase): - """Tests scanupload form""" + """Tests walletedit form""" fixtures = ["auth_users"] @@ -120,7 +120,7 @@ class PostTests(TestCase): with open("core/fixtures/test_upload_file.txt", "r") as testf: response = self.client.post( - f"/scanupload/{testyear}:00", data={"name": "test_upload_file.txt", "uploadfiles": testf} + f"/walletedit/{testyear}:00", data={"name": "test_upload_file.txt", "uploadfiles": testf} ) content = response.content.decode() self.assertEqual(response.status_code, 200) diff --git a/core/models/wallets.py b/core/models/wallets.py index eb93d10..19eaec5 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -40,7 +40,7 @@ class Wallet(models.Model): fp = Path(self.fpath) wname = fp.name wyear = fp.parent.name - wurl = f"/scanupload/{self.walletname}" # .replace('#', ':') + wurl = f"/walletedit/{self.walletname}" # .replace('#', ':') jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json" if not Path(jsonfile).is_file(): diff --git a/core/views/scans.py b/core/views/scans.py index 39aebbc..63b477d 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -253,7 +253,7 @@ def cavewallets(request, caveid): # print(f' - Found one ! {z.walletname=} {zcaveid=}') wallets.add(z) else: - wurl = f"/scanupload/{z.walletname.replace('#',':')}" + wurl = f"/walletedit/{z.walletname.replace('#',':')}" message = f" ! In {z.walletname} there is an unrecognised cave name '{zcaveid}' (out of {len(Gcavelookup):,} cave names and aliases)" print(message) DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl) diff --git a/core/views/uploads.py b/core/views/uploads.py index 6763d8c..3dc14d6 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -1,31 +1,16 @@ -import datetime -import json -import os -import re -import socket import subprocess -import urllib from pathlib import Path from django import forms -from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.core.files.storage import FileSystemStorage -from django.http import HttpResponseRedirect from django.shortcuts import render import settings -from troggle.core.models.caves import Cave -from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry -from troggle.core.models.survex import DrawingFile, SurvexBlock, SurvexFile, SurvexPersonRole -from troggle.core.models.wallets import Wallet +from troggle.core.models.survex import DrawingFile # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* -from troggle.core.models.troggle import DataIssue, Expedition -from troggle.core.views.caves import getCave -from troggle.core.views.scans import caveifywallet, oldwallet # from troggle import settings -from troggle.parsers.scans import contentsjson from .auth import login_required_if_public @@ -37,801 +22,30 @@ from .auth import login_required_if_public todo = """ - Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import -- Refactor scanupload() as it contains all the wallets 'complaints' code from the old script wallets.py - - Need to validate uploaded file as being a valid image file, not a dubious script or hack -- Write equivalent GPX upload form system, similar to scanupload() but in expofiles/gpslogs/ +- Write equivalent GPX upload form system, similar to walletedit() but in expofiles/gpslogs/ Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack - Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says tunnel is only mostly correct XML -- Validate image files using a magic recogniser in scanupload() https://pypi.org/project/reportlab/ or +- Validate image files using a magic recogniser in walletedit() https://pypi.org/project/reportlab/ or https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file - Enable folder creation in dwguploads or as a separate form """ -WALLET_BLANK_JSON = { - "cave": "", - "date": "", - "free text": "", - # "description url": "1623/NNN", - "description written": False, - "electronic survey": False, - "elev drawn": False, - "elev not required": False, - "name": "", - "people": ["Unknown"], - "plan drawn": False, - "plan not required": False, - "qms written": False, - "survex file": [], - "survex not required": False, - "website updated": False, -} - class FilesForm(forms.Form): # not a model-form, just a form-form uploadfiles = forms.FileField() - class FilesRenameForm(forms.Form): # not a model-form, just a form-form uploadfiles = forms.FileField() renameto = forms.CharField(strip=True, required=False) - -class WalletGotoForm(forms.Form): # not a model-form, just a form-form - walletgoto = forms.CharField(strip=True, required=False) - - class TextForm(forms.Form): # not a model-form, just a form-form photographer = forms.CharField(strip=True) - -class WalletForm(forms.Form): # not a model-form, just a form-form - descriptionw = forms.CharField(strip=True, required=False) - people = forms.CharField(strip=True, required=False) - survexnr = forms.CharField(strip=True, required=False) - qmsw = forms.CharField(strip=True, required=False) - date = forms.CharField(strip=True, required=True) # the only required field - websiteupt = forms.CharField(strip=True, required=False) - elevnr = forms.CharField(strip=True, required=False) - cave = forms.CharField(strip=True, required=False) - psg = forms.CharField(strip=True, required=False) - freetext = forms.CharField(strip=True, required=False) - plannr = forms.CharField(strip=True, required=False) - electronic = forms.CharField(strip=True, required=False) - pland = forms.CharField(strip=True, required=False) - elevd = forms.CharField(strip=True, required=False) - # url = forms.CharField(strip=True, required=False) - survex = forms.CharField(strip=True, required=False) - - -xlate = { - # "url": "description url", - "descriptionw": "description written", - "people": "people", - "date": "date", - "cave": "cave", - "plannr": "plan not required", - "survexnr": "survex not required", - "qmsw": "qms written", - "elevnr": "elev not required", - "websiteupt": "website updated", - "electronic": "electronic survey", - "pland": "plan drawn", - "elevd": "elev drawn", - "psg": "name", # a name for this wallet - "freetext": "free text", - "survex": "survex file", -} - - -def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): - """Taken from old script wallets.py and edited to make more comprehensible - Loads the survex files names and processes all complaints - - """ - # If skipping through the wallets on the upload form, the wallet may not yet exist - try: - w = Wallet.objects.get(walletname=wallet) - except ObjectDoesNotExist: - return None, None - - # Date - if not waldata["date"]: - complaints.append( - "A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one." - ) - - # People - if ( - not waldata["people"] - or waldata["people"] == ["NOBODY"] - or waldata["people"] == ["Unknown"] - or waldata["people"] == [""] - ): - complaints.append( - "Someody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data." - ) - - # survex, but get_ticks has already done much of this ?? - survex_complaint = "" - - if waldata["survex file"]: - if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way - waldata["survex file"] = [waldata["survex file"]] - for sx in waldata["survex file"]: - # this logic appears in several places, inc get_ticks(). Refactor. - if sx != "": - if Path(sx).suffix.lower() != ".svx": - sx = sx + ".svx" - svxfiles.append(sx) - if not (Path(settings.SURVEX_DATA) / sx).is_file(): - file_complaint = f"{wallet} Incorrect survex file name. File {sx} was not found in LOSER repo" - complaints.append(file_complaint) - message = f"! {file_complaint}" - print(message) - DataIssue.objects.update_or_create( - parser="scans", message=message, url=wurl - ) # set URL to this wallet folder - else: - try: - sxpath = str(Path(sx).with_suffix("")) - SurvexFile.objects.get(path=sxpath) - except MultipleObjectsReturned: - # can happen if connecting a wallet to a survex file.. i think.. - QSsvxfiles = SurvexFile.objects.filter(path=sxpath) - for s in QSsvxfiles: - print(s.path, s.cave, s.survexdirectory) - # QSsvxfiles[0] # dont' know how this happened, fix later.. - except: - file_complaint = ( - f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.." - ) - complaints.append(file_complaint) - message = f"! {file_complaint}" - print(message) - DataIssue.objects.update_or_create( - parser="scans", message=message, url=wurl - ) # set URL to this wallet folder - - if waldata["survex not required"] and waldata["survex file"] != [""]: - survex_complaint = ( - f'Survex is stated as not required and yet there is a survex file! ({waldata["survex file"]})' - ) - if not waldata["survex not required"] and waldata["survex file"] == [""]: - survex_complaint = "A survex file is required, but has not been specified!" - if survex_complaint: - complaints.append(survex_complaint) - - ticks = w.get_ticks() - - # Notes required - if ticks["N"] != "green": - complaints.append( - "The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey." - ) - - # Plan drawing required - if ticks["P"] != "green": - complaints.append( - "The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found." - ) - - # Elev drawing required - if ticks["E"] != "green": - complaints.append( - "The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found." - ) - - # ETherion - if ticks["T"] != "green": - complaints.append( - "Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox." - ) - - # Description - if not waldata["description written"]: - complaints.append( - "The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done." - ) - # QMs - if not waldata["qms written"] and w.year() and int(w.year()) >= 2015: - complaints.append( - "The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done." - ) - - # Website - if not waldata["website updated"]: - complaints.append( - "The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done." - ) - - # Find the cave, if it exists - if waldata["cave"]: - try: - caveid = waldata["cave"] - if type(caveid) is list: - for i in caveid: - i = i.replace("/", "-") - caveobject = getCave(i) # only the last one gets recorded.. ouch. - else: - caveid = caveid - caveobject = getCave(caveid) - print(f'getCave for id "{waldata["cave"]}" {caveobject}') - # if not caveobject.url == waldata["description url"]: - # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.') - except Cave.MultipleObjectsReturned: - complaints.append(f'The cave ID \'{waldata["cave"]}\' is AMBIGUOUS. Please fix it.') - caveobject = None - except ObjectDoesNotExist: - complaints.append(f'The cave ID \'{waldata["cave"]}\' is not recognised. Please fix it.') - caveobject = None - else: - complaints.append( - 'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"' - ) - caveobject = None - - return complaints, caveobject - - -# @login_required_if_public -def scanupload(request, path=None): - """Upload scanned image files into a wallet on /expofiles - Also display AND EDIT the contents.json data in the wallet. - - This is the main wallet display and edit page. - - The Wallet object and the contents.json file are created when the user - first uploads files. - - This does NOT use a Django model linked to a Django form. Just a simple Django form. - You will find the Django documentation on forms very confusing, - as it covers many very differnet things we do not need. This is simpler. - - This subsumes much of the code which was in the old wallets.py script and so this function is very long - indeed and needs refactoring. - - REWRITE bits using the ticklist, dateify, caveify, populate etc utility functions in core.view.scans.py - """ - git = settings.GIT - filesaved = False - actual_saved = [] - - def read_json(): - """Read JSON from the wallet metadata file in the repo - or fills with blank data if that files can't be read - - Should sanitise to ensure no spurious backslashes e.g. in windows style paths""" - waldata = {} - if contents_path.is_file(): - with open(contents_path) as json_file: - try: - waldata = json.load(json_file) - except: - message = f"! {wallet} Failed to load {contents_path} JSON file" - print(message) - DataIssue.objects.create(parser="scans", message=message, url=wurl) # set URL to this wallet folder - raise - else: # no JSON file exists - print("--- No JSON exists, so creating blank copy") - waldata = WALLET_BLANK_JSON.copy() - if not waldata["survex file"]: - try: - w = Wallet.objects.get(walletname=wallet) - b = SurvexBlock.objects.filter(scanswallet=w) - waldata["survex file"] = [] - for bsf in b: - waldata["survex file"].append(bsf.survexfile.path) - except: - print(f"--- No wallet {wallet} exists in database") - return waldata - - def save_json(jsondict): - # print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}') - if not os.path.exists(contents_path.parent): - print("--- No wallet directory in :drawings: repo, so creating it") - os.makedirs(contents_path.parent) - - with open(contents_path, "w") as jfile: - json.dump(jsondict, jfile, indent=1) - # print(f'--- FINISHED saving to JSON at {contents_path}') - - def make_wallet(walletname): - """We need a wallet Object so that the django template stuff can find the files""" - try: - w, created = Wallet.objects.get_or_create(walletname=walletname) - print(f"--- Wallet string {walletname}, wallet object {w} created new?: {created}") - if created: - w.fpath = Path(settings.SCANS_ROOT, walletname[0:4], walletname) - w.save() - except: - print(f"!-- Wallet string {walletname}, FAIL TO GET or create WALLET OBJECT") - raise - return w - - def commit_json(waldata): - destfolder = contents_path.parent - dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True) - if dr_add.returncode != 0: - msgdata = ( - "Ask a nerd to fix this.\n--" - + dr_add.stderr - + "\n--" - + dr_add.stdout - + "\n--return code: " - + str(dr_add.returncode) - ) - message = ( - f"CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n" + msgdata - ) - print(message) - return render(request, "errors/generic.html", {"message": message}) - else: - if socket.gethostname() != "expo": - comment = f"on dev machine '{socket.gethostname()}' " - else: - comment = "" - if "cave" in waldata: - label = waldata["cave"] - else: - if "name" in waldata: - label = waldata["name"] - else: - label = "" - - dr_commit = subprocess.run( - [git, "commit", "-m", f"JSON update wallet {wallet} {label} {comment}"], - cwd=destfolder, - capture_output=True, - text=True, - ) - # This produces return code = 1 if it commits OK - if dr_commit.returncode != 0: - msgdata = ( - "Ask a nerd to fix this.\n\n" - + dr_commit.stderr - + "\n\n" - + dr_commit.stdout - + "\n\nreturn code: " - + str(dr_commit.returncode) - ) - message = ( - f"Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n" - + msgdata - ) - print(message) - return render(request, "errors/generic.html", {"message": message}) - - def get_logbook_trips(): - return None - - checkboxes = [ - "description written", - "survex not required", - "qms written", - "website updated", - "plan not required", - "plan drawn", - "elev not required", - "elev drawn", - "electronic survey", - ] - if path: - wallet = urllib.parse.unquote(path) - else: - wallet = "2022#00" # improve this later - - year = wallet[:4] - try: - if wallet[4] != "#" and wallet[4] != ":": - # print(f'! - FORM scanupload - {wallet[4]} unurlencoded {unquote(wallet)[4]}') - # print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET') - return oldwallet(request, path) - except: - # if nonumeric wallet name for example - return oldwallet(request, path) - - if str(wallet).lower().endswith("indexpages"): - # print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET') - return walletindex(request, path) - - if not re.match("(19|20)\d\d[:#]\d\d", wallet): - wallet = "2022:00" # improve this later - # print(f'! - FORM scanupload - start {wallet}') - - if path: - pass - # print(f'! - FORM scanupload - start wallet:{wallet}: path:{path}:') - if int(year) < 1977: - year = "1977" - if int(year) > 2050: - year = "2050" - nexty = f"{int(year)+1}" - prevy = f"{int(year)-1}" - - wnumber = wallet[5:] - next = f"{int(wnumber)+1:02d}" - prev = f"{int(wnumber)-1:02d}" - - if int(wnumber) == 0: - prev = f"{int(wnumber):02d}" - - wurl = f"/scanupload/{wallet}".replace("#", ":") - wallet = wallet.replace(":", "#") - dirpath = Path(settings.SCANS_ROOT, year, wallet) - contents_path = Path(settings.DRAWINGS_DATA, "walletjson") / year / wallet / contentsjson - - form = FilesForm() - - if request.method == "POST": - if "psg" in request.POST: # handle metadata form - formj = WalletForm(request.POST) - # Beware. All fields returned as strings. Must re-type them as lists etc. before using or re-saving - # Also lots of hassle with lists of strings interpreted as a single string - # Unset checkboxes do not return any value, checked ones return "True". So need initialising to False - if formj.is_valid(): - posted = request.POST.copy() - posted.pop("csrfmiddlewaretoken") # discard this - wd = WALLET_BLANK_JSON.copy() - for f in checkboxes: - wd[f] = False - # print(f'--- wd ${f}$ - {wd[f]}') - for f in posted: - wd[xlate[f]] = posted[f].replace("'", '"') - - if posted[f] == "True": - wd[xlate[f]] = True - - wd["people"] = wd["people"][1:-1].replace('"', "").split(",") - for i, elem in enumerate(wd["people"]): - wd["people"][i] = elem.strip() - - # print(f'--- ${wd["survex file"]}$ - {type(wd["survex file"])}') - if wd["survex file"]: # allow for no survex file at all - if wd["survex file"][0] == "[": - wd["survex file"] = wd["survex file"][1:-1] - wd["survex file"] = wd["survex file"].replace('"', "").split(",") - for i, elem in enumerate(wd["survex file"]): - wd["survex file"][i] = elem.strip() - # print(f'--- {wd["survex file"]} - {type(wd["survex file"])}') - - save_json(wd) - walletobject = make_wallet(wallet) - commit_json(wd) - - else: - print("--- INVALID JSON Update form submitted") - print(formj.errors) - return render(request, "errors/generic.html", {"message": formj.errors}) - - elif ( - "walletgoto" in request.POST - ): # not editing wallet data or uploading a file.. going direct to a named wallet - formg = WalletGotoForm(request.POST, request.FILES) - if formg.is_valid(): - walletgoto = request.POST["walletgoto"] - - return HttpResponseRedirect(f'/scanupload/{walletgoto.replace("#",":")}') - - else: # not editing wallet data, uploading a file. But should not overwrite metadata at all. - form = FilesForm(request.POST, request.FILES) - - if form.is_valid(): - # print(f'--- FORM scanupload multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}') - multiple = request.FILES.getlist("uploadfiles") - fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary - - waldata = read_json() - actual_saved = [] - if multiple: - for f in multiple: - try: # crashes in Django os.chmod call if on WSL, but does save file! - saved_filename = fs.save(f.name, content=f) - except: - print(f"\n !! Permissions failure ?! on attempting to save scanfile {f.name}") - if "saved_filename" in locals(): - if saved_filename.is_file(): - actual_saved.append(saved_filename) - # print(f'! - FORM scanupload multiple {actual_saved}') - filesaved = True - # print(f'--- FORM scanupload multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}') - save_json(waldata) - walletobject = make_wallet(wallet) - commit_json(waldata) - # - # Not a POST, so a GET starts here. And also control gets here after a POST is processed. - # - files = [] - dirs = [] - # print(f'! - FORM scanupload - start {wallet} {dirpath}') - if dirpath.is_dir(): - create = False # wallet exists because folder exists, even if nothing in it - try: - for f in dirpath.iterdir(): - if f.is_dir(): - for d in f.iterdir(): - dirs.append(f"{f.name}/{d.name}") - if f.is_file(): - files.append(f.name) - except FileNotFoundError: - files.append( - "(No wallet yet. It would be created if you upload a scan and then save the form with a date.)" - ) - else: - create = True - - if len(files) > 0: - files = sorted(files) - - if dirs: - dirs = sorted(dirs) - try: - waldata = read_json() - except: - message = f"Nasty failure in parsing wallets metadata in {contents_path}. Probably backslash not forward slash in filename path" - return render(request, "errors/generic.html", {"message": message}) - - jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wallet[0:4] / wallet / "contents.json" - # print(f'! - FORM scanupload - jsonfile {jsonfile}') - if not Path(jsonfile).is_file(): - metadataurl = "" - else: - metadataurl = Path("/dwgdataraw", "walletjson") / wallet[0:4] / wallet.replace("#", ":") / "contents.json" - psg = "" - freetext = "" - chkplannr = "" - chkpland = "" - svxfiles = [] - trips = [] - checked = {} - context = {} - if waldata: # should always be true as populated by blank data if json file doesn't exist - - # if not type(waldata["people"])==list: - # if waldata["people"][0] == '"': - # waldata["people"] = waldata["people"][1:-1] - # waldata["people"] = list(waldata["people"]) - - if ( - not waldata["date"] - or not waldata["people"] - or waldata["people"] == ["Unknown"] - or waldata["people"] == [""] - or waldata["cave"] == "" - ): # json file does not exist, blank data, or people not typed into JSON file - # refactor into separate functions for no date set or no people set or no cave set - # print(f'No date set') - print(f"\n - Incomplete, empty or default wallet data {wallet} {waldata=}") - refs = [] - dates = [] - team = [] - caverefs = [] - caves = [] - names = [] - svxf = "" - if waldata["survex file"]: - if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way - waldata["survex file"] = [waldata["survex file"]] - for svxf in waldata["survex file"]: - print(f" - {svxf=}") - if svxf: - svx = Path(svxf) - if svx.suffix.lower() != ".svx": - svx = svx.with_suffix(".svx") - f = Path(settings.SURVEX_DATA) / svx - if f.is_file(): - path = svx.parent / svx.stem - # print(f' - {path=}') - try: - svxfile = SurvexFile.objects.get(path=path) - - print(f" - {svxfile=}") - if svxfile.cave: - caves.append(svxfile.cave) - caverefs.append(svxfile.cave.reference()) - blocks = SurvexBlock.objects.filter(survexfile=svxfile) - for b in blocks: - print(f" - - {b=} {b.scanswallet=} {b.date=}") - if b.scanswallet: - refs.append(b.scanswallet) - if b.scanswallet.walletname == wallet: - if b.date: - dates.append(b.date) - if b.name != b.title: - names.append(str(b.name) + "|" + str(b.title)) - else: - names.append(str(b.name)) - # we can use the people, across all blocks that have this *ref - QSpeople = SurvexPersonRole.objects.filter(survexblock=b) - print(f" - - {QSpeople=}") - for p in QSpeople: - print(f" - - {p.personname} ") - team.append(p.personname) - # else: - # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}') - except: - message = "Specified survex file not found - database may be empty, or this survex file is not *included anywhere." - # return render(request, 'errors/generic.html', {'message': message}) - pass - - if dates: - waldata["date"] = min(dates).isoformat() - print(f" - - {team=} ") - team = list(set(team)) - waldata["people"] = team - - caverefs = list(set(caverefs)) - caves = list(set(caves)) - - if len(caverefs) == 1: - waldata["cave"] = caverefs[0] - print(f" - Setting wallet cave to {caverefs[0]}") - # waldata["description url"] = caves[0] - elif len(caverefs) == 0: - waldata["cave"] = "" - # waldata["description url"] = "" - print(f" - No caves in this wallet {wallet}. ") - else: - waldata["cave"] = "several caves" - # waldata["description url"] = "several.." - print( - f" - More than one Cave {caves} in this wallet {wallet}. Not managed in this troggle release." - ) - if len(names) == 1: - if waldata["name"] == "": - waldata["name"] = names[0] - print(f" - Setting wallet name to {names[0]}") - elif len(names) == 0: - waldata["name"] = "" - print(" - Setting wallet name blank") - else: - waldata["name"] = f"several, please edit: {names}" - print( - f" - More than one block name is relevant {names} in this wallet {wallet}. Not managed in this troggle release." - ) - - if "cave" in waldata: - cave = waldata["cave"] # text string - else: - cave = "" - if waldata["name"]: - psg = waldata["name"] - if "free text" in waldata: - freetext = waldata["free text"] - - # find trips and survex files of the same date - if waldata["date"]: - datestr = waldata["date"].replace(".", "-") - try: - samedate = datetime.date.fromisoformat(datestr) - except ValueError: - # probably a single digit day number. HACKUS MAXIMUS. - # clearly we need to fix this when we first import date strings.. - datestr = datestr[:-1] + "0" + datestr[-1] - print(f" - {datestr=} ") - try: - samedate = datetime.date.fromisoformat(datestr) - except: - try: - samedate = datetime.date.fromisoformat(datestr[:10]) - except: - samedate = None - - thisexpo = Expedition.objects.get(year=int(year)) - if samedate: - svxothers = SurvexBlock.objects.filter(date=samedate) - trips = LogbookEntry.objects.filter(date=samedate) - else: - svxothers = None - trips = None - - else: - svxothers = None - trips = None - - # Survex and survex complaints, comes from json file on disc, not as pre-populated as above - complaints, caveobject = get_complaints([], waldata, svxfiles, files, wallet, wurl) - # print(f' - {caveobject=}') - - for f in checkboxes: - if waldata[f]: - checked[f] = "checked" - - survexsize = str(min(len(str(waldata["survex file"])), 46)) - - try: - thiswallet = Wallet.objects.get(walletname=wallet) - caveifywallet(thiswallet) - thiswallet.ticks = thiswallet.get_ticks() # the complaints in colour form - # fixsurvextick(thiswallet, thiswallet.ticks) - print(thiswallet) - except: - thiswallet = None - context = { - "year": year, - "prev": prev, - "next": next, - "prevy": prevy, - "nexty": nexty, - "files": files, - "dirs": dirs, - "waldata": waldata, - "svxfiles": svxfiles, - "checked": checked, - "trips": trips, - "manywallets": [thiswallet], - "svxothers": svxothers, - "create": create, - "metadataurl": metadataurl, - "complaints": complaints, - "caveobject": caveobject, - "people": waldata["people"], - "peoplesize": str(len(str(waldata["people"]))), - "filesaved": filesaved, - "actual_saved": actual_saved, - } - - return render( - request, - "walletform.html", - { - "form": form, - "wallet": wallet, - **context, - "date": waldata["date"], - #'url': waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))), - "survex": waldata["survex file"], - "survexsize": survexsize, - "cave": cave, - "psg": psg, - "freetext": freetext, - "psgsize": str(max(12, len(str(psg)))), - "freetextsize": str(max(60, len(str(freetext)))), - }, - ) - else: # no wallet data: should never happen as there should be default data in all cases - context = { - "year": year, - "prev": prev, - "next": next, - "prevy": prevy, - "nexty": nexty, - "files": files, - "dirs": dirs, - "waldata": waldata, - "svxfiles": svxfiles, - "checked": checked, - "create": create, - "people": "", - "peoplesize": 12, - "filesaved": filesaved, - "actual_saved": actual_saved, - } - - return render( - request, - "walletform.html", - { - "form": form, - "wallet": wallet, - **context, - "date": "", - #'url': "", 'urlsize': 12, - "survex": "", - "survexsize": 46, - "cave": cave, - "psg": psg, - "freetext": freetext, - "psgsize": 12, - "freetextsize": 20, - }, - ) - - @login_required_if_public def photoupload(request, folder=None): """Upload photo image files into /expofiles/photos/// diff --git a/core/views/wallets.py b/core/views/wallets.py new file mode 100644 index 0000000..7507ed9 --- /dev/null +++ b/core/views/wallets.py @@ -0,0 +1,817 @@ +import datetime +import json +import os +import re +import socket +import subprocess +import urllib +from pathlib import Path + +from django import forms +from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist +from django.core.files.storage import FileSystemStorage +from django.http import HttpResponseRedirect +from django.shortcuts import render + +import settings +from troggle.core.models.caves import Cave +from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry +from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole +from troggle.core.models.troggle import DataIssue, Expedition +from troggle.core.models.wallets import Wallet + +from troggle.core.views.caves import getCave +from troggle.core.views.scans import caveifywallet, oldwallet +from troggle.core.views.uploads import FilesForm + +from troggle.parsers.scans import contentsjson + + +# from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt + +"""File upload 'views' +""" + +todo = """ +- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import + +- Refactor walletedit() as it contains all the wallets 'complaints' code from the old script wallets.py + +- Need to validate uploaded file as being a valid image file, not a dubious script or hack + +- Write equivalent GPX upload form system, similar to walletedit() but in expofiles/gpslogs/ + Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack + +- Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says tunnel is only mostly correct XML + +- Validate image files using a magic recogniser in walletedit() https://pypi.org/project/reportlab/ or + https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file + +- Enable folder creation in dwguploads or as a separate form + +""" +WALLET_BLANK_JSON = { + "cave": "", + "date": "", + "free text": "", + # "description url": "1623/NNN", + "description written": False, + "electronic survey": False, + "elev drawn": False, + "elev not required": False, + "name": "", + "people": ["Unknown"], + "plan drawn": False, + "plan not required": False, + "qms written": False, + "survex file": [], + "survex not required": False, + "website updated": False, +} + + +class WalletGotoForm(forms.Form): # not a model-form, just a form-form + walletgoto = forms.CharField(strip=True, required=False) + + +class WalletForm(forms.Form): # not a model-form, just a form-form + descriptionw = forms.CharField(strip=True, required=False) + people = forms.CharField(strip=True, required=False) + survexnr = forms.CharField(strip=True, required=False) + qmsw = forms.CharField(strip=True, required=False) + date = forms.CharField(strip=True, required=True) # the only required field + websiteupt = forms.CharField(strip=True, required=False) + elevnr = forms.CharField(strip=True, required=False) + cave = forms.CharField(strip=True, required=False) + psg = forms.CharField(strip=True, required=False) + freetext = forms.CharField(strip=True, required=False) + plannr = forms.CharField(strip=True, required=False) + electronic = forms.CharField(strip=True, required=False) + pland = forms.CharField(strip=True, required=False) + elevd = forms.CharField(strip=True, required=False) + # url = forms.CharField(strip=True, required=False) + survex = forms.CharField(strip=True, required=False) + + +xlate = { + # "url": "description url", + "descriptionw": "description written", + "people": "people", + "date": "date", + "cave": "cave", + "plannr": "plan not required", + "survexnr": "survex not required", + "qmsw": "qms written", + "elevnr": "elev not required", + "websiteupt": "website updated", + "electronic": "electronic survey", + "pland": "plan drawn", + "elevd": "elev drawn", + "psg": "name", # a name for this wallet + "freetext": "free text", + "survex": "survex file", +} + + +def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): + """Taken from old script wallets.py and edited to make more comprehensible + Loads the survex files names and processes all complaints + + """ + # If skipping through the wallets on the upload form, the wallet may not yet exist + try: + w = Wallet.objects.get(walletname=wallet) + except ObjectDoesNotExist: + return None, None + + # Date + if not waldata["date"]: + complaints.append( + "A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one." + ) + + # People + if ( + not waldata["people"] + or waldata["people"] == ["NOBODY"] + or waldata["people"] == ["Unknown"] + or waldata["people"] == [""] + ): + complaints.append( + "Someody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data." + ) + + # survex, but get_ticks has already done much of this ?? + survex_complaint = "" + + if waldata["survex file"]: + if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way + waldata["survex file"] = [waldata["survex file"]] + for sx in waldata["survex file"]: + # this logic appears in several places, inc get_ticks(). Refactor. + if sx != "": + if Path(sx).suffix.lower() != ".svx": + sx = sx + ".svx" + svxfiles.append(sx) + if not (Path(settings.SURVEX_DATA) / sx).is_file(): + file_complaint = f"{wallet} Incorrect survex file name. File {sx} was not found in LOSER repo" + complaints.append(file_complaint) + message = f"! {file_complaint}" + print(message) + DataIssue.objects.update_or_create( + parser="scans", message=message, url=wurl + ) # set URL to this wallet folder + else: + try: + sxpath = str(Path(sx).with_suffix("")) + SurvexFile.objects.get(path=sxpath) + except MultipleObjectsReturned: + # can happen if connecting a wallet to a survex file.. i think.. + QSsvxfiles = SurvexFile.objects.filter(path=sxpath) + for s in QSsvxfiles: + print(s.path, s.cave, s.survexdirectory) + # QSsvxfiles[0] # dont' know how this happened, fix later.. + except: + file_complaint = ( + f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.." + ) + complaints.append(file_complaint) + message = f"! {file_complaint}" + print(message) + DataIssue.objects.update_or_create( + parser="scans", message=message, url=wurl + ) # set URL to this wallet folder + + if waldata["survex not required"] and waldata["survex file"] != [""]: + survex_complaint = ( + f'Survex is stated as not required and yet there is a survex file! ({waldata["survex file"]})' + ) + if not waldata["survex not required"] and waldata["survex file"] == [""]: + survex_complaint = "A survex file is required, but has not been specified!" + if survex_complaint: + complaints.append(survex_complaint) + + ticks = w.get_ticks() + + # Notes required + if ticks["N"] != "green": + complaints.append( + "The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey." + ) + + # Plan drawing required + if ticks["P"] != "green": + complaints.append( + "The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found." + ) + + # Elev drawing required + if ticks["E"] != "green": + complaints.append( + "The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found." + ) + + # ETherion + if ticks["T"] != "green": + complaints.append( + "Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox." + ) + + # Description + if not waldata["description written"]: + complaints.append( + "The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done." + ) + # QMs + if not waldata["qms written"] and w.year() and int(w.year()) >= 2015: + complaints.append( + "The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done." + ) + + # Website + if not waldata["website updated"]: + complaints.append( + "The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done." + ) + + # Find the cave, if it exists + if waldata["cave"]: + try: + caveid = waldata["cave"] + if type(caveid) is list: + for i in caveid: + i = i.replace("/", "-") + caveobject = getCave(i) # only the last one gets recorded.. ouch. + else: + caveid = caveid + caveobject = getCave(caveid) + print(f'getCave for id "{waldata["cave"]}" {caveobject}') + # if not caveobject.url == waldata["description url"]: + # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.') + except Cave.MultipleObjectsReturned: + complaints.append(f'The cave ID \'{waldata["cave"]}\' is AMBIGUOUS. Please fix it.') + caveobject = None + except ObjectDoesNotExist: + complaints.append(f'The cave ID \'{waldata["cave"]}\' is not recognised. Please fix it.') + caveobject = None + else: + complaints.append( + 'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"' + ) + caveobject = None + + return complaints, caveobject + + +# @login_required_if_public +def walletedit(request, path=None): + """Upload scanned image files into a wallet on /expofiles + Also display AND EDIT the contents.json data in the wallet. + + This is the main wallet display and edit page. + + The Wallet object and the contents.json file are created when the user + first uploads files. + + This does NOT use a Django model linked to a Django form. Just a simple Django form. + You will find the Django documentation on forms very confusing, + as it covers many very differnet things we do not need. This is simpler. + + This subsumes much of the code which was in the old wallets.py script and so this function is very long + indeed and needs refactoring. + + REWRITE bits using the ticklist, dateify, caveify, populate etc utility functions in core.view.scans.py + """ + git = settings.GIT + filesaved = False + actual_saved = [] + + def read_json(): + """Read JSON from the wallet metadata file in the repo + or fills with blank data if that files can't be read + + Should sanitise to ensure no spurious backslashes e.g. in windows style paths""" + waldata = {} + if contents_path.is_file(): + with open(contents_path) as json_file: + try: + waldata = json.load(json_file) + except: + message = f"! {wallet} Failed to load {contents_path} JSON file" + print(message) + DataIssue.objects.create(parser="scans", message=message, url=wurl) # set URL to this wallet folder + raise + else: # no JSON file exists + print("--- No JSON exists, so creating blank copy") + waldata = WALLET_BLANK_JSON.copy() + if not waldata["survex file"]: + try: + w = Wallet.objects.get(walletname=wallet) + b = SurvexBlock.objects.filter(scanswallet=w) + waldata["survex file"] = [] + for bsf in b: + waldata["survex file"].append(bsf.survexfile.path) + except: + print(f"--- No wallet {wallet} exists in database") + return waldata + + def save_json(jsondict): + # print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}') + if not os.path.exists(contents_path.parent): + print("--- No wallet directory in :drawings: repo, so creating it") + os.makedirs(contents_path.parent) + + with open(contents_path, "w") as jfile: + json.dump(jsondict, jfile, indent=1) + # print(f'--- FINISHED saving to JSON at {contents_path}') + + def make_wallet(walletname): + """We need a wallet Object so that the django template stuff can find the files""" + try: + w, created = Wallet.objects.get_or_create(walletname=walletname) + print(f"--- Wallet string {walletname}, wallet object {w} created new?: {created}") + if created: + w.fpath = Path(settings.SCANS_ROOT, walletname[0:4], walletname) + w.save() + except: + print(f"!-- Wallet string {walletname}, FAIL TO GET or create WALLET OBJECT") + raise + return w + + def commit_json(waldata): + destfolder = contents_path.parent + dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True) + if dr_add.returncode != 0: + msgdata = ( + "Ask a nerd to fix this.\n--" + + dr_add.stderr + + "\n--" + + dr_add.stdout + + "\n--return code: " + + str(dr_add.returncode) + ) + message = ( + f"CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n" + msgdata + ) + print(message) + return render(request, "errors/generic.html", {"message": message}) + else: + if socket.gethostname() != "expo": + comment = f"on dev machine '{socket.gethostname()}' " + else: + comment = "" + if "cave" in waldata: + label = waldata["cave"] + else: + if "name" in waldata: + label = waldata["name"] + else: + label = "" + + dr_commit = subprocess.run( + [git, "commit", "-m", f"JSON update wallet {wallet} {label} {comment}"], + cwd=destfolder, + capture_output=True, + text=True, + ) + # This produces return code = 1 if it commits OK + if dr_commit.returncode != 0: + msgdata = ( + "Ask a nerd to fix this.\n\n" + + dr_commit.stderr + + "\n\n" + + dr_commit.stdout + + "\n\nreturn code: " + + str(dr_commit.returncode) + ) + message = ( + f"Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n" + + msgdata + ) + print(message) + return render(request, "errors/generic.html", {"message": message}) + + def get_logbook_trips(): + return None + + checkboxes = [ + "description written", + "survex not required", + "qms written", + "website updated", + "plan not required", + "plan drawn", + "elev not required", + "elev drawn", + "electronic survey", + ] + if path: + wallet = urllib.parse.unquote(path) + else: + wallet = "2022#00" # improve this later + + year = wallet[:4] + try: + if wallet[4] != "#" and wallet[4] != ":": + # print(f'! - FORM walletedit - {wallet[4]} unurlencoded {unquote(wallet)[4]}') + # print(f'! - FORM walletedit - start {wallet} REDIRECT TO OLDWALLET') + return oldwallet(request, path) + except: + # if nonumeric wallet name for example + return oldwallet(request, path) + + if str(wallet).lower().endswith("indexpages"): + # print(f'! - FORM walletedit - start {wallet} REDIRECT TO OLDWALLET') + return walletindex(request, path) + + if not re.match("(19|20)\d\d[:#]\d\d", wallet): + wallet = "2022:00" # improve this later + # print(f'! - FORM walletedit - start {wallet}') + + if path: + pass + # print(f'! - FORM walletedit - start wallet:{wallet}: path:{path}:') + if int(year) < 1977: + year = "1977" + if int(year) > 2050: + year = "2050" + nexty = f"{int(year)+1}" + prevy = f"{int(year)-1}" + + wnumber = wallet[5:] + next = f"{int(wnumber)+1:02d}" + prev = f"{int(wnumber)-1:02d}" + + if int(wnumber) == 0: + prev = f"{int(wnumber):02d}" + + wurl = f"/walletedit/{wallet}".replace("#", ":") + wallet = wallet.replace(":", "#") + dirpath = Path(settings.SCANS_ROOT, year, wallet) + contents_path = Path(settings.DRAWINGS_DATA, "walletjson") / year / wallet / contentsjson + + form = FilesForm() + + if request.method == "POST": + if "psg" in request.POST: # handle metadata form + formj = WalletForm(request.POST) + # Beware. All fields returned as strings. Must re-type them as lists etc. before using or re-saving + # Also lots of hassle with lists of strings interpreted as a single string + # Unset checkboxes do not return any value, checked ones return "True". So need initialising to False + if formj.is_valid(): + posted = request.POST.copy() + posted.pop("csrfmiddlewaretoken") # discard this + wd = WALLET_BLANK_JSON.copy() + for f in checkboxes: + wd[f] = False + # print(f'--- wd ${f}$ - {wd[f]}') + for f in posted: + wd[xlate[f]] = posted[f].replace("'", '"') + + if posted[f] == "True": + wd[xlate[f]] = True + + wd["people"] = wd["people"][1:-1].replace('"', "").split(",") + for i, elem in enumerate(wd["people"]): + wd["people"][i] = elem.strip() + + # print(f'--- ${wd["survex file"]}$ - {type(wd["survex file"])}') + if wd["survex file"]: # allow for no survex file at all + if wd["survex file"][0] == "[": + wd["survex file"] = wd["survex file"][1:-1] + wd["survex file"] = wd["survex file"].replace('"', "").split(",") + for i, elem in enumerate(wd["survex file"]): + wd["survex file"][i] = elem.strip() + # print(f'--- {wd["survex file"]} - {type(wd["survex file"])}') + + save_json(wd) + walletobject = make_wallet(wallet) + commit_json(wd) + + else: + print("--- INVALID JSON Update form submitted") + print(formj.errors) + return render(request, "errors/generic.html", {"message": formj.errors}) + + elif ( + "walletgoto" in request.POST + ): # not editing wallet data or uploading a file.. going direct to a named wallet + formg = WalletGotoForm(request.POST, request.FILES) + if formg.is_valid(): + walletgoto = request.POST["walletgoto"] + + return HttpResponseRedirect(f'/walletedit/{walletgoto.replace("#",":")}') + + else: # not editing wallet data, uploading a file. But should not overwrite metadata at all. + form = FilesForm(request.POST, request.FILES) + + if form.is_valid(): + # print(f'--- FORM walletedit multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}') + multiple = request.FILES.getlist("uploadfiles") + fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary + + waldata = read_json() + actual_saved = [] + if multiple: + for f in multiple: + try: # crashes in Django os.chmod call if on WSL, but does save file! + saved_filename = fs.save(f.name, content=f) + except: + print(f"\n !! Permissions failure ?! on attempting to save scanfile {f.name}") + if "saved_filename" in locals(): + if saved_filename.is_file(): + actual_saved.append(saved_filename) + # print(f'! - FORM walletedit multiple {actual_saved}') + filesaved = True + # print(f'--- FORM walletedit multiple BUT EMPTY METADATA supposedly {WALLET_BLANK_JSON["date"]=}') + save_json(waldata) + walletobject = make_wallet(wallet) + commit_json(waldata) + # + # Not a POST, so a GET starts here. And also control gets here after a POST is processed. + # + files = [] + dirs = [] + # print(f'! - FORM walletedit - start {wallet} {dirpath}') + if dirpath.is_dir(): + create = False # wallet exists because folder exists, even if nothing in it + try: + for f in dirpath.iterdir(): + if f.is_dir(): + for d in f.iterdir(): + dirs.append(f"{f.name}/{d.name}") + if f.is_file(): + files.append(f.name) + except FileNotFoundError: + files.append( + "(No wallet yet. It would be created if you upload a scan and then save the form with a date.)" + ) + else: + create = True + + if len(files) > 0: + files = sorted(files) + + if dirs: + dirs = sorted(dirs) + try: + waldata = read_json() + except: + message = f"Nasty failure in parsing wallets metadata in {contents_path}. Probably backslash not forward slash in filename path" + return render(request, "errors/generic.html", {"message": message}) + + jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wallet[0:4] / wallet / "contents.json" + # print(f'! - FORM walletedit - jsonfile {jsonfile}') + if not Path(jsonfile).is_file(): + metadataurl = "" + else: + metadataurl = Path("/dwgdataraw", "walletjson") / wallet[0:4] / wallet.replace("#", ":") / "contents.json" + psg = "" + freetext = "" + chkplannr = "" + chkpland = "" + svxfiles = [] + trips = [] + checked = {} + context = {} + if waldata: # should always be true as populated by blank data if json file doesn't exist + + # if not type(waldata["people"])==list: + # if waldata["people"][0] == '"': + # waldata["people"] = waldata["people"][1:-1] + # waldata["people"] = list(waldata["people"]) + + if ( + not waldata["date"] + or not waldata["people"] + or waldata["people"] == ["Unknown"] + or waldata["people"] == [""] + or waldata["cave"] == "" + ): # json file does not exist, blank data, or people not typed into JSON file + # refactor into separate functions for no date set or no people set or no cave set + # print(f'No date set') + print(f"\n - Incomplete, empty or default wallet data {wallet} {waldata=}") + refs = [] + dates = [] + team = [] + caverefs = [] + caves = [] + names = [] + svxf = "" + if waldata["survex file"]: + if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way + waldata["survex file"] = [waldata["survex file"]] + for svxf in waldata["survex file"]: + print(f" - {svxf=}") + if svxf: + svx = Path(svxf) + if svx.suffix.lower() != ".svx": + svx = svx.with_suffix(".svx") + f = Path(settings.SURVEX_DATA) / svx + if f.is_file(): + path = svx.parent / svx.stem + # print(f' - {path=}') + try: + svxfile = SurvexFile.objects.get(path=path) + + print(f" - {svxfile=}") + if svxfile.cave: + caves.append(svxfile.cave) + caverefs.append(svxfile.cave.reference()) + blocks = SurvexBlock.objects.filter(survexfile=svxfile) + for b in blocks: + print(f" - - {b=} {b.scanswallet=} {b.date=}") + if b.scanswallet: + refs.append(b.scanswallet) + if b.scanswallet.walletname == wallet: + if b.date: + dates.append(b.date) + if b.name != b.title: + names.append(str(b.name) + "|" + str(b.title)) + else: + names.append(str(b.name)) + # we can use the people, across all blocks that have this *ref + QSpeople = SurvexPersonRole.objects.filter(survexblock=b) + print(f" - - {QSpeople=}") + for p in QSpeople: + print(f" - - {p.personname} ") + team.append(p.personname) + # else: + # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}') + except: + message = "Specified survex file not found - database may be empty, or this survex file is not *included anywhere." + # return render(request, 'errors/generic.html', {'message': message}) + pass + + if dates: + waldata["date"] = min(dates).isoformat() + print(f" - - {team=} ") + team = list(set(team)) + waldata["people"] = team + + caverefs = list(set(caverefs)) + caves = list(set(caves)) + + if len(caverefs) == 1: + waldata["cave"] = caverefs[0] + print(f" - Setting wallet cave to {caverefs[0]}") + # waldata["description url"] = caves[0] + elif len(caverefs) == 0: + waldata["cave"] = "" + # waldata["description url"] = "" + print(f" - No caves in this wallet {wallet}. ") + else: + waldata["cave"] = "several caves" + # waldata["description url"] = "several.." + print( + f" - More than one Cave {caves} in this wallet {wallet}. Not managed in this troggle release." + ) + if len(names) == 1: + if waldata["name"] == "": + waldata["name"] = names[0] + print(f" - Setting wallet name to {names[0]}") + elif len(names) == 0: + waldata["name"] = "" + print(" - Setting wallet name blank") + else: + waldata["name"] = f"several, please edit: {names}" + print( + f" - More than one block name is relevant {names} in this wallet {wallet}. Not managed in this troggle release." + ) + + if "cave" in waldata: + cave = waldata["cave"] # text string + else: + cave = "" + if waldata["name"]: + psg = waldata["name"] + if "free text" in waldata: + freetext = waldata["free text"] + + # find trips and survex files of the same date + if waldata["date"]: + datestr = waldata["date"].replace(".", "-") + try: + samedate = datetime.date.fromisoformat(datestr) + except ValueError: + # probably a single digit day number. HACKUS MAXIMUS. + # clearly we need to fix this when we first import date strings.. + datestr = datestr[:-1] + "0" + datestr[-1] + print(f" - {datestr=} ") + try: + samedate = datetime.date.fromisoformat(datestr) + except: + try: + samedate = datetime.date.fromisoformat(datestr[:10]) + except: + samedate = None + + thisexpo = Expedition.objects.get(year=int(year)) + if samedate: + svxothers = SurvexBlock.objects.filter(date=samedate) + trips = LogbookEntry.objects.filter(date=samedate) + else: + svxothers = None + trips = None + + else: + svxothers = None + trips = None + + # Survex and survex complaints, comes from json file on disc, not as pre-populated as above + complaints, caveobject = get_complaints([], waldata, svxfiles, files, wallet, wurl) + # print(f' - {caveobject=}') + + for f in checkboxes: + if waldata[f]: + checked[f] = "checked" + + survexsize = str(min(len(str(waldata["survex file"])), 46)) + + try: + thiswallet = Wallet.objects.get(walletname=wallet) + caveifywallet(thiswallet) + thiswallet.ticks = thiswallet.get_ticks() # the complaints in colour form + # fixsurvextick(thiswallet, thiswallet.ticks) + print(thiswallet) + except: + thiswallet = None + context = { + "year": year, + "prev": prev, + "next": next, + "prevy": prevy, + "nexty": nexty, + "files": files, + "dirs": dirs, + "waldata": waldata, + "svxfiles": svxfiles, + "checked": checked, + "trips": trips, + "manywallets": [thiswallet], + "svxothers": svxothers, + "create": create, + "metadataurl": metadataurl, + "complaints": complaints, + "caveobject": caveobject, + "people": waldata["people"], + "peoplesize": str(len(str(waldata["people"]))), + "filesaved": filesaved, + "actual_saved": actual_saved, + } + + return render( + request, + "walletform.html", + { + "form": form, + "wallet": wallet, + **context, + "date": waldata["date"], + #'url': waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))), + "survex": waldata["survex file"], + "survexsize": survexsize, + "cave": cave, + "psg": psg, + "freetext": freetext, + "psgsize": str(max(12, len(str(psg)))), + "freetextsize": str(max(60, len(str(freetext)))), + }, + ) + else: # no wallet data: should never happen as there should be default data in all cases + context = { + "year": year, + "prev": prev, + "next": next, + "prevy": prevy, + "nexty": nexty, + "files": files, + "dirs": dirs, + "waldata": waldata, + "svxfiles": svxfiles, + "checked": checked, + "create": create, + "people": "", + "peoplesize": 12, + "filesaved": filesaved, + "actual_saved": actual_saved, + } + + return render( + request, + "walletform.html", + { + "form": form, + "wallet": wallet, + **context, + "date": "", + #'url': "", 'urlsize': 12, + "survex": "", + "survexsize": 46, + "cave": cave, + "psg": psg, + "freetext": freetext, + "psgsize": 12, + "freetextsize": 20, + }, + ) diff --git a/templates/base.html b/templates/base.html index 84676a5..4c07b0b 100644 --- a/templates/base.html +++ b/templates/base.html @@ -32,7 +32,7 @@ Survex | All Survex | Scans | - Upload Scans | + Upload Scans | Drawings | Upload Drawings | Upload Photos | diff --git a/templates/tasks.html b/templates/tasks.html index ba7fe56..f4f4001 100644 --- a/templates/tasks.html +++ b/templates/tasks.html @@ -66,7 +66,7 @@ ul {list-style: disc}

Upload new data

  • Upload a Tunnel or Therion drawing
    • -
  • Upload a scan into a survey wallet
    • +
  • Upload a scan into a survey wallet
  • Upload or type in a Survex file - rewrite the area/number data in the browser title bar, from 'caves-1623/999/999.svx' to the new number diff --git a/templates/wallet_new.html b/templates/wallet_new.html index a6d331d..2f80816 100644 --- a/templates/wallet_new.html +++ b/templates/wallet_new.html @@ -1,6 +1,6 @@

    To create a wallet, if you have only a survex file and nothing to upload into the wallet, -you currently need to got to Upload Scans, navigate to the new wallet number +you currently need to got to Upload Scans, navigate to the new wallet number and then upload a dummy file, e.g. 'nothingyet.txt'. You can then edit the wallet details: the date, the people involved, and the url of the survexfile which you will already have uploaded using e.g. /survexfile/caves-1623/290/mynewsurvex.svx diff --git a/templates/walletform.html b/templates/walletform.html index 4e86470..948279b 100644 --- a/templates/walletform.html +++ b/templates/walletform.html @@ -23,13 +23,13 @@

    -{{prevy}} +{{prevy}}  ...  -{{year}}:{{prev}} +{{year}}:{{prev}} ← {{wallet}} → -{{year}}:{{next}} +{{year}}:{{next}}  ...  -{{nexty}} +{{nexty}}

    diff --git a/urls.py b/urls.py index 744928b..72a7fc3 100644 --- a/urls.py +++ b/urls.py @@ -22,8 +22,8 @@ from troggle.core.views.other import (controlpanel, exportlogbook, frontpage, from troggle.core.views.prospect import prospecting from troggle.core.views.scans import (allscans, cavewallets, scansingle, walletslistperson, walletslistyear) -from troggle.core.views.uploads import dwgupload, photoupload, scanupload - +from troggle.core.views.uploads import dwgupload, photoupload +from troggle.core.views.wallets import walletedit """This sets the actualurlpatterns[] and urlpatterns[] lists which django uses to resolve urls - in both directions as these are declarative. @@ -87,8 +87,8 @@ trogglepatterns = [ re_path(r'^admin/', admin.site.urls), # includes admin login & logout urls # Uploads - uploading a file - path('scanupload/', scanupload, name='scanupload'), # path=2020#01 - path('scanupload/', scanupload, name='scanupload'), # path=2020#01 + path('walletedit/', walletedit, name='walletedit'), # path=2020#01 + path('walletedit/', walletedit, name='walletedit'), # path=2020#01 path('photoupload/', photoupload, name='photoupload'), # restricted to current year path('photoupload/', photoupload, name='photoupload'), # restricted to current year path('dwgupload/', dwgupload, name='dwgupload'), @@ -172,7 +172,7 @@ trogglepatterns = [ # The survey scans in the wallets. This short-cuts SCANS_URL which is not used anymore and is defunct path('survey_scans/', allscans, name="allscans"), # all the scans in all wallets - path('survey_scans//', scanupload, name="singlewallet"), # replaced singlewallet() + path('survey_scans//', walletedit, name="singlewallet"), # replaced singlewallet() path('survey_scans//', scansingle, name="scansingle"), # works, but html href goes direct to /expofiles/ too path('cave/scans/', cavewallets, name="cavewallets"), # like allscans, but for just one cave