From 432be660a4782183777a74bec54082e6edf19542 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sun, 29 Jan 2023 16:23:58 +0000 Subject: [PATCH] separated out Model for wallet to its own file --- core/admin.py | 4 +- core/models/survex.py | 284 +-------------------------------------- core/models/wallets.py | 293 +++++++++++++++++++++++++++++++++++++++++ parsers/drawings.py | 3 +- parsers/logbooks.py | 6 +- parsers/scans.py | 3 +- parsers/survex.py | 8 +- 7 files changed, 310 insertions(+), 291 deletions(-) create mode 100644 core/models/wallets.py diff --git a/core/admin.py b/core/admin.py index cfdeb22..f7c75db 100644 --- a/core/admin.py +++ b/core/admin.py @@ -8,8 +8,8 @@ from troggle.core.models.caves import (QM, Area, Cave, CaveAndEntrance, Entrance, LogbookEntry, PersonTrip) from troggle.core.models.survex import (DrawingFile, SingleScan, SurvexBlock, SurvexDirectory, SurvexFile, - SurvexPersonRole, SurvexStation, - Wallet) + SurvexPersonRole, SurvexStation) +from troggle.core.models.wallets import Wallet from troggle.core.models.troggle import (DataIssue, Expedition, Person, PersonExpedition) from troggle.core.views.other import exportlogbook diff --git a/core/models/survex.py b/core/models/survex.py index 97c6a3d..e8538f1 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -11,6 +11,7 @@ from django.conf import settings from django.db import models from django.urls import reverse +from troggle.core.models.wallets import Wallet # from troggle.core.models.troggle import DataIssue # circular import. Hmm class SurvexDirectory(models.Model): @@ -98,6 +99,8 @@ class SurvexStation(models.Model): # Single SurvexBlock # class SurvexBlockLookUpManager(models.Manager): + """Don't know what this does, suspect it is part of the Django admin + system""" def lookup(self, name): if name == "": blocknames = [] @@ -124,7 +127,7 @@ class SurvexBlock(models.Model): survexfile = models.ForeignKey("SurvexFile", blank=True, null=True,on_delete=models.SET_NULL) survexpath = models.CharField(max_length=200) # the path for the survex stations - scanswallet = models.ForeignKey("Wallet", null=True,on_delete=models.SET_NULL) # only ONE wallet per block. Th emost recent seen overwites.. ugh. + scanswallet = models.ForeignKey("Wallet", null=True,on_delete=models.SET_NULL) # only ONE wallet per block. The most recent seen overwites.. ugh. legsall = models.IntegerField(null=True) # summary data for this block legslength = models.FloatField(null=True) @@ -158,285 +161,6 @@ class SurvexPersonRole(models.Model): def __str__(self): return str(self.personname) + " - " + str(self.survexblock) -class Wallet(models.Model): - '''We do not keep the JSON values in the database, we query them afresh each time, - but we will change this when we need to do a Django query on e.g. personame - ''' - fpath = models.CharField(max_length=200) - walletname = models.CharField(max_length=200) - walletdate = models.DateField(blank=True, null=True) - walletyear = models.DateField(blank=True, null=True) - - class Meta: - ordering = ('walletname',) - - def get_absolute_url(self): - return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)})) - - def get_json(self): - """Read the JSON file for the wallet and do stuff - """ - #jsonfile = Path(self.fpath, 'contents.json') - - # Get from git repo instead - # :drawings: walletjson/2022/2022#01/contents.json - # fpath = /mnt/d/EXPO/expofiles/surveyscans/1999/1999#02 - fp = Path(self.fpath) - wname = fp.name - wyear = fp.parent.name - wurl = f"/scanupload/{self.walletname}" # .replace('#', ':') - - jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json" - if not Path(jsonfile).is_file(): - #print(f'{jsonfile} is not a file') - return None - else: - with open(jsonfile) as json_f: - try: - waldata = json.load(json_f) - except: - message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file" - #print(message) - raise - if waldata["date"]: - datestr = waldata["date"].replace('.','-') - try: - thisdate = datetime.date.fromisoformat(datestr) - except ValueError: - # probably a single digit day number. HACKUS MAXIMUS. - # clearly we need to fix this when we first import date strings.. - datestr = datestr[:-1] + '0' + datestr[-1] - print(f' - {datestr=} ') - try: - thisdate = datetime.date.fromisoformat(datestr) - self.walletdate = thisdate - self.save() - try: - waldata["date"] = thisdate.isoformat() - except: - message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file" - from troggle.core.models.troggle import \ - DataIssue - DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) - except: - message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file" - from troggle.core.models.troggle import DataIssue - DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) - return waldata - - def year(self): - '''This gets the year syntactically without opening and reading the JSON - ''' - if len(self.walletname) < 5: - return None - if self.walletname[4] != "#": - return None - year = int(self.walletname[0:4]) - if year < 1975 or year > 2050: - return None - else: - self.walletyear = datetime.date(year, 1, 1) - self.save() - return str(year) - - - # Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it - def date(self): - """Reads all the JSON data just to get the JSNON date. - """ - if self.walletdate: - return self.walletdate - if not self.get_json(): - return None - jsondata = self.get_json() # use walrus operator? - - datestr = jsondata["date"] - if not datestr: - return None - else: - datestr = datestr.replace('.','-') - try: - samedate = datetime.date.fromisoformat(datestr) - self.walletdate = samedate.isoformat() - except: - try: - samedate = datetime.date.fromisoformat(datestr[:10]) - self.walletdate = samedate.isoformat() - except: - samedate = None - self.save() - return self.walletdate - - def people(self): - if not self.get_json(): - return None - jsondata = self.get_json() - return jsondata["people"] - - def cave(self): - if not self.get_json(): - return None - jsondata = self.get_json() - return jsondata["cave"] - - def name(self): - if not self.get_json(): - return None - jsondata = self.get_json() - return jsondata["name"] - - def get_fnames(self): - '''Filenames without the suffix, i.e. without the ".jpg" - ''' - dirpath = Path(settings.SCANS_ROOT, self.fpath) # does nowt as fpath is a rooted path already - files = [] - if not self.fpath: - files.append(f"Incorrect path to wallet contents: '{self.fpath}'") - return files - if not dirpath.is_dir(): - files.append(f"Incorrect path to wallet contents: '{self.fpath}'") - return files - else: - try: - for f in dirpath.iterdir(): - if f.is_file(): - files.append(Path(f.name).stem) - else: - files.append(f"-{Path(f.name).stem}-") - except FileNotFoundError: - files.append("FileNotFoundError") - pass - return files - - def fixsurvextick(self, tick): - blocks = SurvexBlock.objects.filter(scanswallet = self) - result = tick - for b in blocks: - if b.survexfile: # if any exist in db, no check for validity or a real file. Refactor. - result = "seagreen" # slightly different shade of green - return result - - def get_ticks(self): - """Reads all the JSON data and sets the colour of the completion tick for each condition - """ - ticks = {} - waldata = self.get_json() - if not waldata: - ticks["S"] = "black" - ticks["C"] = "black" - ticks["Q"] = "black" - ticks["N"] = "black" - ticks["P"] = "black" - ticks["E"] = "black" - ticks["T"] = "black" - ticks["W"] = "black" - return ticks - ticks = {} - - # Initially, are there any required survex files present ? - # Note that we can't set the survexblock here on the wallet as that info is only available while parsing the survex file - survexok = "red" - ticks["S"] = "red" - if waldata["survex not required"]: - survexok = "green" - ticks["S"] = "green" - else: - if waldata["survex file"]: - if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way - waldata["survex file"] = [waldata["survex file"]] - ngood = 0 - nbad = 0 - ticks["S"] = "purple" - for sx in waldata["survex file"]: - #this logic appears in several places, inc uploads.py). Refactor. - if sx !="": - if Path(sx).suffix.lower() != ".svx": - sx = sx + ".svx" - if (Path(settings.SURVEX_DATA) / sx).is_file(): - ngood += 1 - else: - nbad += 1 - if nbad == 0 and ngood >= 1: - ticks["S"] = "green" - elif nbad >= 1 and ngood >= 1: - ticks["S"] = "orange" - elif nbad >= 1 and ngood == 0: - ticks["S"] = "red" - else: - ticks["S"] = "black" - - # Cave Description - if waldata["description written"]: - ticks["C"] = "green" - else: - ticks["C"] = survexok - # QMs - if waldata["qms written"]: - ticks["Q"] = "green" - else: - ticks["Q"] = survexok - if not self.year(): - ticks["Q"] = "darkgrey" - else: - if int(self.year()) < 2015: - ticks["Q"] = "lightgrey" - - - # Notes, Plan, Elevation; Tunnel - if waldata["electronic survey"]: - ticks["N"] = "green" - ticks["P"] = "green" - ticks["E"] = "green" - ticks["T"] = "green" - else: - - files = self.get_fnames() - - # Notes required - notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False) - notes_scanned = reduce(operator.or_, [f.endswith("notes") for f in files], notes_scanned) - if notes_scanned: - ticks["N"] = "green" - else: - ticks["N"] = "red" - - # Plan drawing required - plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False) - plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned) - plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"]) - if plan_drawing_required: - ticks["P"] = "red" - else: - ticks["P"] = "green" - - # Elev drawing required - elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False) - elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned) - elev_scanned = reduce(operator.or_, [f.endswith("elevation") for f in files], elev_scanned) - elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"]) - if elev_drawing_required: - ticks["E"] = "red" - else: - ticks["E"] = "green" - - # Tunnel / Therion - if elev_drawing_required or plan_drawing_required: - ticks["T"] = "red" - else: - ticks["T"] = "green" - - - # Website - if waldata["website updated"]: - ticks["W"] = "green" - else: - ticks["W"] = "red" - - return ticks - - def __str__(self): - return "[" + str(self.walletname) + " (Wallet)]" - class SingleScan(models.Model): """A single file holding an image. Could be raw notes, an elevation plot or whatever """ diff --git a/core/models/wallets.py b/core/models/wallets.py new file mode 100644 index 0000000..e2d3b7a --- /dev/null +++ b/core/models/wallets.py @@ -0,0 +1,293 @@ +import datetime +import json +import operator +import os +import re +from functools import reduce +from pathlib import Path +from urllib.parse import urljoin + +from django.conf import settings +from django.db import models +from django.urls import reverse + +# from troggle.core.models.troggle import DataIssue # circular import. Hmm + +class Wallet(models.Model): + '''We do not keep the JSON values in the database, we query them afresh each time, + but we will change this when we need to do a Django query on e.g. personame + ''' + fpath = models.CharField(max_length=200) + walletname = models.CharField(max_length=200) + walletdate = models.DateField(blank=True, null=True) + walletyear = models.DateField(blank=True, null=True) + + class Meta: + ordering = ('walletname',) + + def get_absolute_url(self): + return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)})) + + def get_json(self): + """Read the JSON file for the wallet and do stuff + """ + #jsonfile = Path(self.fpath, 'contents.json') + + # Get from git repo instead + # :drawings: walletjson/2022/2022#01/contents.json + # fpath = /mnt/d/EXPO/expofiles/surveyscans/1999/1999#02 + fp = Path(self.fpath) + wname = fp.name + wyear = fp.parent.name + wurl = f"/scanupload/{self.walletname}" # .replace('#', ':') + + jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json" + if not Path(jsonfile).is_file(): + #print(f'{jsonfile} is not a file') + return None + else: + with open(jsonfile) as json_f: + try: + waldata = json.load(json_f) + except: + message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file" + #print(message) + raise + if waldata["date"]: + datestr = waldata["date"].replace('.','-') + try: + thisdate = datetime.date.fromisoformat(datestr) + except ValueError: + # probably a single digit day number. HACKUS MAXIMUS. + # clearly we need to fix this when we first import date strings.. + datestr = datestr[:-1] + '0' + datestr[-1] + print(f' - {datestr=} ') + try: + thisdate = datetime.date.fromisoformat(datestr) + self.walletdate = thisdate + self.save() + try: + waldata["date"] = thisdate.isoformat() + except: + message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file" + from troggle.core.models.troggle import \ + DataIssue + DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) + except: + message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file" + from troggle.core.models.troggle import DataIssue + DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) + return waldata + + def year(self): + '''This gets the year syntactically without opening and reading the JSON + ''' + if len(self.walletname) < 5: + return None + if self.walletname[4] != "#": + return None + year = int(self.walletname[0:4]) + if year < 1975 or year > 2050: + return None + else: + self.walletyear = datetime.date(year, 1, 1) + self.save() + return str(year) + + + # Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it + def date(self): + """Reads all the JSON data just to get the JSNON date. + """ + if self.walletdate: + return self.walletdate + if not self.get_json(): + return None + jsondata = self.get_json() # use walrus operator? + + datestr = jsondata["date"] + if not datestr: + return None + else: + datestr = datestr.replace('.','-') + try: + samedate = datetime.date.fromisoformat(datestr) + self.walletdate = samedate.isoformat() + except: + try: + samedate = datetime.date.fromisoformat(datestr[:10]) + self.walletdate = samedate.isoformat() + except: + samedate = None + self.save() + return self.walletdate + + def people(self): + if not self.get_json(): + return None + jsondata = self.get_json() + return jsondata["people"] + + def cave(self): + if not self.get_json(): + return None + jsondata = self.get_json() + return jsondata["cave"] + + def name(self): + if not self.get_json(): + return None + jsondata = self.get_json() + return jsondata["name"] + + def get_fnames(self): + '''Filenames without the suffix, i.e. without the ".jpg" + ''' + dirpath = Path(settings.SCANS_ROOT, self.fpath) # does nowt as fpath is a rooted path already + files = [] + if not self.fpath: + files.append(f"Incorrect path to wallet contents: '{self.fpath}'") + return files + if not dirpath.is_dir(): + files.append(f"Incorrect path to wallet contents: '{self.fpath}'") + return files + else: + try: + for f in dirpath.iterdir(): + if f.is_file(): + files.append(Path(f.name).stem) + else: + files.append(f"-{Path(f.name).stem}-") + except FileNotFoundError: + files.append("FileNotFoundError") + pass + return files + + def fixsurvextick(self, tick): + blocks = SurvexBlock.objects.filter(scanswallet = self) + result = tick + for b in blocks: + if b.survexfile: # if any exist in db, no check for validity or a real file. Refactor. + result = "seagreen" # slightly different shade of green + return result + + def get_ticks(self): + """Reads all the JSON data and sets the colour of the completion tick for each condition + """ + ticks = {} + waldata = self.get_json() + if not waldata: + ticks["S"] = "black" + ticks["C"] = "black" + ticks["Q"] = "black" + ticks["N"] = "black" + ticks["P"] = "black" + ticks["E"] = "black" + ticks["T"] = "black" + ticks["W"] = "black" + return ticks + ticks = {} + + # Initially, are there any required survex files present ? + # Note that we can't set the survexblock here on the wallet as that info is only available while parsing the survex file + survexok = "red" + ticks["S"] = "red" + if waldata["survex not required"]: + survexok = "green" + ticks["S"] = "green" + else: + if waldata["survex file"]: + if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way + waldata["survex file"] = [waldata["survex file"]] + ngood = 0 + nbad = 0 + ticks["S"] = "purple" + for sx in waldata["survex file"]: + #this logic appears in several places, inc uploads.py). Refactor. + if sx !="": + if Path(sx).suffix.lower() != ".svx": + sx = sx + ".svx" + if (Path(settings.SURVEX_DATA) / sx).is_file(): + ngood += 1 + else: + nbad += 1 + if nbad == 0 and ngood >= 1: + ticks["S"] = "green" + elif nbad >= 1 and ngood >= 1: + ticks["S"] = "orange" + elif nbad >= 1 and ngood == 0: + ticks["S"] = "red" + else: + ticks["S"] = "black" + + # Cave Description + if waldata["description written"]: + ticks["C"] = "green" + else: + ticks["C"] = survexok + # QMs + if waldata["qms written"]: + ticks["Q"] = "green" + else: + ticks["Q"] = survexok + if not self.year(): + ticks["Q"] = "darkgrey" + else: + if int(self.year()) < 2015: + ticks["Q"] = "lightgrey" + + + # Notes, Plan, Elevation; Tunnel + if waldata["electronic survey"]: + ticks["N"] = "green" + ticks["P"] = "green" + ticks["E"] = "green" + ticks["T"] = "green" + else: + + files = self.get_fnames() + + # Notes required + notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False) + notes_scanned = reduce(operator.or_, [f.endswith("notes") for f in files], notes_scanned) + if notes_scanned: + ticks["N"] = "green" + else: + ticks["N"] = "red" + + # Plan drawing required + plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False) + plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned) + plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"]) + if plan_drawing_required: + ticks["P"] = "red" + else: + ticks["P"] = "green" + + # Elev drawing required + elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False) + elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned) + elev_scanned = reduce(operator.or_, [f.endswith("elevation") for f in files], elev_scanned) + elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"]) + if elev_drawing_required: + ticks["E"] = "red" + else: + ticks["E"] = "green" + + # Tunnel / Therion + if elev_drawing_required or plan_drawing_required: + ticks["T"] = "red" + else: + ticks["T"] = "green" + + + # Website + if waldata["website updated"]: + ticks["W"] = "green" + else: + ticks["W"] = "red" + + return ticks + + def __str__(self): + return "[" + str(self.walletname) + " (Wallet)]" diff --git a/parsers/drawings.py b/parsers/drawings.py index 963046d..0273210 100644 --- a/parsers/drawings.py +++ b/parsers/drawings.py @@ -4,8 +4,9 @@ import stat from pathlib import Path import settings -from troggle.core.models.survex import DrawingFile, Wallet +from troggle.core.models.survex import DrawingFile from troggle.core.models.troggle import DataIssue +from troggle.core.models.wallets import Wallet """Searches through all the :drawings: repository looking for tunnel and therion files diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 1d23215..22cdf84 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -66,7 +66,7 @@ ENTRIES = { "2017": 74, "2016": 86, "2015": 80, - "2014": 66, + "2014": 67, "2013": 52, "2012": 76, "2011": 71, @@ -366,14 +366,14 @@ def parser_html(year, expedition, txt, seq=""): if check in dupl: dupl[check] += 1 triptitle = f"{triptitle} #{dupl[check]}" - print(f" - {triptitle} -- {date}") + print(f" - {triptitle} -- {ldate}") else: dupl[check] = 1 tu = tidy_time_underground(tu) trippersons, author = tidy_trip_persons(trippeople, expedition, tu, tid) tripcave = tidy_trip_cave(place) - tripcontent = tidy_trip_image_urls(tripcontent, date) + tripcontent = tidy_trip_image_urls(tripcontent, ldate) tid = tidy_tid(tid, triptitle) entrytuple = (ldate, place, tripcave, triptitle, tripcontent, trippersons, author, expedition, tu, tid) diff --git a/parsers/scans.py b/parsers/scans.py index d3fb9ab..00538fe 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -2,8 +2,9 @@ import datetime from pathlib import Path import settings -from troggle.core.models.survex import SingleScan, Wallet +from troggle.core.models.survex import SingleScan from troggle.core.models.troggle import DataIssue +from troggle.core.models.wallets import Wallet """Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. """ diff --git a/parsers/survex.py b/parsers/survex.py index b741e30..bbe61a7 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -121,7 +121,7 @@ def get_offending_filename(path): """ return "/survexfile/" + path + ".svx" -trip_people_cache = {} # DANGEROUS, should clean it on PUSH/POP begin/end +trip_people_cache = {} # per survexblock, so robust wrt PUSH/POP begin/end def get_team_on_trip(survexblock): """Uses a cache to avoid a database query if it doesn't need to. Only used for complete team.""" @@ -145,7 +145,7 @@ def get_people_on_trip(survexblock): return list(set(people)) -trip_person_cache = {} # pre survexblock, so robust wrt PUSH/POP begin/end +trip_person_cache = {} # per survexblock, so robust wrt PUSH/POP begin/end def put_person_on_trip(survexblock, personexpedition, tm): """Uses a cache to avoid a database query if it doesn't need to. Only used for a single person""" @@ -170,7 +170,7 @@ def put_person_on_trip(survexblock, personexpedition, tm): trip_person_cache[(survexblock, personexpedition)] = 1 return False -person_pending_cache = {} # pre survexblock, so robust wrt PUSH/POP begin/end +person_pending_cache = {} # per survexblock, so robust wrt PUSH/POP begin/end def add_to_pending(survexblock, tm): """Collects team names before we have a date so cannot validate against expo attendance yet""" @@ -354,7 +354,7 @@ class LoadingSurvex: knowing the year. Unless its parent has an identified expo""" if survexblock.parent.name == "troggle_unseens": - # Bolluxed up if we try to inherit from this random junk + # Bolluxed up if we try to inherit from this random junk, so don't. return expo = survexblock.expedition # may be None if no *date yet