expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-21 23:01:52 +00:00
Code Issues 19 Projects 3 Activity

separated out Model for wallet to its own file

Browse Source
This commit is contained in:
Philip Sargent 2023-01-29 16:23:58 +00:00
parent f73640522b
commit 432be660a4
7 changed files with 310 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/admin.py
View File

@ -8,8 +8,8 @@ from troggle.core.models.caves import (QM, Area, Cave, CaveAndEntrance,
Entrance, LogbookEntry, PersonTrip)
from troggle.core.models.survex import (DrawingFile, SingleScan, SurvexBlock,
SurvexDirectory, SurvexFile,
SurvexPersonRole, SurvexStation,
Wallet)
SurvexPersonRole, SurvexStation)
from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import (DataIssue, Expedition, Person,
PersonExpedition)
from troggle.core.views.other import exportlogbook

284
core/models/survex.py
View File

@ -11,6 +11,7 @@ from django.conf import settings
from django.db import models
from django.urls import reverse
from troggle.core.models.wallets import Wallet
# from troggle.core.models.troggle import DataIssue # circular import. Hmm
class SurvexDirectory(models.Model):
@ -98,6 +99,8 @@ class SurvexStation(models.Model):
# Single SurvexBlock
#
class SurvexBlockLookUpManager(models.Manager):
"""Don't know what this does, suspect it is part of the Django admin
system"""
def lookup(self, name):
if name == "":
blocknames = []
@ -124,7 +127,7 @@ class SurvexBlock(models.Model):
survexfile = models.ForeignKey("SurvexFile", blank=True, null=True,on_delete=models.SET_NULL)
survexpath = models.CharField(max_length=200) # the path for the survex stations
scanswallet = models.ForeignKey("Wallet", null=True,on_delete=models.SET_NULL) # only ONE wallet per block. Th emost recent seen overwites.. ugh.
scanswallet = models.ForeignKey("Wallet", null=True,on_delete=models.SET_NULL) # only ONE wallet per block. The most recent seen overwites.. ugh.
legsall = models.IntegerField(null=True) # summary data for this block
legslength = models.FloatField(null=True)
@ -158,285 +161,6 @@ class SurvexPersonRole(models.Model):
def __str__(self):
return str(self.personname) + " - " + str(self.survexblock)
class Wallet(models.Model):
'''We do not keep the JSON values in the database, we query them afresh each time,
but we will change this when we need to do a Django query on e.g. personame
'''
fpath = models.CharField(max_length=200)
walletname = models.CharField(max_length=200)
walletdate = models.DateField(blank=True, null=True)
walletyear = models.DateField(blank=True, null=True)
class Meta:
ordering = ('walletname',)
def get_absolute_url(self):
return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)}))
def get_json(self):
"""Read the JSON file for the wallet and do stuff
"""
#jsonfile = Path(self.fpath, 'contents.json')
# Get from git repo instead
# :drawings: walletjson/2022/2022#01/contents.json
# fpath = /mnt/d/EXPO/expofiles/surveyscans/1999/1999#02
fp = Path(self.fpath)
wname = fp.name
wyear = fp.parent.name
wurl = f"/scanupload/{self.walletname}" # .replace('#', ':')
jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
if not Path(jsonfile).is_file():
#print(f'{jsonfile} is not a file')
return None
else:
with open(jsonfile) as json_f:
try:
waldata = json.load(json_f)
except:
message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
#print(message)
raise
if waldata["date"]:
datestr = waldata["date"].replace('.','-')
try:
thisdate = datetime.date.fromisoformat(datestr)
except ValueError:
# probably a single digit day number. HACKUS MAXIMUS.
# clearly we need to fix this when we first import date strings..
datestr = datestr[:-1] + '0' + datestr[-1]
print(f' - {datestr=} ')
try:
thisdate = datetime.date.fromisoformat(datestr)
self.walletdate = thisdate
self.save()
try:
waldata["date"] = thisdate.isoformat()
except:
message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file"
from troggle.core.models.troggle import \
DataIssue
DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl)
except:
message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
from troggle.core.models.troggle import DataIssue
DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl)
return waldata
def year(self):
'''This gets the year syntactically without opening and reading the JSON
'''
if len(self.walletname) < 5:
return None
if self.walletname[4] != "#":
return None
year = int(self.walletname[0:4])
if year < 1975 or year > 2050:
return None
else:
self.walletyear = datetime.date(year, 1, 1)
self.save()
return str(year)
# Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it
def date(self):
"""Reads all the JSON data just to get the JSNON date.
"""
if self.walletdate:
return self.walletdate
if not self.get_json():
return None
jsondata = self.get_json() # use walrus operator?
datestr = jsondata["date"]
if not datestr:
return None
else:
datestr = datestr.replace('.','-')
try:
samedate = datetime.date.fromisoformat(datestr)
self.walletdate = samedate.isoformat()
except:
try:
samedate = datetime.date.fromisoformat(datestr[:10])
self.walletdate = samedate.isoformat()
except:
samedate = None
self.save()
return self.walletdate
def people(self):
if not self.get_json():
return None
jsondata = self.get_json()
return jsondata["people"]
def cave(self):
if not self.get_json():
return None
jsondata = self.get_json()
return jsondata["cave"]
def name(self):
if not self.get_json():
return None
jsondata = self.get_json()
return jsondata["name"]
def get_fnames(self):
'''Filenames without the suffix, i.e. without the ".jpg"
'''
dirpath = Path(settings.SCANS_ROOT, self.fpath) # does nowt as fpath is a rooted path already
files = []
if not self.fpath:
files.append(f"Incorrect path to wallet contents: '{self.fpath}'")
return files
if not dirpath.is_dir():
files.append(f"Incorrect path to wallet contents: '{self.fpath}'")
return files
else:
try:
for f in dirpath.iterdir():
if f.is_file():
files.append(Path(f.name).stem)
else:
files.append(f"-{Path(f.name).stem}-")
except FileNotFoundError:
files.append("FileNotFoundError")
pass
return files
def fixsurvextick(self, tick):
blocks = SurvexBlock.objects.filter(scanswallet = self)
result = tick
for b in blocks:
if b.survexfile: # if any exist in db, no check for validity or a real file. Refactor.
result = "seagreen" # slightly different shade of green
return result
def get_ticks(self):
"""Reads all the JSON data and sets the colour of the completion tick for each condition
"""
ticks = {}
waldata = self.get_json()
if not waldata:
ticks["S"] = "black"
ticks["C"] = "black"
ticks["Q"] = "black"
ticks["N"] = "black"
ticks["P"] = "black"
ticks["E"] = "black"
ticks["T"] = "black"
ticks["W"] = "black"
return ticks
ticks = {}
# Initially, are there any required survex files present ?
# Note that we can't set the survexblock here on the wallet as that info is only available while parsing the survex file
survexok = "red"
ticks["S"] = "red"
if waldata["survex not required"]:
survexok = "green"
ticks["S"] = "green"
else:
if waldata["survex file"]:
if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
waldata["survex file"] = [waldata["survex file"]]
ngood = 0
nbad = 0
ticks["S"] = "purple"
for sx in waldata["survex file"]:
#this logic appears in several places, inc uploads.py). Refactor.
if sx !="":
if Path(sx).suffix.lower() != ".svx":
sx = sx + ".svx"
if (Path(settings.SURVEX_DATA) / sx).is_file():
ngood += 1
else:
nbad += 1
if nbad == 0 and ngood >= 1:
ticks["S"] = "green"
elif nbad >= 1 and ngood >= 1:
ticks["S"] = "orange"
elif nbad >= 1 and ngood == 0:
ticks["S"] = "red"
else:
ticks["S"] = "black"
# Cave Description
if waldata["description written"]:
ticks["C"] = "green"
else:
ticks["C"] = survexok
# QMs
if waldata["qms written"]:
ticks["Q"] = "green"
else:
ticks["Q"] = survexok
if not self.year():
ticks["Q"] = "darkgrey"
else:
if int(self.year()) < 2015:
ticks["Q"] = "lightgrey"
# Notes, Plan, Elevation; Tunnel
if waldata["electronic survey"]:
ticks["N"] = "green"
ticks["P"] = "green"
ticks["E"] = "green"
ticks["T"] = "green"
else:
files = self.get_fnames()
# Notes required
notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False)
notes_scanned = reduce(operator.or_, [f.endswith("notes") for f in files], notes_scanned)
if notes_scanned:
ticks["N"] = "green"
else:
ticks["N"] = "red"
# Plan drawing required
plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False)
plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned)
plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"])
if plan_drawing_required:
ticks["P"] = "red"
else:
ticks["P"] = "green"
# Elev drawing required
elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False)
elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned)
elev_scanned = reduce(operator.or_, [f.endswith("elevation") for f in files], elev_scanned)
elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"])
if elev_drawing_required:
ticks["E"] = "red"
else:
ticks["E"] = "green"
# Tunnel / Therion
if elev_drawing_required or plan_drawing_required:
ticks["T"] = "red"
else:
ticks["T"] = "green"
# Website
if waldata["website updated"]:
ticks["W"] = "green"
else:
ticks["W"] = "red"
return ticks
def __str__(self):
return "[" + str(self.walletname) + " (Wallet)]"
class SingleScan(models.Model):
"""A single file holding an image. Could be raw notes, an elevation plot or whatever
"""

293
core/models/wallets.py Normal file
View File

@ -0,0 +1,293 @@
import datetime
import json
import operator
import os
import re
from functools import reduce
from pathlib import Path
from urllib.parse import urljoin
from django.conf import settings
from django.db import models
from django.urls import reverse
# from troggle.core.models.troggle import DataIssue # circular import. Hmm
class Wallet(models.Model):
'''We do not keep the JSON values in the database, we query them afresh each time,
but we will change this when we need to do a Django query on e.g. personame
'''
fpath = models.CharField(max_length=200)
walletname = models.CharField(max_length=200)
walletdate = models.DateField(blank=True, null=True)
walletyear = models.DateField(blank=True, null=True)
class Meta:
ordering = ('walletname',)
def get_absolute_url(self):
return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)}))
def get_json(self):
"""Read the JSON file for the wallet and do stuff
"""
#jsonfile = Path(self.fpath, 'contents.json')
# Get from git repo instead
# :drawings: walletjson/2022/2022#01/contents.json
# fpath = /mnt/d/EXPO/expofiles/surveyscans/1999/1999#02
fp = Path(self.fpath)
wname = fp.name
wyear = fp.parent.name
wurl = f"/scanupload/{self.walletname}" # .replace('#', ':')
jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
if not Path(jsonfile).is_file():
#print(f'{jsonfile} is not a file')
return None
else:
with open(jsonfile) as json_f:
try:
waldata = json.load(json_f)
except:
message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
#print(message)
raise
if waldata["date"]:
datestr = waldata["date"].replace('.','-')
try:
thisdate = datetime.date.fromisoformat(datestr)
except ValueError:
# probably a single digit day number. HACKUS MAXIMUS.
# clearly we need to fix this when we first import date strings..
datestr = datestr[:-1] + '0' + datestr[-1]
print(f' - {datestr=} ')
try:
thisdate = datetime.date.fromisoformat(datestr)
self.walletdate = thisdate
self.save()
try:
waldata["date"] = thisdate.isoformat()
except:
message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file"
from troggle.core.models.troggle import \
DataIssue
DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl)
except:
message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
from troggle.core.models.troggle import DataIssue
DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl)
return waldata
def year(self):
'''This gets the year syntactically without opening and reading the JSON
'''
if len(self.walletname) < 5:
return None
if self.walletname[4] != "#":
return None
year = int(self.walletname[0:4])
if year < 1975 or year > 2050:
return None
else:
self.walletyear = datetime.date(year, 1, 1)
self.save()
return str(year)
# Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it
def date(self):
"""Reads all the JSON data just to get the JSNON date.
"""
if self.walletdate:
return self.walletdate
if not self.get_json():
return None
jsondata = self.get_json() # use walrus operator?
datestr = jsondata["date"]
if not datestr:
return None
else:
datestr = datestr.replace('.','-')
try:
samedate = datetime.date.fromisoformat(datestr)
self.walletdate = samedate.isoformat()
except:
try:
samedate = datetime.date.fromisoformat(datestr[:10])
self.walletdate = samedate.isoformat()
except:
samedate = None
self.save()
return self.walletdate
def people(self):
if not self.get_json():
return None
jsondata = self.get_json()
return jsondata["people"]
def cave(self):
if not self.get_json():
return None
jsondata = self.get_json()
return jsondata["cave"]
def name(self):
if not self.get_json():
return None
jsondata = self.get_json()
return jsondata["name"]
def get_fnames(self):
'''Filenames without the suffix, i.e. without the ".jpg"
'''
dirpath = Path(settings.SCANS_ROOT, self.fpath) # does nowt as fpath is a rooted path already
files = []
if not self.fpath:
files.append(f"Incorrect path to wallet contents: '{self.fpath}'")
return files
if not dirpath.is_dir():
files.append(f"Incorrect path to wallet contents: '{self.fpath}'")
return files
else:
try:
for f in dirpath.iterdir():
if f.is_file():
files.append(Path(f.name).stem)
else:
files.append(f"-{Path(f.name).stem}-")
except FileNotFoundError:
files.append("FileNotFoundError")
pass
return files
def fixsurvextick(self, tick):
blocks = SurvexBlock.objects.filter(scanswallet = self)
result = tick
for b in blocks:
if b.survexfile: # if any exist in db, no check for validity or a real file. Refactor.
result = "seagreen" # slightly different shade of green
return result
def get_ticks(self):
"""Reads all the JSON data and sets the colour of the completion tick for each condition
"""
ticks = {}
waldata = self.get_json()
if not waldata:
ticks["S"] = "black"
ticks["C"] = "black"
ticks["Q"] = "black"
ticks["N"] = "black"
ticks["P"] = "black"
ticks["E"] = "black"
ticks["T"] = "black"
ticks["W"] = "black"
return ticks
ticks = {}
# Initially, are there any required survex files present ?
# Note that we can't set the survexblock here on the wallet as that info is only available while parsing the survex file
survexok = "red"
ticks["S"] = "red"
if waldata["survex not required"]:
survexok = "green"
ticks["S"] = "green"
else:
if waldata["survex file"]:
if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
waldata["survex file"] = [waldata["survex file"]]
ngood = 0
nbad = 0
ticks["S"] = "purple"
for sx in waldata["survex file"]:
#this logic appears in several places, inc uploads.py). Refactor.
if sx !="":
if Path(sx).suffix.lower() != ".svx":
sx = sx + ".svx"
if (Path(settings.SURVEX_DATA) / sx).is_file():
ngood += 1
else:
nbad += 1
if nbad == 0 and ngood >= 1:
ticks["S"] = "green"
elif nbad >= 1 and ngood >= 1:
ticks["S"] = "orange"
elif nbad >= 1 and ngood == 0:
ticks["S"] = "red"
else:
ticks["S"] = "black"
# Cave Description
if waldata["description written"]:
ticks["C"] = "green"
else:
ticks["C"] = survexok
# QMs
if waldata["qms written"]:
ticks["Q"] = "green"
else:
ticks["Q"] = survexok
if not self.year():
ticks["Q"] = "darkgrey"
else:
if int(self.year()) < 2015:
ticks["Q"] = "lightgrey"
# Notes, Plan, Elevation; Tunnel
if waldata["electronic survey"]:
ticks["N"] = "green"
ticks["P"] = "green"
ticks["E"] = "green"
ticks["T"] = "green"
else:
files = self.get_fnames()
# Notes required
notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False)
notes_scanned = reduce(operator.or_, [f.endswith("notes") for f in files], notes_scanned)
if notes_scanned:
ticks["N"] = "green"
else:
ticks["N"] = "red"
# Plan drawing required
plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False)
plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned)
plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"])
if plan_drawing_required:
ticks["P"] = "red"
else:
ticks["P"] = "green"
# Elev drawing required
elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False)
elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned)
elev_scanned = reduce(operator.or_, [f.endswith("elevation") for f in files], elev_scanned)
elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"])
if elev_drawing_required:
ticks["E"] = "red"
else:
ticks["E"] = "green"
# Tunnel / Therion
if elev_drawing_required or plan_drawing_required:
ticks["T"] = "red"
else:
ticks["T"] = "green"
# Website
if waldata["website updated"]:
ticks["W"] = "green"
else:
ticks["W"] = "red"
return ticks
def __str__(self):
return "[" + str(self.walletname) + " (Wallet)]"

3
parsers/drawings.py
View File

@ -4,8 +4,9 @@ import stat
from pathlib import Path
import settings
from troggle.core.models.survex import DrawingFile, Wallet
from troggle.core.models.survex import DrawingFile
from troggle.core.models.troggle import DataIssue
from troggle.core.models.wallets import Wallet
"""Searches through all the :drawings: repository looking
for tunnel and therion files

6
parsers/logbooks.py
View File

@ -66,7 +66,7 @@ ENTRIES = {
"2017": 74,
"2016": 86,
"2015": 80,
"2014": 66,
"2014": 67,
"2013": 52,
"2012": 76,
"2011": 71,
@ -366,14 +366,14 @@ def parser_html(year, expedition, txt, seq=""):
if check in dupl:
dupl[check] += 1
triptitle = f"{triptitle} #{dupl[check]}"
print(f" - {triptitle} -- {date}")
print(f" - {triptitle} -- {ldate}")
else:
dupl[check] = 1
tu = tidy_time_underground(tu)
trippersons, author = tidy_trip_persons(trippeople, expedition, tu, tid)
tripcave = tidy_trip_cave(place)
tripcontent = tidy_trip_image_urls(tripcontent, date)
tripcontent = tidy_trip_image_urls(tripcontent, ldate)
tid = tidy_tid(tid, triptitle)
entrytuple = (ldate, place, tripcave, triptitle, tripcontent, trippersons, author, expedition, tu, tid)

3
parsers/scans.py
View File

@ -2,8 +2,9 @@ import datetime
from pathlib import Path
import settings
from troggle.core.models.survex import SingleScan, Wallet
from troggle.core.models.survex import SingleScan
from troggle.core.models.troggle import DataIssue
from troggle.core.models.wallets import Wallet
"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
"""

8
parsers/survex.py
View File

@ -121,7 +121,7 @@ def get_offending_filename(path):
"""
return "/survexfile/" + path + ".svx"
trip_people_cache = {} # DANGEROUS, should clean it on PUSH/POP begin/end
trip_people_cache = {} # per survexblock, so robust wrt PUSH/POP begin/end
def get_team_on_trip(survexblock):
"""Uses a cache to avoid a database query if it doesn't need to.
Only used for complete team."""
@ -145,7 +145,7 @@ def get_people_on_trip(survexblock):
return list(set(people))
trip_person_cache = {} # pre survexblock, so robust wrt PUSH/POP begin/end
trip_person_cache = {} # per survexblock, so robust wrt PUSH/POP begin/end
def put_person_on_trip(survexblock, personexpedition, tm):
"""Uses a cache to avoid a database query if it doesn't need to.
Only used for a single person"""
@ -170,7 +170,7 @@ def put_person_on_trip(survexblock, personexpedition, tm):
trip_person_cache[(survexblock, personexpedition)] = 1
return False
person_pending_cache = {} # pre survexblock, so robust wrt PUSH/POP begin/end
person_pending_cache = {} # per survexblock, so robust wrt PUSH/POP begin/end
def add_to_pending(survexblock, tm):
"""Collects team names before we have a date so cannot validate against
expo attendance yet"""
@ -354,7 +354,7 @@ class LoadingSurvex:
knowing the year. Unless its parent has an identified expo"""
if survexblock.parent.name == "troggle_unseens":
# Bolluxed up if we try to inherit from this random junk
# Bolluxed up if we try to inherit from this random junk, so don't.
return
expo = survexblock.expedition # may be None if no *date yet