diff --git a/core/models/caves.py b/core/models/caves.py index e077055..9d98e22 100644 --- a/core/models/caves.py +++ b/core/models/caves.py @@ -88,7 +88,7 @@ class Cave(TroggleModel): url = models.CharField(max_length=300, blank=True, null=True, unique = True) class Meta: - # we do not enforce uniqueness at the db level as that causes confusing errors for users. + # we do not enforce uniqueness at the db level as that causes confusing errors for newbie maintainers # unique_together = (("area", "kataster_number"), ("area", "unofficial_number")) ordering = ("kataster_code", "unofficial_number") diff --git a/core/models/wallets.py b/core/models/wallets.py index bae919a..3f59da0 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -10,6 +10,8 @@ from django.conf import settings from django.db import models from django.urls import reverse +from troggle.core.views.caves import get_cave_leniently + # from troggle.core.models.survex import SurvexBlock # from troggle.core.models.troggle import DataIssue # circular import. Hmm @@ -74,12 +76,15 @@ archaic_wallets = [ class Wallet(models.Model): """We do not keep the JSON values in the database, we query them afresh each time, but we may change this if we need to do a Django query on e.g. personame + + ManyToMany field uses modern Django: a hidden Class, unlike CaveAndEntrances which is explict and visible. """ fpath = models.CharField(max_length=200) walletname = models.CharField(max_length=200) walletdate = models.DateField(blank=True, null=True) walletyear = models.DateField(blank=True, null=True) + caves = models.ManyToManyField("Cave", related_name="wallets") class Meta: ordering = ("walletname",) @@ -139,7 +144,31 @@ class Wallet(models.Model): from troggle.core.models.troggle import DataIssue DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) return waldata - + + def allcaves(self): + """Reads all the JSON data just to get the JSON date.""" + if not (jsondata := self.get_json()): # WALRUS + print(f"Failed to read JSON file {self}") + return None + cavelist = jsondata["cave"] + if type(cavelist) is list: + for i in cavelist: + if i != "": + i = i.replace("/", "-") + caveobject = get_cave_leniently(i) + self.caves.add(caveobject) # new many-to-many field + else: + # either single cave or the square barckets have been removed and it s a singoe string + ids = cavelist.split(",") + for i in ids: + j = i.replace("'","").replace("/", "-").strip('[] "') + if i != "": + try: + caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data + self.caves.add(caveobject) + except: + print(f"FAIL adding cave to wallet.caves '{j}'") + pass def year(self): """This gets the year syntactically without opening and reading the JSON""" if len(self.walletname) < 5: @@ -180,6 +209,7 @@ class Wallet(models.Model): self.save() return self.walletdate + # for gods sake redo this, it parse JSON twice every time.. def people(self): if not self.get_json(): return None diff --git a/core/views/caves.py b/core/views/caves.py index bd559bb..bc5d6d7 100644 --- a/core/views/caves.py +++ b/core/views/caves.py @@ -40,6 +40,20 @@ todo = """ https://zerotobyte.com/how-to-use-django-select-related-and-prefetch-related/ """ +def get_cave_leniently(caveid): + try: + c = getCave(caveid) + if c: + return c + except: + # print(f"get_cave_leniently FAIL {caveid}") + try: + c = getCave("1623-"+caveid) + if c: + return c + except: + return None + def getCaves(cave_id): """Only gets called if a call to getCave() raises a MultipleObjects exception @@ -59,7 +73,8 @@ def getCaves(cave_id): def getCave(cave_id): - """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm. + """Returns a cave object when given a cave name or number. + It is used by views including cavehref, ent, wallets and qm. TO DO: search GCavelookup first, which should raise a MultpleObjectsReturned exception if there are duplicates""" diff --git a/core/views/scans.py b/core/views/scans.py index c7295a9..4d9f851 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -52,20 +52,15 @@ def populatewallet(w): def caveifywallet(w): - """Gets the cave from the list of survex files, - only selects one of them though. Only used for display. - FIX THIS to display many caves - """ + """Gets the caves from the list of survex files, + """ # print(f' - Caveify {w=}') blocknames = [] blocks = SurvexBlock.objects.filter(scanswallet=w) for b in blocks: # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath if b.survexfile.cave: - w.caveobj = ( - b.survexfile.cave - ) # just gets the last one, randomly. SHould make this a list or many:many ideally - w.cave = w.caveobj + w.caves.add(b.survexfile.cave) if b.name: blocknames.append(b.name) @@ -120,11 +115,10 @@ def is_cave(wallet, id): if not id: return False Gcavelookup = GetCaveLookup() - id = id.strip("' []'") if id in Gcavelookup: return True else: - # Historic wallets used just 2 or 3 digits and were all 1623 area. So, just for these wallets, + # Historic wallets were all 1623 area. So, just for wallets, # assume it is 1623-xxx if f"1623-{id}" in Gcavelookup: print(f" - Should modify wallet {wallet} to use 1623- prefix for cave <{id}>") @@ -145,26 +139,26 @@ def fillblankothers(w): Gcavelookup = GetCaveLookup() + caveifywallet(w) + wcaveid = w.cave() if not wcaveid or wcaveid == "": - caveifywallet(w) - else: if type(wcaveid) == list: for i in wcaveid: - i = i.strip("' []'") + i = i.strip("' []\"") if is_cave(w,i): - w.caveobj = Gcavelookup[i] # just sets it to the last one found. nasty. bug waiting to happen + w.caves.add(Gcavelookup[i]) elif wcaveid.find(',') != -1: # it's a list of cave ids as a string ids = wcaveid.split(',') for i in ids: - i = i.strip("' []'") + i = i.strip("' []\"") if is_cave(w,i): - w.caveobj = Gcavelookup[i] # just sets it to the last one found. nasty. bug waiting to happen + w.caves.add(Gcavelookup[i]) else: if is_cave(w,wcaveid): - w.caveobj = Gcavelookup[wcaveid.strip("' []'")] - + w.caves.add(Gcavelookup[i]) + def fixsurvextick(w, ticks): ticks["S"] = w.fixsurvextick(ticks["S"]) diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py index 55b51da..a9022f3 100644 --- a/core/views/wallets_edit.py +++ b/core/views/wallets_edit.py @@ -23,7 +23,7 @@ from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.models.wallets import Wallet, YEAR_RANGE, make_valid_date from troggle.core.views.auth import login_required_if_public -from troggle.core.views.caves import getCave +from troggle.core.views.caves import getCave, get_cave_leniently from troggle.core.views.scans import caveifywallet, oldwallet from troggle.core.views.uploads import FilesForm @@ -108,7 +108,8 @@ xlate = { "survex": "survex file", } - + + def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): """Taken from old script wallets.py and edited to make more comprehensible Loads the survex files names and processes all complaints @@ -231,20 +232,29 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): ) # Find the cave, if it exists + # Just for wallets, we are lenient about whether the 1623- prefix has been written down. if waldata["cave"]: + caveobject = None try: caveid = waldata["cave"] if type(caveid) is list: for i in caveid: i = i.replace("/", "-") - caveobject = getCave(i) # only the last one gets recorded.. ouch. + caveobject = get_cave_leniently(i) + w.caves.add(caveobject) # new many-to-many field + #print(w.caves) else: - caveid = caveid # ?urk? why? - try: - caveobject = getCave(caveid) # may fail if garbage value ,e.g. space, in wallet data - except: - caveobject = None - print(f'getCave for id "{waldata["cave"]}" {caveobject}') + # either single cave or the square barckets have been removed + ids = caveid.split(",") + for i in ids: + j = i.replace("'","").strip('[] "') + #print(j) + try: + caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data + w.caves.add(caveobject) + except: + pass + print(f'get_cave_leniently from "{waldata["cave"]}" => {caveobject}') # if not caveobject.url == waldata["description url"]: # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.') except Cave.MultipleObjectsReturned: diff --git a/parsers/caves.py b/parsers/caves.py index 8b6f328..fe8c92c 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -443,6 +443,7 @@ def validate_station(station): if dot == -1: print(dot) # no full stop found. Bad station identifier. + # should just skip really, and log an error raise else: return True diff --git a/parsers/scans.py b/parsers/scans.py index d091a3d..378db66 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -24,6 +24,9 @@ def set_JSONwalletdate(wallet): we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here.""" _ = wallet.date() # don't need return value. Sets .walletdate as side effect +def set_caves(wallet): + _ = wallet.allcaves() # don't need return value. Just calling this saves it as w.caves This ONLY gets the list on the wallet JSON + def load_all_scans(): """This iterates through the scans directories (either here or on the remote server) and builds up the models we can access later. @@ -114,6 +117,7 @@ def load_all_scans(): wallet = Wallet(fpath=fpath, walletname=walletname) set_walletyear(wallet) wallet.save() + set_caves(wallet) wallets[walletname] = wallet singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet) @@ -159,6 +163,7 @@ def load_all_scans(): # BUT can't check linked survex blocks as they haven't been imported yet set_JSONwalletdate(wallet) set_walletyear(wallet) + set_caves(wallet) if not created: print( f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?" diff --git a/templates/personwallets.html b/templates/personwallets.html index 55a0e0e..4f5a403 100644 --- a/templates/personwallets.html +++ b/templates/personwallets.html @@ -23,7 +23,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c

Note that names in italics are copied from the related survex file block name.
- + {% for wallet in manywallets|dictsort:"walletname" %} @@ -37,15 +37,11 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c {%for p in wallet.persons%} {{p.fullname}}{%if not forloop.last %}, {% endif %} {% endfor %} - +
WalletWallet DateWallet NameTeam>CaveScansSurvex blocksDrawings using these scans
WalletWallet DateWallet NameTeamCave(s)ScansSurvex blocksDrawings using these scans
{{wallet.walletname}} - {% if wallet.cave %} - {% if wallet.caveobj.slug %} - {{wallet.cave}} - {% else %} - {{wallet.cave}} - {% endif %} - {% else %} - {% endif %} + {% for c in wallet.caves.all %} + {{c}} + {% endfor %} + {{wallet.singlescan_set.all|length}} diff --git a/templates/wallet_table.html b/templates/wallet_table.html index 7cf6f49..3f90152 100644 --- a/templates/wallet_table.html +++ b/templates/wallet_table.html @@ -1,6 +1,6 @@ - + @@ -22,15 +22,11 @@ - +
WalletWallet DateCaveWallet NameSurvex survey length
WalletWallet DateCave(s)Wallet NameSurvex survey length S{{wallet.walletname}} {% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %} - {% if wallet.cave %} - {% if wallet.caveobj.slug %} - {{wallet.cave}} - {% else %} - {{wallet.cave}} - {% endif %} - {% else %} - {% endif %} + {% for c in wallet.caves.all %} + {{c}} + {% endfor %} + {% if wallet.name %}{{wallet.name|truncatechars:20}}{% else %}{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}{% endif %} {% for survexblock in wallet.survexblock_set.all %} diff --git a/templates/walletform.html b/templates/walletform.html index 2b4cf8c..ef07665 100644 --- a/templates/walletform.html +++ b/templates/walletform.html @@ -149,12 +149,13 @@ and also the exported files in standard formats: svx, svg etc. See why
- {% if cave %}Cave ID: - {% if caveobject %}{{cave}} which implies "{{caveobject}}"
- {% else %} - {{cave}}
- {% endif %} - {% endif %} + {% for wallet in manywallets %} + Cave IDs: + + {% for c in wallet.caves.all %} + {{c}} + {% endfor %}
+ {% if psg %}Survey area: {{psg}}
{% endif %} {% if svxfiles %}Survex files: {% for svx in svxfiles %} @@ -163,7 +164,7 @@ and also the exported files in standard formats: svx, svg etc. See why
{% endif %} - + {% endfor %}
diff --git a/templates/yearwallets.html b/templates/yearwallets.html index e89431d..7385cc4 100644 --- a/templates/yearwallets.html +++ b/templates/yearwallets.html @@ -32,7 +32,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c

Total underground survey length: {{length_ug|floatformat:"1g"}} m
- + {% for wallet in manywallets|dictsort:"walletname" %} @@ -46,15 +46,12 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c {%for p in wallet.persons%} {{p.fullname}}{%if not forloop.last %}, {% endif %} {% endfor %} - + +
WalletWallet DateWallet NamePeopleCaveScansSurvex blocksDrawings using these scans
WalletWallet DateWallet NamePeopleCave(s)ScansSurvex blocksDrawings using these scans
{{wallet.walletname}} - {% if wallet.cave %} - {% if wallet.caveobj.slug %} - {{wallet.cave}} - {% else %} - {{wallet.cave}} - {% endif %} - {% else %} - {% endif %} + {% for c in wallet.caves.all %} + {{c}} + {% endfor %} + {{wallet.singlescan_set.all|length}}