diff --git a/core/models/wallets.py b/core/models/wallets.py index 3f59da0..1cc42e8 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -10,6 +10,7 @@ from django.conf import settings from django.db import models from django.urls import reverse +from troggle.core.models.troggle import DataIssue from troggle.core.views.caves import get_cave_leniently # from troggle.core.models.survex import SurvexBlock @@ -143,8 +144,26 @@ class Wallet(models.Model): message = f"! {str(self.walletname)} Date format not ISO {waldata['date']}. Failed to load from {jsonfile} JSON file" from troggle.core.models.troggle import DataIssue DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) + + return waldata - + + def check_survexlist(self): + wurl = f"/walletedit/{self.walletname}".replace('#', ':') + if not (waldata := self.get_json()): # WALRUS + return None + if waldata["survex file"]: + if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way + waldata["survex file"] = [waldata["survex file"]] + for sx in waldata["survex file"]: + # this logic appears in several places, inc get_ticks(). and wallets_edit.py Refactor. + if sx != "": + if Path(sx).suffix.lower() != ".svx": + sx = sx + ".svx" + if not (Path(settings.SURVEX_DATA) / sx).is_file(): + message=f"{self} Survex file {sx} was not found in LOSER repo" + DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) + def allcaves(self): """Reads all the JSON data just to get the JSON date.""" if not (jsondata := self.get_json()): # WALRUS @@ -209,23 +228,19 @@ class Wallet(models.Model): self.save() return self.walletdate - # for gods sake redo this, it parse JSON twice every time.. def people(self): - if not self.get_json(): + if not (jsondata := self.get_json()): # WALRUS return None - jsondata = self.get_json() return jsondata["people"] def cave(self): - if not self.get_json(): + if not (jsondata := self.get_json()): # WALRUS return None - jsondata = self.get_json() return jsondata["cave"] def name(self): - if not self.get_json(): + if not (jsondata := self.get_json()): # WALRUS return None - jsondata = self.get_json() return jsondata["name"] def get_fnames(self): diff --git a/core/views/caves.py b/core/views/caves.py index bc5d6d7..c845f01 100644 --- a/core/views/caves.py +++ b/core/views/caves.py @@ -405,6 +405,7 @@ def edit_cave(request, path="", slug=None): if form.is_valid(): # and ceFormSet.is_valid(): # print(f'! POST is valid. {cave}') cave = form.save(commit=False) + print(cave) if not cave.filename: cave.filename = cave.areacode + "-" + cave.number() + ".html" if not cave.url: diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py index c99201b..541048d 100644 --- a/core/views/wallets_edit.py +++ b/core/views/wallets_edit.py @@ -156,7 +156,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): message = f"! {file_complaint}" print(message) DataIssue.objects.update_or_create( - parser="scans", message=message, url=wurl + parser="wallets", message=message, url=wurl ) # set URL to this wallet folder else: try: @@ -165,6 +165,10 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): except MultipleObjectsReturned: # can happen if connecting a wallet to a survex file.. i think.. QSsvxfiles = SurvexFile.objects.filter(path=sxpath) + message = f"! {wallet} Urk, multiple SurvexFile objects {sxpath}" + DataIssue.objects.update_or_create( + parser="wallets", message=message, url=wurl + ) # set URL to this wallet folder for s in QSsvxfiles: print(s.path, s.cave, s.primary) # QSsvxfiles[0] # dont' know how this happened, fix later.. @@ -176,7 +180,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): message = f"! {file_complaint}" print(message) DataIssue.objects.update_or_create( - parser="scans", message=message, url=wurl + parser="wallets", message=message, url=wurl ) # set URL to this wallet folder if waldata["survex not required"] and waldata["survex file"] != [""]: diff --git a/parsers/scans.py b/parsers/scans.py index 378db66..96c490a 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -17,7 +17,7 @@ git = settings.GIT def set_walletyear(wallet): - _ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear + _ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear Syntactic. def set_JSONwalletdate(wallet): """At this point in the import process, the survex files have not been imported so @@ -30,12 +30,9 @@ def set_caves(wallet): def load_all_scans(): """This iterates through the scans directories (either here or on the remote server) and builds up the models we can access later. - - It does NOT read or validate anything in the JSON data attached to each wallet. Those checks - are done at runtime, when a wallet is accessed, not at import time. - Loads people as a simple string of fullnames. We should replace this with a list of Person slugs. - + Loads people as a simple string of fullnames. We should replace this with a list of Person slugs, + and change the wallet editor to save People as slugs. """ print(" - Loading Survey Scans") @@ -115,11 +112,11 @@ def load_all_scans(): print("", flush=True, end="") # Create the wallet object. But we don't have a date for it yet. wallet = Wallet(fpath=fpath, walletname=walletname) + wallets[walletname] = wallet set_walletyear(wallet) wallet.save() set_caves(wallet) - wallets[walletname] = wallet - + singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet) singlescan.save() @@ -160,7 +157,6 @@ def load_all_scans(): wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath) wallets[walletname] = wallet # Now also load the json - # BUT can't check linked survex blocks as they haven't been imported yet set_JSONwalletdate(wallet) set_walletyear(wallet) set_caves(wallet) @@ -170,8 +166,17 @@ def load_all_scans(): ) wallet.save() print(f"\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets") - + # Only the 1999 wallets have filenames which mean that the walletyear will be unset: wallets = Wallet.objects.filter(walletyear=None) for w in wallets: - w.walletyear = datetime.date(1999, 1, 1) + w.walletyear = datetime.date(1999, 1, 1) + + # Although the survex files haven't been processed yet, we can at least check if the wallets refer to a real file or not + for wallet in Wallet.objects.all(): + # this reads JSON + wallet.check_survexlist() + + + + diff --git a/templates/manywallets.html b/templates/manywallets.html index 88d30d2..a7ed11c 100644 --- a/templates/manywallets.html +++ b/templates/manywallets.html @@ -27,28 +27,37 @@ otherwise they come from *ref statements in survex files as of the most recent d + - -{% for scanswallet in manywallets %} + +{% for wallet in manywallets %} - - + + + + + + {% endfor %}
Scans folderFilesSurvex blocksCave
Scans folderFilesSurvex blocksCave (new)Cave (old)
{{scanswallet.walletname}}{{scanswallet.singlescan_set.all|length}}{{wallet.walletname}}{{wallet.singlescan_set.all|length}} - {% for survexblock in scanswallet.survexblock_set.all %} + {% for survexblock in wallet.survexblock_set.all %} {{survexblock}} {% endfor %} + {% for c in wallet.caves.all %} + {{c}} + {% endfor %} + - {% for survexblock in scanswallet.survexblock_set.all %} + {% for survexblock in wallet.survexblock_set.all %} {% ifchanged survexblock.survexfile.cave %} {{survexblock.survexfile.cave.slug}} {% endifchanged %} {% empty %} - {% if scanswallet.cave %} - {{scanswallet.cave}} + {% if wallet.cave %} + {{wallet.cave}} {% endif %} {% endfor %}