From 87d9804864e157b451c92e6fb71165aa4925cd47 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Thu, 26 Oct 2023 18:28:59 +0300 Subject: [PATCH] findng variant data on survexblocks and setting wallet *ref --- core/models/survex.py | 2 +- core/models/wallets.py | 2 +- databaseReset.py | 6 +++- parsers/imports.py | 7 ++++- parsers/survex.py | 60 ++++++++++++++++++++++++++------------ templates/yearwallets.html | 1 + 6 files changed, 56 insertions(+), 22 deletions(-) diff --git a/core/models/survex.py b/core/models/survex.py index 6e2a06f..da35336 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -161,7 +161,7 @@ class SurvexBlock(models.Model): date = models.DateField(blank=True, null=True) expedition = models.ForeignKey("Expedition", blank=True, null=True, on_delete=models.SET_NULL) - # if the survexfile object is deleted, then all the suvex-blocks in it should be too, + # if the survexfile object is deleted, then all the survex-blocks in it should be too, # though a block can span more than one file... survexfile = models.ForeignKey("SurvexFile", blank=True, null=True, on_delete=models.CASCADE) # survexpath = models.CharField(max_length=200, blank=True, null=True) No need for this anymore diff --git a/core/models/wallets.py b/core/models/wallets.py index 2de8f9d..3078e2d 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -283,7 +283,7 @@ class Wallet(models.Model): return None filelist = Wallet.input_to_list(jsondata["survex file"]) - #print(f"'{self} {jsondata['survex file']}' => {filelist}") + # print(f"'{self} {jsondata['survex file']}' => {filelist}") return filelist def get_fnames(self): diff --git a/databaseReset.py b/databaseReset.py index 1be77c9..ada1c62 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -53,6 +53,7 @@ from troggle.parsers.imports import ( import_people, import_QMs, import_survex, + import_survex_checks, import_surveyscans, ) @@ -395,7 +396,8 @@ def usage(): scans - the survey scans in all the wallets (must run before survex) drawings - read in the Tunnel & Therion files - which scans the survey scans too survex - read in the survex files - all the survex blocks and entrances x/y/z - ents - read just the entrances x/y/z (must run after survex) + survex_ck - set caves and people on wallets, check wallets for *ref + ents - read just the entrances x/y/z (must run after survex) dumplogbooks - Not used. write out autologbooks (not working? use http://localhost:8000/controlpanel ) logbook - read a single logbook. Defautl set in python code @@ -442,6 +444,8 @@ if __name__ == "__main__": jq.enq("reinit", reinit_db) elif "ents" in sys.argv: jq.enq("survex", import_ents) + elif "survex_ck" in sys.argv: + jq.enq("survex", import_survex_checks) elif "test2" in sys.argv: jq.enq("QMs", import_QMs) jq.enq("drawings", import_drawingsfiles) diff --git a/parsers/imports.py b/parsers/imports.py index a469986..8b1df5f 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -54,12 +54,17 @@ def import_survex(): print(" - Survex Blocks") with transaction.atomic(): troggle.parsers.survex.LoadSurvexBlocks() - print(" - Survex entrances x/y/z Positions") with transaction.atomic(): troggle.parsers.survex.survexifywallets() + print(" - Survex entrances x/y/z Positions") with transaction.atomic(): troggle.parsers.locations.LoadPositions() +def import_survex_checks(): + print(" - Survex: check wallet references and set persons, caves") + with transaction.atomic(): + troggle.parsers.survex.survexifywallets() + def import_ents(): print(" - Survex entrances x/y/z Positions") with transaction.atomic(): diff --git a/parsers/survex.py b/parsers/survex.py index e668853..70eec8a 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -2484,19 +2484,41 @@ def MakeFileRoot(svxpath): return fileroot def set_survexblocks(wallet): - if svxfiles := wallet.survexfiles(): # reads from JSON, should be cached + """Need to find the optimal Django way of doing this query. + It's a mess now""" + if svxfiles := wallet.survexfiles(): # reads from JSON, should be cached already for svx in svxfiles: - # svx is a string, need to resolve to a survexfile object - #o = SurvexFile.objects.get(path=svx) - blocks = SurvexBlock.objects.filter(survexfile__path=svx) - for b in blocks: - if b.scanswallet == wallet: - pass - else: - b.scanswallet = wallet - b.save() - # print(f"setting {wallet} on {b.survexfile} : {b}") - + if svx: + if svx.endswith(".svx"): + svx = svx.replace(".svx","") + try: + # there are survex files we ignore when troggle parses, and some of these are referred to in wallets + sfile = SurvexFile.objects.get(path=svx) #.select_related("survexblocks") + # print(sfile) + except: + continue + blocks = SurvexBlock.objects.filter(survexfile=sfile) + for b in blocks: + try: + if b.scanswallet == wallet: + pass + elif b.scanswallet: + if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1): + print(f"not set{wallet} on {b.survexfile} : {b} as already set to {b.scanswallet}") + else: + b.scanswallet = wallet + b.save() + if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1): + print(f"setting {wallet} on {b.survexfile} : {b}") + except: + if not hasattr(b,"date"): + print(f" Block {b} on {b.survexfile} HAS NO DATE SET ") + elif not b.date: + print(f" Block {b} on {b.survexfile} HAS NULL DATE ") + else: + print(f" exception {wallet} on {b.survexfile} : {b}") + + def survexifywallets(): """Gets the caves from the list of survexblocks @@ -2514,7 +2536,7 @@ def survexifywallets(): w.persons.add(spr.person) duration = time.time() - start - print(f" - TIME: add people to wallets {duration:7.2f} s", file=sys.stderr) + print(f" - {duration:7.2f} s to add people to wallets ", file=sys.stderr) start = time.time() wallets = Wallet.objects.all() @@ -2522,7 +2544,7 @@ def survexifywallets(): set_survexblocks(w) # reads JSON, sets survexblocks if survexfiles specified on wallet JSON duration = time.time() - start - print(f" - TIME: set survexblock:wallet using JSON survexfiles {duration:7.2f} s", file=sys.stderr) + print(f" - {duration:7.2f} s to set survexblock:wallet using JSON survexfiles ", file=sys.stderr) start = time.time() for w in wallets: @@ -2533,15 +2555,16 @@ def survexifywallets(): w.save() duration = time.time() - start - print(f" - TIME: add caves to wallets {duration:7.2f} s", file=sys.stderr) + print(f" - {duration:7.2f} s to add caves to wallets ", file=sys.stderr) start = time.time() - + # Find the survex blocks which are 'ours' i.e. ignore all those (ARGE etc) without expo people attached. cuccblocks = set() for spr in SurvexPersonRole.objects.all(): cuccblocks.add(spr.survexblock) + # Because we have just run set_survexblocks(w), this should only complain if there is no *ref and no wallet that links to its parent file sentinelbad = Wallet.objects.get(walletname="1983#00") for b in cuccblocks: if b.date > date(2001, 1, 1): # do we care about older ones? 1999 certainly has different wallet system @@ -2551,12 +2574,13 @@ def survexifywallets(): b.scanswallet = b.parent.scanswallet continue message = f" ! *REF missing {b.date} {b.survexfile}.svx : '{b}'" - # print(message, file=sys.stderr) + if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1): + print(message, file=sys.stderr) url = get_offending_filename(b.survexfile.path) DataIssue.objects.update_or_create(parser="ref", message=message, url=url) duration = time.time() - start - print(f" - TIME: check missing *ref on survexblocks {duration:7.2f} s", file=sys.stderr) + print(f" - {duration:7.2f} s to check missing *ref on survexblocks ", file=sys.stderr) start = time.time() diff --git a/templates/yearwallets.html b/templates/yearwallets.html index 55b3289..5a945d5 100644 --- a/templates/yearwallets.html +++ b/templates/yearwallets.html @@ -8,6 +8,7 @@ plans and elevations. It also contains scans of centre-line survex output on which hand-drawn passage sections are drawn. These hand-drawn passages will eventually be traced to produce Tunnel or Therion drawings and eventually the final complete cave survey. +

Link to logbooks and calendar for {{year}}. {% include 'wallet_new.html' %}