From 29c5c82337bd249dffcc40e631a181c0af4a79f6 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Wed, 5 Oct 2022 23:18:11 +0300 Subject: [PATCH] polishing outputs for importing unseen survex files --- parsers/survex.py | 50 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/parsers/survex.py b/parsers/survex.py index 2740213..d2fdd8b 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -338,11 +338,16 @@ class LoadingSurvex(): survexblock.date = datetime.strptime(line, '%Y') # sets to January 1st setdate(year) else: + # these errors are reporting the wrong survexblock, which is actually a SurvexFile (!) message = "! DATE Error unrecognised '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path) print(self.insp+message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path)) + print(f" {survexblock.parent=}") # fails as SUrvexFile has no .parent ...ugh. + print(f" {survexblock.survexpath=}") + print(f" {survexblock.survexfile=}") + #raise - def LoadSurvexLeg(self, survexblock, sline, comment): + def LoadSurvexLeg(self, survexblock, sline, comment, svxline): """This reads compass, clino and tape data but only keeps the tape lengths, the rest is discarded after error-checking. Now skipping the error checking - returns as soon as the leg is not one we count. @@ -376,11 +381,20 @@ class LoadingSurvex(): if self.datastar["type"] != "normal": return + ls = sline.lower().split() + # NORMAL, so there should be 5 fields + # from the content, this is clearly reading fixedpts/gps/gps00raw.svx, but not reporting it by that name + if len(ls) < 5: + print("! Fewer than 5 fields in NORMAL in ", survexblock.survexfile.path, survexfile, survexfile.parent) + print(" datastar NORMAL:", self.datastar) + print(f" Line (split): {ls}, comment: {comment}") + print(f" Line: {sline}\nsvxline: {svxline}") + message = f' ! Not 5 fields in line \'{sline.lower()}\' {self.datastar=} {ls=} in\n{survexblock}\n{survexblock.survexfile}\n{survexblock.survexfile.path}' + DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path)) + datastar = self.datastar # shallow copy: alias but the things inside are the same things survexleg = SurvexLeg() - ls = sline.lower().split() - # skip all splay legs try: if ls[datastar["from"]] == ".." or ls[datastar["from"]] == ".": @@ -789,14 +803,18 @@ class LoadingSurvex(): # Yes we didn't find this cave, but we know it is a pending one. So not an error. # print(f'! ALREADY PENDING {caveid}',file=sys.stderr) return + id = caveid[5:] + if id in self.pending: + print(f'! ALREADY PENDING {id}',file=sys.stderr) + return message = f" ! Warning: cave identifier '{caveid}' (guessed from file path) is not a known cave. Need to add to expoweb/cave_data/pending.txt ? In '{includelabel}.svx' at depth:[{len(depth)}]." print("\n"+message) print("\n"+message,file=sys.stderr) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(includelabel)) - print(f' # datastack in LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr) - for dict in self.datastack: - print(f'<{dict["type"].upper()} >', end="",file=sys.stderr) + # print(f' # datastack in LoadSurvexFile:{includelabel}', file=sys.stderr) + # for dict in self.datastack: + # print(f' type: <{dict["type"].upper()} >', file=sys.stderr) def LoadSurvexFile(self, svxid): @@ -973,11 +991,12 @@ class LoadingSurvex(): nonlocal blockcount blockcount +=1 - if blockcount % 10 ==0 : + if blockcount % 20 ==0 : print(".", file=sys.stderr,end='') - if blockcount % 200 ==0 : + if blockcount % 400 ==0 : print("\n", file=sys.stderr,end='') - print(" - MEM:{:7.3f} MB in use".format(get_process_memory()),file=sys.stderr) + mem=get_process_memory() + print(" - MEM:{:7.3f} MB in use".format(mem),file=sys.stderr) print(" ", file=sys.stderr,end='') sys.stderr.flush() @@ -1190,7 +1209,7 @@ class LoadingSurvex(): # yes we are reading a *command starstatement(star) else: # not a *cmd so we are reading data OR a ";" rx_comment failed. We hope. - self.LoadSurvexLeg(survexblock, sline, comment) + self.LoadSurvexLeg(survexblock, sline, comment, svxline) self.legsnumber = nlegstotal self.slength = slengthtotal @@ -1480,7 +1499,7 @@ def FindAndLoadSurvex(survexblockroot): flinear.write(" - MEM:{:.3f} MB ADDITIONALLY USED\n".format(mem1-mem0)) flinear.write(" - {:,} survex files in linear include list \n".format(len(svx_scan.svxfileslist))) - print(" - {:,} runs of survex 'cavern' refreshing .3d files".format(svx_scan.caverncount),file=sys.stderr) + print("\n - {:,} runs of survex 'cavern' refreshing .3d files".format(svx_scan.caverncount),file=sys.stderr) print(" - {:,} survex files from tree in linear include list".format(len(svx_scan.svxfileslist)),file=sys.stderr) mem1 = get_process_memory() @@ -1512,9 +1531,14 @@ def FindAndLoadSurvex(survexblockroot): for o in excpts: if str(x).strip().startswith(o): removals.append(x) + # special fix for file not actually in survex format + unseens.remove(Path("fixedpts/gps/gps00raw")) + for x in removals: unseens.remove(x) - print(f" - {len(unseens)} survex files found which were not included in main tree. ({len(svx_scan.svxfileslist)} in main tree)", file=sys.stderr) + # for x in unseens: + # print(f"'{x}', ", end='', file=sys.stderr) + print(f"\n - {len(unseens)} survex files found which were not included in main tree. ({len(svx_scan.svxfileslist)} in main tree)", file=sys.stderr) print(f" -- Now loading the previously-omitted survex files.", file=sys.stderr) with open(Path(settings.SURVEX_DATA, '_unseens.svx'), 'w') as u: @@ -1608,7 +1632,7 @@ def FindAndLoadSurvex(survexblockroot): # ps = pstats.Stats(pr2, stream=f) # ps.sort_stats(SortKey.CUMULATIVE) # ps.print_stats() - + mem1 = get_process_memory() print("\n - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr) print(" - MEM:{:7.3f} MB ADDITIONALLY USED".format(mem1-mem0),file=sys.stderr)