diff --git a/parsers/survex.py b/parsers/survex.py index b1672b2..9837a83 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -137,6 +137,12 @@ def stash_data_issue(parser=None, message=None, url=None, sb=None): use a set, we do not want identically duplicate issues """ global dataissues + # try: + # if sb: + # url2 = get_offending_filename(sb.survexfile.path) + # except Exception as e: + # print(f" ! stash_data_issue() '{e}' '{sb=}' -- '{url=}'", file=sys.stderr) + # raise dataissues.add((parser, message, url, sb)) def store_data_issues(): @@ -153,10 +159,10 @@ def store_data_issues(): if url is None: if sb is not None: try: - url = get_offending_filename(sb.survexfile) + url = get_offending_filename(sb.survexfile.path) except Exception as e: - print(f" ! store_data_issues() {e} '{sb=}' -- '{url=}'", file=sys.stderr) - url = get_offending_filename(sb) + print(f" ! store_data_issues() '{e}' '{sb=}' -- '{url=}'", file=sys.stderr) + url = get_offending_filename(sb) # assumed to be text di_list.append(DataIssue(parser=parser, message=message, url=url)) # Now commit to db DataIssue.objects.bulk_create(di_list) @@ -679,7 +685,7 @@ class LoadingSurvex: # message = f" ! No survexblock.date inheritable in '{survexblock}' in '{survexblock.survexfile.path}', setting to 1976" # print(self.insp + message) # stash_data_issue( - # parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + # parser="survex", message=message, url=None, sb=survexblock # ) # expoyear = "1976" return @@ -759,13 +765,13 @@ class LoadingSurvex: elif known_foreigner(tm): # note, not using .lower() message = f"- *team {expo.year} '{tm}' known foreigner on *team {survexblock.survexfile.path} ({survexblock}) in '{line=}'" print(self.insp + message) - # stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path)) + # stash_data_issue(parser='survex', message=message, url=None, sb=survexblock) else: # we know the date and expo, but can't find the person message = f"! *team {expo.year} '{tm}' FAIL personexpedition lookup on *team {survexblock.survexfile.path} ({survexblock}) in '{line=}' {tm=}" print(self.insp + message) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) else: self.add_to_pending(survexblock, tm) @@ -790,7 +796,7 @@ class LoadingSurvex: message = f"! *team {survexblock.survexfile.path} ({survexblock}) role 'unknown' in wrong place '{line=}'" print(self.insp + message) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) record_team_member(line.replace("unknown ","").strip('\"'), survexblock) elif no_role_people := self.rx_teamone.match(line): @@ -804,7 +810,7 @@ class LoadingSurvex: message = f"! *team {survexblock.survexfile.path} ({survexblock}) Bad '{line=}'" print(self.insp + message) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) def LoadSurvexSet(self, survexblock, line): @@ -822,7 +828,7 @@ class LoadingSurvex: ) print(self.insp + message) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) def LoadSurvexFix(self, survexblock, line): @@ -916,12 +922,12 @@ class LoadingSurvex: if scale: message = f"✓ *UNITS SCALE '{line}' ({survexblock}) {survexblock.survexfile.path} {len(scale.groups())} {scale.groups()[0]=}" print(self.insp + message) - stash_data_issue(parser="survexunits", message=message, url=None, sb=(survexblock.survexfile.path)) + stash_data_issue(parser="survexunits", message=message, url=None, sb=survexblock) self.unitsfactor = float(scale.groups()[0]) else: message = f"! *UNITS SCALE FAIL '{line}' ({survexblock}) {survexblock.survexfile.path} " print(self.insp + message) - stash_data_issue(parser="survexunits", message=message, url=None, sb=(survexblock.survexfile.path)) + stash_data_issue(parser="survexunits", message=message, url=None, sb=survexblock) metres = re.match(r"(?i).*(METRIC|METRES|METERS)$", line) if metres: @@ -929,7 +935,7 @@ class LoadingSurvex: else: message = f"! *UNITS not meters - not converted '{line}' ({survexblock}) {survexblock.survexfile.path}" print(self.insp + message) - stash_data_issue(parser="survexunits", message=message, url=None, sb=(survexblock.survexfile.path)) + stash_data_issue(parser="survexunits", message=message, url=None, sb=survexblock) return return @@ -943,7 +949,7 @@ class LoadingSurvex: f"✓ *UNITS NUMERICAL conversion [{factor}x] '{line}' ({survexblock}) {survexblock.survexfile.path}" ) print(self.insp + message) - stash_data_issue(parser="survexunits", message=message) + stash_data_issue(parser="survexunits", message=message, sb=survexblock) feet = re.match(r"(?i).*feet$", line) metres = re.match(r"(?i).*(METRIC|METRES|METERS)$", line) @@ -954,7 +960,7 @@ class LoadingSurvex: else: message = f"! *UNITS in YARDS!? - not converted '{line}' ({survexblock}) {survexblock.survexfile.path}" print(self.insp + message) - stash_data_issue(parser="survexunits", message=message) + stash_data_issue(parser="survexunits", message=message, sb=survexblock) def get_expo_from_year(self, year, line, survexblock): # cacheing to save DB query on every block @@ -971,7 +977,7 @@ class LoadingSurvex: ) print(self.insp + message) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) if expeditions: expo = expeditions[0] @@ -980,7 +986,7 @@ class LoadingSurvex: expo = Expedition.objects.get(year="1976") message = f"! DATE INCORRECT. There is no expedition for the year {year}. {survexblock.survexfile.path} ({survexblock}) - set to 1976." print(self.insp + message) - stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path)) + stash_data_issue(parser='survex', message=message, url=None, sb=survexblock) return expo def LoadSurvexDate(self, survexblock, line): @@ -1015,14 +1021,14 @@ class LoadingSurvex: if len(team) > 0: message = f"! *team {expo.year} Multiple *date in one block? Already someone on team when *date seen. {survexblock.survexfile.path} ({survexblock}) in '{line}'" print(self.insp + message) - stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path)) + stash_data_issue(parser='survex', message=message, url=None, sb=survexblock) if teamnames := self.get_team_pending(survexblock._blockid): for tm in teamnames: if known_foreigner(tm): message = f"- *team {expo.year} '{tm}' known foreigner *date (misordered) {survexblock.survexfile.path} ({survexblock}) in '{line}'" print(self.insp + message) - # stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path)) + # stash_data_issue(parser='survex', message=message, url=None, sb=survexblock) else: pe = GetPersonExpeditionNameLookup(expo).get(tm.lower()) if pe: @@ -1034,7 +1040,7 @@ class LoadingSurvex: stash_data_issue( parser="survex", message=message, - url=None, sb=(survexblock.survexfile.path), + url=None, sb=survexblock, ) oline = line @@ -1046,7 +1052,7 @@ class LoadingSurvex: else: message = f"! DATE Warning LONG DATE '{oline}' ({survexblock}) {survexblock.survexfile.path}" print(self.insp+message) - stash_data_issue(parser='xSvxDate', message=message, url=None, sb=(survexblock.survexfile.path)) + stash_data_issue(parser='xSvxDate', message=message, url=None, sb=survexblock) if len(line) == 10: @@ -1059,7 +1065,7 @@ class LoadingSurvex: message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}" print(self.insp + message) stash_data_issue( - parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=survexblock ) survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m") # sets to first of month elif len(line) == 4: @@ -1067,7 +1073,7 @@ class LoadingSurvex: message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}" print(self.insp + message) stash_data_issue( - parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=survexblock ) survexblock.date = datetime.strptime(line, "%Y") # sets to January 1st elif len(line) == 9 or len(line) == 8: @@ -1075,7 +1081,7 @@ class LoadingSurvex: message = f"! DATE format WARNING, single digit day or month number,'{oline}' [{line[-5]}][{line[-2]}] ({survexblock}) {survexblock.survexfile.path}" print(self.insp + message) stash_data_issue( - parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=survexblock ) if line[-2] == "-" or line[-2] == ".": line = line[:-1] + '0' + line[-1] @@ -1092,7 +1098,7 @@ class LoadingSurvex: ) print(self.insp + message) stash_data_issue( - parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=survexblock ) else: # these errors are reporting the wrong survexblock, which is actually a SurvexFile (!) @@ -1102,7 +1108,7 @@ class LoadingSurvex: ) print(self.insp + message) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) print(f" {type(survexblock)=}") # survexblock.parent fails as a SurvexFile has no .parent ...ugh. print(f" {survexblock.survexfile=}") @@ -1148,7 +1154,7 @@ class LoadingSurvex: if self.datastar["type"] == "cartesian": # message = f" ! CARTESIAN data in {survexblock.survexfile.path} {self.unitsfactor=}." # stash_data_issue( - # parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + # parser="survexleg", message=message, url=None, sb=survexblock # ) ls = sline.lower().split() # CARTESIAN, so there should be 5 fields: from to dx dy dz @@ -1157,7 +1163,7 @@ class LoadingSurvex: print(f" Line: {sline}\nsvxline: {svxline}") message = f" ! Not 5 CARTESIAN fields in line '{sline.lower()}' {self.datastar=} {ls=} in\n{survexblock}\n{survexblock.survexfile}\n{survexblock.survexfile.path}" stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) return leglength = math.sqrt(float(ls[2])**2 + float(ls[3])**2 + float(ls[4])**2) @@ -1189,7 +1195,7 @@ class LoadingSurvex: print(f" Line: {sline}\nsvxline: {svxline}") message = f" ! Not 5 fields in line '{sline.lower()}' {self.datastar=} {ls=} in\n{survexblock}\n{survexblock.survexfile}\n{survexblock.survexfile.path}" stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) datastar = self.datastar # shallow copy: alias but the things inside are the same things @@ -1216,14 +1222,14 @@ class LoadingSurvex: message = f" ! Suspected splay, not declared, in line {ls} in {survexblock.survexfile.path}" print(self.insp + message) stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) return except: message = f" ! datastar parsing from/to incorrect in line {ls} in {survexblock.survexfile.path}" print(self.insp + message) stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) return @@ -1233,7 +1239,7 @@ class LoadingSurvex: message = f" ! datastar parsing incorrect in line {ls} in {survexblock.survexfile.path}" print(self.insp + message) stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) survexleg.tape = invalid_tape return @@ -1248,7 +1254,7 @@ class LoadingSurvex: message = f" ! Units: Length scaled {tape}m '{ls}' in ({survexblock.survexfile.path}) units:{self.units} factor:{self.unitsfactor}x" print(self.insp + message) stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) if self.units == "feet": tape = float(tape) / METRESINFEET @@ -1256,7 +1262,7 @@ class LoadingSurvex: message = f" ! Units: converted to {tape:.3f}m from {self.units} '{ls}' in ({survexblock.survexfile.path})" print(self.insp + message) stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) survexleg.tape = float(tape) self.legsnumber += 1 @@ -1264,7 +1270,7 @@ class LoadingSurvex: message = f" ! Value Error: Tape misread in line'{ls}' in {survexblock.survexfile.path} units:{self.units}" print(self.insp + message) stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) survexleg.tape = invalid_tape try: @@ -1276,7 +1282,7 @@ class LoadingSurvex: ) print(self.insp + message) stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) try: @@ -1285,7 +1291,7 @@ class LoadingSurvex: message = f" ! Value Error: Compass not found in line {ls} in {survexblock.survexfile.path}" print(self.insp + message) stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) lcompass = invalid_compass @@ -1297,7 +1303,7 @@ class LoadingSurvex: print((" Line:", ls)) message = f" ! Value Error: Clino misread in line '{sline.lower()}' {datastar=} {self.datastar=} {ls=} in\n{survexblock}\n{survexblock.survexfile}\n{survexblock.survexfile.path}" stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) lclino = invalid_clino @@ -1318,7 +1324,7 @@ class LoadingSurvex: print((" Line:", ls)) message = " ! Value Error: lcompass:'{}' line {} in '{}'".format(lcompass, ls, survexblock.survexfile.path) stash_data_issue( - parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survexleg", message=message, url=None, sb=survexblock ) survexleg.compass = invalid_compass @@ -1468,7 +1474,7 @@ class LoadingSurvex: print(message) print(message, file=sys.stderr) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) return else: @@ -1480,7 +1486,7 @@ class LoadingSurvex: print(message) print(message, file=sys.stderr) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) return False @@ -1508,7 +1514,7 @@ class LoadingSurvex: print(message) print(message, file=sys.stderr) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) self.datastar["type"] = ls[0] @@ -1631,7 +1637,7 @@ class LoadingSurvex: if not cavepath.startswith("fixedpts"): message = f" ! Warning: no cave identifiable for '{svxid}.svx' {cavepath=} " print("\n" + message) - stash_data_issue(parser="survex", message=message, url="{svxid}.svx", sb=(svxid)) + stash_data_issue(parser="survex", message=message, url="{svxid}.svx") return False def LoadSurvexFile(self, svxid): @@ -1733,13 +1739,13 @@ class LoadingSurvex: # message = f' ! QM TICK find FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" qmlist:"{qm}" in "{survexblock.survexfile.path}" + completion_description:"{qmtick.group(3)}" ' # print(message) # stash_data_issue( - # parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + # parser="survex", message=message, url=None, sb=survexblock # ) # if len(qm) > 1: # message = f' ! QM TICK MULTIPLE found FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" in "{survexblock.survexfile.path}" + completion_description:"{qmtick.group(3)}" ' # print(message) # stash_data_issue( - # parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + # parser="survex", message=message, url=None, sb=survexblock # ) # qm[0].ticked = True # # qm[0].ticked_date = qmtick.group(2) # not in data model yet @@ -1768,7 +1774,7 @@ class LoadingSurvex: message = f" ! QM{qm_no} INVALID code '{qm_grade}' [{blockname}] '{survexblock.survexfile.path}'" print(insp + message) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) qm_nearest = qmline.group(3) @@ -1840,7 +1846,7 @@ class LoadingSurvex: message = f" ! QM{qm_no} FAIL to create {qm_nearest} in'{survexblock.survexfile.path}' found {len(qms)}:{qms}" print(insp + message) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) @@ -1858,7 +1864,7 @@ class LoadingSurvex: message = f' ! QM Unrecognised QM number in "{survexblock.survexfile.path}" line:{comment}' print(message) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) return False qmline = self.rx_qm.match(comment) @@ -1868,7 +1874,7 @@ class LoadingSurvex: message = f' ! QM Unrecognised as valid in "{survexblock.survexfile.path}" QM{qml.group(1)=} "{qml.group(2)=}" : regex failure typo?' print(message) stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + parser="survex", message=message, url=None, sb=survexblock ) def LoadSurvexComment(self, survexblock, comment):