xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

making it work with 1626 no schonberg

Browse Source
This commit is contained in:
Philip Sargent
2022-07-23 19:26:47 +03:00
parent 7288bd9da3
commit a1e538e93a
6 changed files with 159494 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
parsers/survex.py
View File

@@ -86,6 +86,9 @@ class MapLocations(object):
def __str__(self):
return "{} map locations".format(len(self.p))
def get_offending_filename(path):
return "/survexfile/" + path + ".svx"
class SurvexLeg():
"""No longer a models.Model subclass, so no longer a database table
@@ -186,7 +189,7 @@ class LoadingSurvex():
def __init__(self):
self.caveslist = GetCaveLookup()
pass
def LoadSurvexFallThrough(self, survexblock, line, cmd):
if cmd == "require":
pass # should we check survex version available for processing?
@@ -195,13 +198,13 @@ class LoadingSurvex():
pass # we ignore all these, which is fine.
else:
if cmd in ["include", "data", "flags", "title", "entrance","set", "units", "alias", "ref"]:
message = "! Unparsed [*{}]: '{}' {} - not an error (probably)".format(cmd, line, survexblock.survexfile.path)
print((self.insp+message))
DataIssue.objects.create(parser='survex', message=message)
message = "! Warning. Unparsed [*{}]: '{}' {} - not an error (probably)".format(cmd, line, survexblock.survexfile.path)
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
else:
message = "! Bad svx command: [*{}] {} ({}) {}".format(cmd, line, survexblock, survexblock.survexfile.path)
print((self.insp+message))
DataIssue.objects.create(parser='survex', message=message)
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
def LoadSurvexTeam(self, survexblock, line):
"""Interpeting the *team fields means interpreting older style survex as well as current survex standard,
@@ -241,7 +244,7 @@ class LoadingSurvex():
self.flagsstar["splayalias"] = True
else:
message = "! Bad *ALIAS: '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
print((self.insp+message))
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexUnits(self, survexblock, line):
@@ -255,7 +258,7 @@ class LoadingSurvex():
self.unitsfactor = float(factor)
if debugprint:
message = "! *UNITS NUMERICAL conversion [{}x] '{}' ({}) {}".format(factor, line, survexblock, survexblock.survexfile.path)
print((self.insp+message))
print(self.insp+message)
DataIssue.objects.create(parser='survexunits', message=message)
feet = re.match("(?i).*feet$",line)
@@ -266,7 +269,7 @@ class LoadingSurvex():
self.units = "metres"
else:
message = "! *UNITS in YARDS!? - not converted '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
print((self.insp+message))
print(self.insp+message)
DataIssue.objects.create(parser='survexunits', message=message)
def LoadSurvexDate(self, survexblock, line):
@@ -282,7 +285,7 @@ class LoadingSurvex():
expeditions = findexpedition(year)
if len(expeditions) != 1 :
message = f"! More than one expedition in year {year} '{line}' ({survexblock}) {survexblock.survexfile.path}"
print((self.insp+message))
print(self.insp+message)
DataIssue.objects.create(parser='survexunits', message=message)
expo= expeditions[0]
@@ -311,8 +314,8 @@ class LoadingSurvex():
setdate(year)
else:
message = "! DATE unrecognised '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
print((self.insp+message))
DataIssue.objects.create(parser='survex', message=message)
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
def LoadSurvexLeg(self, survexblock, sline, comment):
"""This reads compass, clino and tape data but only keeps the tape lengths,
@@ -352,30 +355,36 @@ class LoadingSurvex():
ls = sline.lower().split()
# skip all splay legs
if ls[datastar["from"]] == ".." or ls[datastar["from"]] == ".":
if debugprint:
print("Splay in ", survexblock.survexfile.path)
return
if ls[datastar["to"]] == ".." or ls[datastar["to"]] == ".":
if debugprint:
print("Splay in ", survexblock.survexfile.path)
return
if self.flagsstar["splayalias"]:
if ls[datastar["from"]] == "-":
try:
if ls[datastar["from"]] == ".." or ls[datastar["from"]] == ".":
if debugprint:
print("Aliased splay in ", survexblock.survexfile.path)
print("Splay in ", survexblock.survexfile.path)
return
if ls[datastar["to"]] == "-":
if ls[datastar["to"]] == ".." or ls[datastar["to"]] == ".":
if debugprint:
print("Aliased splay in ", survexblock.survexfile.path)
print("Splay in ", survexblock.survexfile.path)
return
if self.flagsstar["splayalias"]:
if ls[datastar["from"]] == "-":
if debugprint:
print("Aliased splay in ", survexblock.survexfile.path)
return
if ls[datastar["to"]] == "-":
if debugprint:
print("Aliased splay in ", survexblock.survexfile.path)
return
except:
message = ' ! datastar parsing from/to incorrect in line %s in %s' % (ls, survexblock.survexfile.path)
print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
return
try:
tape = ls[datastar["tape"]]
except:
message = ' ! datastar parsing incorrect in line %s in %s' % (ls, survexblock.survexfile.path)
print((self.insp+message))
DataIssue.objects.create(parser='survexleg', message=message)
print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexleg.tape = invalid_tape
return
# e.g. '29/09' or '(06.05)' in the tape measurement
@@ -387,34 +396,34 @@ class LoadingSurvex():
tape = float(tape) * self.unitsfactor
if debugprint:
message = " ! Units: Length scaled {}m '{}' in ({}) units:{} factor:{}x".format(tape, ls, survexblock.survexfile.path, self.units, self.unitsfactor)
print((self.insp+message))
DataIssue.objects.create(parser='survexleg', message=message)
print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
if self.units =="feet":
tape = float(tape) / METRESINFEET
if debugprint:
message = " ! Units: converted to {:.3f}m from {} '{}' in ({})".format(tape, self.units, ls, survexblock.survexfile.path)
print((self.insp+message))
DataIssue.objects.create(parser='survexleg', message=message)
print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexleg.tape = float(tape)
self.legsnumber += 1
except ValueError:
message = " ! Value Error: Tape misread in line'{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units)
print((self.insp+message))
DataIssue.objects.create(parser='survexleg', message=message)
print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexleg.tape = invalid_tape
try:
survexblock.legslength += survexleg.tape
self.slength += survexleg.tape
except ValueError:
message = " ! Value Error: Tape length not added '{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units)
print((self.insp+message))
DataIssue.objects.create(parser='survexleg', message=message)
print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
try:
lcompass = ls[datastar["compass"]]
except:
message = ' ! Value Error: Compass not found in line %s in %s' % (ls, survexblock.survexfile.path)
print((self.insp+message))
print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message)
lcompass = invalid_compass
@@ -425,7 +434,7 @@ class LoadingSurvex():
print((" datastar:", datastar))
print((" Line:", ls))
message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path)
DataIssue.objects.create(parser='survexleg', message=message)
DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
lclino = invalid_clino
if lclino == "up":
@@ -445,7 +454,7 @@ class LoadingSurvex():
print((" Line:", ls))
message = " ! Value Error: lcompass:'{}' line {} in '{}'".format(lcompass,
ls, survexblock.survexfile.path)
DataIssue.objects.create(parser='survexleg', message=message)
DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexleg.compass = invalid_compass
# delete the object to save memory
@@ -453,17 +462,17 @@ class LoadingSurvex():
def LoadSurvexRef(self, survexblock, args):
#print(self.insp+ "*REF ---- '"+ args +"'")
url=f'/survexfile/{survexblock.survexfile.path}'
url= get_offending_filename(survexblock.survexfile.path)
# *REF but also ; Ref years from 1960 to 2039
refline = self.rx_ref_text.match(args)
if refline:
# a textual reference such as "1996-1999 Not-KH survey book pp 92-95"
print(self.insp+ "*REF quoted text so ignored:"+ args)
print(f'{self.insp} *REF quoted text so ignored:{args} in {survexblock.survexfile.path}')
return
if len(args)< 4:
message = " ! Empty or BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path)
print((self.insp+message))
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
return
@@ -484,18 +493,18 @@ class LoadingSurvex():
wallet = "0" + wallet
if not (int(yr)>1960 and int(yr)<2039):
message = " ! Wallet year out of bounds {yr} '{refscan}' {survexblock.survexfile.path}"
print((self.insp+message))
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
refscan = "%s#%s%s" % (yr, letterx, wallet)
try:
if int(wallet)>99:
message = " ! Wallet *REF {} - too big in '{}'".format(refscan, survexblock.survexfile.path)
print((self.insp+message))
message = " ! Wallet *REF {} - very big (more than 99) so probably wrong in '{}'".format(refscan, survexblock.survexfile.path)
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
except:
message = " ! Wallet *REF {} - not numeric in '{}' -- parsing continues".format(refscan, survexblock.survexfile.path)
print((self.insp+message))
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
manywallets = Wallet.objects.filter(walletname=refscan)
if manywallets:
@@ -504,11 +513,11 @@ class LoadingSurvex():
survexblock.save()
if len(manywallets) > 1:
message = " ! Wallet *REF {} - more than one found {} scan folders in {}".format(refscan, len(manywallets), survexblock.survexfile.path)
print((self.insp+message))
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
else:
message = " ! Wallet *REF '{}' - NOT found '{}'".format(refscan, survexblock.survexfile.path)
print((self.insp+message))
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
def TickSurvexQM(self, survexblock, qmtick):
@@ -520,11 +529,11 @@ class LoadingSurvex():
#raise
message = f' ! QM TICK find FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" qmlist:"{qm}" in "{survexblock.survexfile.path}" + comment:"{qmtick.group(3)}" '
print(message)
DataIssue.objects.create(parser='survex', message=message, url=f'/survexfile/{survexblock.survexfile.path}.svx')
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
if len(qm)>1:
message = f' ! QM TICK MULTIPLE found FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" in "{survexblock.survexfile.path}" + comment:"{qmtick.group(3)}" '
print(message)
DataIssue.objects.create(parser='survex', message=message, url=f'/survexfile/{survexblock.survexfile.path}.svx')
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
qm[0].ticked = True
qm[0].save()
@@ -578,14 +587,10 @@ class LoadingSurvex():
expoyear = str(survexblock.date.year),
cave = survexblock.survexfile.cave)
qm.save
if survexblock.survexfile.cave.kataster_number == "359":
message = " ! QM{} '{}' CREATED in DB in '{}'".format(qm_no, qm_nearest,survexblock.survexfile.path)
print(insp+message)
DataIssue.objects.create(parser='survex', message=message)
except:
message = " ! QM{} FAIL to create {} in'{}'".format(qm_no, qm_nearest,survexblock.survexfile.path)
print(insp+message)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
def LoadSurvexDataNormal(self,survexblock,args):
"""Sets the order for data elements in this and following blocks, e.g.
@@ -614,12 +619,19 @@ class LoadingSurvex():
message = " ! - Unrecognised *data normal statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath)
print(message)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
return
else:
datastar = self.datastardefault
# ls = ["normal", "from", "to", "tape", "compass", "clino" ]
for i in range(1, len(ls)): # len[0] is "normal"
if ls[i].lower() =="newline":
message = f" ! - ABORT *data statement has NEWLINE in it in {survexblock.survexfile.path}. Not parsed by torggle. '{args}'"
print(message)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
return False
if ls[i] in ["bearing","compass"]:
datastar["compass"] = i-1
if ls[i] in ["clino","gradient"]:
@@ -638,7 +650,7 @@ class LoadingSurvex():
message = " ! - Unrecognised *data statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath)
print(message)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
def LoadSurvexFlags(self, args):
# Valid flags are DUPLICATE, SPLAY, and SURFACE, and a flag may be preceded with NOT to turn it off.
@@ -709,7 +721,7 @@ class LoadingSurvex():
self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory
return self.svxdirs[headpath.lower()]
def ReportNonCaveIncludes(self, headpath, includelabel):
def ReportNonCaveIncludes(self, headpath, includelabel, depth):
"""Ignore surface, kataser and gpx *include survex files
"""
if headpath in self.ignorenoncave:
@@ -723,11 +735,11 @@ class LoadingSurvex():
#print("\n"+message)
#print("\n"+message,file=sys.stderr)
return
message = f" ! Error: '{headpath}' FAILURE (while creating '{includelabel}' in db - not a cave or in the ignore list of surface surveys. )"
message = f" ! Error: FAILURE '{headpath}' while creating '{includelabel}' at depth:'{depth}'. Not a cave or in the ignore list:'{self.ignoreprefix}'"
# getting this triggered for gpx/2018 (cavern error) but not for gpx/2017 (no content).
print("\n"+message)
print("\n"+message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(headpath))
print(f' # datastack in LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr)
for dict in self.datastack:
print(f'{dict["type"].upper()} ', end="",file=sys.stderr)
@@ -763,14 +775,14 @@ class LoadingSurvex():
message = " ! 'None' SurvexDirectory returned from GetSurvexDirectory({})".format(headpath)
print(message)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url = f'/survexfile/{svxid}')
if cave:
newdirectory.cave = cave
newfile.cave = cave
# print(f"\n - New directory {newdirectory} for cave {newdirectory.cave}",file=sys.stderr)
else: # probably a surface survey
self.ReportNonCaveIncludes(headpath, svxid)
else: # probably a surface survey, or a cave in a new area e.g. 1624 not previously managed, and not in the pending list
self.ReportNonCaveIncludes(headpath, svxid, depth)
if not newfile.survexdirectory:
message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(svxid)
@@ -845,7 +857,7 @@ class LoadingSurvex():
else:
message = f' ! QM Unrecognised as valid in "{survexblock.survexfile.path}" QM{qml.group(1)} "{qml.group(2)}" : regex failure, typo?'
print(message)
DataIssue.objects.create(parser='survex', message=message, url=f'/survexfile/{survexblock.survexfile.path}.svx')
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
included = self.rx_comminc.match(comment)
@@ -1068,7 +1080,11 @@ class LoadingSurvex():
print(" # CHANGE 'any' flag now:'{}' was:{} ".format(self.flagsstar["skiplegs"], oldflags["skiplegs"]))
elif self.rx_data.match(cmd):
self.LoadSurvexDataNormal(survexblock, args)
if self.LoadSurvexDataNormal(survexblock, args):
pass
else:
# Abort, we do not cope with this *data format
return
elif self.rx_alias.match(cmd):
self.LoadSurvexAlias(survexblock, args)
elif self.rx_entrance.match(cmd):
@@ -1085,7 +1101,7 @@ class LoadingSurvex():
message = " ! -ERROR *include command not expected here {}. Re-run a full Survex import.".format(path)
print(message)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, )
else:
self.LoadSurvexFallThrough(survexblock, args, cmd)
@@ -1120,7 +1136,7 @@ class LoadingSurvex():
self.legsnumber = nlegstotal
self.slength = slengthtotal
def PushdownStackScan(self, survexblock, path, fin, flinear, fcollate):
"""Follows the *include links in all the survex files from the root file 1623.svx
and reads only the *include and *begin and *end statements. It produces a linearised
@@ -1130,23 +1146,24 @@ class LoadingSurvex():
indent = " " * self.depthinclude
sys.stderr.flush();
self.callcount +=1
if self.callcount % 10 ==0 :
print(".", file=sys.stderr,end='')
if self.callcount % 500 ==0 :
print("\n ", file=sys.stderr,end='')
if path in self.svxfileslist:
message = " * Warning. Duplicate detected in *include list at callcount:{} depth:{} file:{}".format(self.callcount, self.depthinclude, path)
message = f" * Warning. Duplicate detected. We have already seen this *include '{path}' from another survex file. Detected at callcount:{self.callcount} depth:{self.depthinclude}"
print(message)
print(message,file=flinear)
print("\n"+message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
if self.svxfileslist.count(path) > 20:
message = " ! ERROR. Survex file already seen 20x. Probably an infinite loop so fix your *include statements that include this. Aborting. {}".format(path)
message = " ! ERROR. Survex file already *included 20x. Probably an infinite loop so fix your *include statements that include this. Aborting. {}".format(path)
print(message)
print(message,file=flinear)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
return
self.svxfileslist.append(path)
@@ -1157,8 +1174,7 @@ class LoadingSurvex():
message = f" ! ERROR *include file '{path}' in '{survexblock}' has UnicodeDecodeError"
print(message)
print(message,file=sys.stderr)
offendingfile = "/survexfile/" + path + ".svx"
DataIssue.objects.create(parser='survex', message=message, url=offendingfile)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
return # skip this survex file and all things *included in it
@@ -1173,7 +1189,7 @@ class LoadingSurvex():
message = message + f" - line {self.lineno} {survexblock}\n - Parsing aborted. NERD++ needed to fix it"
print(message)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
return # skip this survex file and all things *included in it
includestmt =self.rx_include.match(svxline)
@@ -1208,7 +1224,7 @@ class LoadingSurvex():
print(message)
print(message,file=flinear)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop))
fcollate.write(";*edulcni {}\n".format(pop))
fininclude.close()
@@ -1218,7 +1234,7 @@ class LoadingSurvex():
message = " ! ERROR *include file not found for:'{}'".format(includepath)
print(message)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
elif re.match("(?i)begin$", cmd):
self.depthbegin += 1
depth = " " * self.depthbegin
@@ -1240,7 +1256,7 @@ class LoadingSurvex():
print(message)
print(message,file=flinear)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
self.depthbegin -= 1
pass
@@ -1302,6 +1318,12 @@ class LoadingSurvex():
logpath = Path(fullpath + ".log")
outputdir = Path(svxpath).parent
if not svxpath.is_file():
message = f' ! BAD survex file "{fullpath}" specified in *include (somewhere).. '
DataIssue.objects.create(parser='entrances', message=message)
print(message)
return
if not logpath.is_file(): # always run if logfile not there
runcavern()
return