expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-22 15:21:52 +00:00
Code Issues 19 Projects 3 Activity

cleanup and better DatIssue msgs

Browse Source
This commit is contained in:
Philip Sargent 2020-06-13 01:24:46 +01:00
parent 269b8840ad
commit 44fe6a8b89
1 changed files with 43 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
parsers/survex.py
View File

@ -20,7 +20,8 @@ from troggle.core.views_caves import MapLocations
A 'survexscansfolder' is what we today call a "survey scans folder" or a "wallet". A 'survexscansfolder' is what we today call a "survey scans folder" or a "wallet".
""" """
line_leg_regex = re.compile(r"[\d\-+.]+$") rx_braskets= re.compile(r"[()]")
rx_line_length = re.compile(r"[\d\-+.]+$")
survexlegsalllength = 0.0 survexlegsalllength = 0.0
survexlegsnumber = 0 survexlegsnumber = 0
@ -36,8 +37,10 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
# this next fails for two surface survey svx files which use / for decimal point # this next fails for two surface survey svx files which use / for decimal point
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05) # e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
if stardata["type"] == "normal": if stardata["type"] == "normal":
tape = rx_braskets.sub("",ls[stardata["tape"]])
tape = tape.replace("/",".")
try: try:
survexleg.tape = float(ls[stardata["tape"]]) survexleg.tape = float(tape)
survexlegsnumber += 1 survexlegsnumber += 1
except ValueError: except ValueError:
print(("! Tape misread in", survexblock.survexfile.path)) print(("! Tape misread in", survexblock.survexfile.path))
@ -82,8 +85,8 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
survexleg.compass = 1000 survexleg.compass = 1000
survexleg.clino = -90.0 survexleg.clino = -90.0
else: else:
assert line_leg_regex.match(lcompass), ls assert rx_line_length.match(lcompass), ls
assert line_leg_regex.match(lclino) and lclino != "-", ls assert rx_line_length.match(lclino) and lclino != "-", ls
survexleg.compass = float(lcompass) survexleg.compass = float(lcompass)
survexleg.clino = float(lclino) survexleg.clino = float(lclino)
@ -119,15 +122,17 @@ def LoadSurvexLinePassage(survexblock, stardata, sline, comment):
# do not import *data passage.. data which is LRUD not tape/compass/clino # do not import *data passage.. data which is LRUD not tape/compass/clino
pass pass
# This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g.
# *DATA normal from to length gradient bearing ignore ignore ignore ignore
stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4} stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"} stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"}
regex_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$") rx_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$")
regex_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(X)?\s*(\d+)') rx_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(X)?\s*(\d+)')
regex_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$') rx_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$')
# years from 1960 to 2039 # years from 1960 to 2039
regex_starref = re.compile(r'^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$(?i)') rx_starref = re.compile(r'^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$(?i)')
# regex_starref = re.compile("""?x # VERBOSE mode - can't get this to work # rx_starref = re.compile("""?x # VERBOSE mode - can't get this to work
# ^\s*\*ref # look for *ref at start of line # ^\s*\*ref # look for *ref at start of line
# [\s.:]* # some spaces, stops or colons # [\s.:]* # some spaces, stops or colons
# ((?:19[6789]\d)|(?:20[0123]\d)) # a date from 1960 to 2039 - captured as one field # ((?:19[6789]\d)|(?:20[0123]\d)) # a date from 1960 to 2039 - captured as one field
@ -136,10 +141,9 @@ regex_starref = re.compile(r'^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#
# ?\s*(.*?\d+.*?) # maybe a space, then at least one digit in the string - captured # ?\s*(.*?\d+.*?) # maybe a space, then at least one digit in the string - captured
# $(?i)""", re.X) # the end (do the whole thing case insensitively) # $(?i)""", re.X) # the end (do the whole thing case insensitively)
rx_team = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)")
regex_team = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)") rx_team_member = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)")
regex_team_member = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)") rx_qm = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')
regex_qm = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')
insp = "" insp = ""
callcount = 0 callcount = 0
@ -157,7 +161,6 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
global callcount global callcount
global survexlegsnumber global survexlegsnumber
# uncomment to print out all files during parsing
print(insp+" - Reading file: " + survexblock.survexfile.path + " <> " + survexfile.path) print(insp+" - Reading file: " + survexblock.survexfile.path + " <> " + survexfile.path)
stamp = datetime.now() stamp = datetime.now()
lineno = 0 lineno = 0
@ -172,33 +175,17 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", survexblock.survexfile.path) path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", survexblock.survexfile.path)
if path_match: if path_match:
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2)) pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
# print(insp+'Match')
# print(insp+os_cave)
cave = models_caves.getCaveByReference(pos_cave) cave = models_caves.getCaveByReference(pos_cave)
if cave: if cave:
survexfile.cave = cave survexfile.cave = cave
svxlines = '' svxlines = ''
svxlines = fin.read().splitlines() svxlines = fin.read().splitlines()
# print(insp+'Cave - preloop ' + str(survexfile.cave))
# print(insp+survexblock)
for svxline in svxlines: for svxline in svxlines:
# print(insp+survexblock)
# print(insp+svxline)
# if not svxline:
# print(insp+' - Not survex')
# return
# textlines.append(svxline)
lineno += 1 lineno += 1
# print(insp+' - Line: %d' % lineno)
# break the line at the comment # break the line at the comment
sline, comment = regex_comment.match(svxline.strip()).groups() sline, comment = rx_comment.match(svxline.strip()).groups()
# detect ref line pointing to the scans directory # detect ref line pointing to the scans directory
mref = comment and regex_ref.match(comment) mref = comment and rx_ref.match(comment)
if mref: if mref:
yr, letterx, wallet = mref.groups() yr, letterx, wallet = mref.groups()
if not letterx: if not letterx:
@ -208,28 +195,23 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
if len(wallet)<2: if len(wallet)<2:
wallet = "0" + wallet wallet = "0" + wallet
refscan = "%s#%s%s" % (yr, letterx, wallet ) refscan = "%s#%s%s" % (yr, letterx, wallet )
#print(insp+' - Wallet ;ref - %s - looking for survexscansfolder' % refscan)
survexscansfolders = models_survex.SurvexScansFolder.objects.filter(walletname=refscan) survexscansfolders = models_survex.SurvexScansFolder.objects.filter(walletname=refscan)
if survexscansfolders: if survexscansfolders:
survexblock.survexscansfolder = survexscansfolders[0] survexblock.survexscansfolder = survexscansfolders[0]
#survexblock.refscandir = "%s/%s%%23%s" % (mref.group(1), mref.group(1), mref.group(2))
survexblock.save() survexblock.save()
# print(insp+' - Wallet ; ref - %s - found in survexscansfolders' % refscan)
else: else:
message = ' ! Wallet ; ref - %s - NOT found in survexscansfolders %s-%s-%s' % (refscan,yr,letterx,wallet) message = ' ! Wallet ; ref {} - NOT found in survexscansfolders {}'.format(refscan, survexblock.survexfile.path)
print((insp+message)) print((insp+message))
models.DataIssue.objects.create(parser='survex', message=message) models.DataIssue.objects.create(parser='survex', message=message)
# This whole section should be moved if we can have *QM become a proper survex command # This whole section should be moved if we can have *QM become a proper survex command
# Spec of QM in SVX files, currently commented out need to add to survex # Spec of QM in SVX files, currently commented out need to add to survex
# needs to match regex_qm # needs to match rx_qm
# ;Serial number grade(A/B/C/D/X) nearest-station resolution-station description # ;Serial number grade(A/B/C/D/X) nearest-station resolution-station description
# ;QM1 a hobnob_hallway_2.42 hobnob-hallway_3.42 junction of keyhole passage # ;QM1 a hobnob_hallway_2.42 hobnob-hallway_3.42 junction of keyhole passage
# ;QM1 a hobnob_hallway_2.42 - junction of keyhole passage # ;QM1 a hobnob_hallway_2.42 - junction of keyhole passage
qmline = comment and regex_qm.match(comment) qmline = comment and rx_qm.match(comment)
if qmline: if qmline:
# print(insp+qmline.groups())
#(u'1', u'B', u'miraclemaze', u'1.17', u'-', None, u'\tcontinuation of rift')
qm_no = qmline.group(1) qm_no = qmline.group(1)
qm_grade = qmline.group(2) qm_grade = qmline.group(2)
qm_from_section = qmline.group(3) qm_from_section = qmline.group(3)
@ -252,11 +234,9 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
from_section = models_survex.SurvexBlock.objects.filter(name=qm_from_section) from_section = models_survex.SurvexBlock.objects.filter(name=qm_from_section)
# If we can find a section (survex note chunck, named) # If we can find a section (survex note chunck, named)
if len(from_section) > 0: if len(from_section) > 0:
# print(insp+from_section[0])
from_station = models_survex.SurvexStation.objects.filter(block=from_section[0], name=qm_from_station) from_station = models_survex.SurvexStation.objects.filter(block=from_section[0], name=qm_from_station)
# If we can find a from station then we have the nearest station and can import it # If we can find a from station then we have the nearest station and can import it
if len(from_station) > 0: if len(from_station) > 0:
# print(insp+from_station[0])
qm = models_caves.QM.objects.create(number=qm_no, qm = models_caves.QM.objects.create(number=qm_no,
nearest_station=from_station[0], nearest_station=from_station[0],
grade=qm_grade.upper(), grade=qm_grade.upper(),
@ -265,12 +245,11 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
# print(insp+' - QM found but resolved') # print(insp+' - QM found but resolved')
pass pass
#print(insp+'Cave -sline ' + str(cave))
if not sline: if not sline:
continue continue
# detect the star ref command # detect the star ref command
mstar = regex_starref.match(sline) mstar = rx_starref.match(sline)
if mstar: if mstar:
yr,letterx,wallet = mstar.groups() yr,letterx,wallet = mstar.groups()
if not letterx: if not letterx:
@ -286,26 +265,20 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
if survexscansfolders: if survexscansfolders:
survexblock.survexscansfolder = survexscansfolders[0] survexblock.survexscansfolder = survexscansfolders[0]
survexblock.save() survexblock.save()
# print(insp+' - Wallet *REF - %s - found in survexscansfolders' % refscan)
else: else:
message = ' ! Wallet *REF - %s - NOT found in survexscansfolders %s-%s-%s' % (refscan,yr,letterx,wallet) message = ' ! Wallet *REF {} - NOT found in survexscansfolders {}'.format(refscan, survexblock.survexfile.path)
print((insp+message)) print((insp+message))
models.DataIssue.objects.create(parser='survex', message=message) models.DataIssue.objects.create(parser='survex', message=message)
continue continue
# detect the star command # detect the star command
mstar = regex_star.match(sline) mstar = rx_star.match(sline)
if not mstar: if not mstar:
if "from" in stardata: if "from" in stardata:
# print(insp+'Cave ' + str(survexfile.cave))
# print(insp+survexblock)
LoadSurvexLineLeg(survexblock, stardata, sline, comment, survexfile.cave) LoadSurvexLineLeg(survexblock, stardata, sline, comment, survexfile.cave)
# print(insp+' - From: ')
# print(insp+stardata)
pass pass
elif stardata["type"] == "passage": elif stardata["type"] == "passage":
LoadSurvexLinePassage(survexblock, stardata, sline, comment) LoadSurvexLinePassage(survexblock, stardata, sline, comment)
# print(insp+' - Passage: ')
#Missing "station" in stardata. #Missing "station" in stardata.
continue continue
@ -352,12 +325,10 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
previousnlegs = survexlegsnumber previousnlegs = survexlegsnumber
name = line.lower() name = line.lower()
print((insp+' - Begin found for: ' + name)) print((insp+' - Begin found for: ' + name))
# print(insp+'Block cave: ' + str(survexfile.cave))
survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0) survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0)
survexblockdown.save() survexblockdown.save()
survexblock.save() survexblock.save()
survexblock = survexblockdown survexblock = survexblockdown
# print(insp+survexblockdown)
textlinesdown = [ ] textlinesdown = [ ]
insp += "> " insp += "> "
RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown) RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown)
@ -369,21 +340,17 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
if iblankbegins: if iblankbegins:
iblankbegins -= 1 iblankbegins -= 1
else: else:
#survexblock.text = "".join(textlines)
# .text not used, using it for number of legs per block # .text not used, using it for number of legs per block
legsinblock = survexlegsnumber - previousnlegs legsinblock = survexlegsnumber - previousnlegs
print(insp+"LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber)) print(insp+"LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber))
survexblock.text = str(legsinblock) survexblock.text = str(legsinblock)
survexblock.save() survexblock.save()
# print(insp+' - End found: ')
endstamp = datetime.now() endstamp = datetime.now()
timetaken = endstamp - stamp timetaken = endstamp - stamp
# print(insp+' - Time to process: ' + str(timetaken))
return return
elif re.match("date$(?i)", cmd): elif re.match("date$(?i)", cmd):
if len(line) == 10: if len(line) == 10:
#print(insp+' - Date found: ' + line)
survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone()) survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone())
expeditions = models.Expedition.objects.filter(year=line[:4]) expeditions = models.Expedition.objects.filter(year=line[:4])
if expeditions: if expeditions:
@ -395,9 +362,9 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
elif re.match("team$(?i)", cmd): elif re.match("team$(?i)", cmd):
pass pass
# print(insp+' - Team found: ') # print(insp+' - Team found: ')
mteammember = regex_team.match(line) mteammember = rx_team.match(line)
if mteammember: if mteammember:
for tm in regex_team_member.split(mteammember.group(2)): for tm in rx_team_member.split(mteammember.group(2)):
if tm: if tm:
personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower()) personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
if (personexpedition, tm) not in teammembers: if (personexpedition, tm) not in teammembers:
@ -409,7 +376,6 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
personrole.save() personrole.save()
elif cmd == "title": elif cmd == "title":
#print(insp+' - Title found: ')
survextitle = models_survex.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexfile.cave) survextitle = models_survex.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexfile.cave)
survextitle.save() survextitle.save()
pass pass
@ -419,11 +385,8 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
pass pass
elif cmd == "data": elif cmd == "data":
#print(insp+' - Data found: ')
ls = line.lower().split() ls = line.lower().split()
stardata = { "type":ls[0] } stardata = { "type":ls[0] }
#print(insp+' - Star data: ', stardata)
#print(insp+ls)
for i in range(0, len(ls)): for i in range(0, len(ls)):
stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1 stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1
if ls[0] in ["normal", "cartesian", "nosurvey"]: if ls[0] in ["normal", "cartesian", "nosurvey"]:
@ -434,18 +397,27 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
assert ls[0] == "passage", line assert ls[0] == "passage", line
elif cmd == "equate": elif cmd == "equate":
#print(insp+' - Equate found: ')
LoadSurvexEquate(survexblock, line) LoadSurvexEquate(survexblock, line)
elif cmd == "set" and re.match("names(?i)", line):
pass
elif cmd == "flags":
# Here we could set on/off 'splay', 'not splay', 'surface', 'not surface', or 'duplicate'
# but this data is only used for sense-checking not to actually calculate anything important
pass
elif cmd == "fix": elif cmd == "fix":
#print(insp+' - Fix found: ')
survexblock.MakeSurvexStation(line.split()[0]) survexblock.MakeSurvexStation(line.split()[0])
elif cmd in ["alias", "calibrate", "cs","entrance", "export", "case",
"declination", "infer","instrument", "sd", "units"]:
# we ignore all these, which is fine.
pass
else: else:
#print(insp+' - Stuff') if cmd not in ["include", "data", "flags", "title", "set", "ref"]:
if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument", message = "! Bad svx command: [*{}] {} ({}) {}".format(cmd, line, survexblock, survexblock.survexfile.path)
"calibrate", "set", "infer", "alias", "cs", "declination", "case"]: print((insp+message))
message = "! Bad svx command in line:%s %s %s %s" % (cmd, line, survexblock, survexblock.survexfile.path) models.DataIssue.objects.create(parser='survex', message=message)
else:
message = "! Unparsed [*{}]: '{}' {}".format(cmd, line, survexblock.survexfile.path)
print((insp+message)) print((insp+message))
models.DataIssue.objects.create(parser='survex', message=message) models.DataIssue.objects.create(parser='survex', message=message)