xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

State push/pop working

Browse Source
This commit is contained in:
Philip Sargent 2020-07-03 14:53:36 +01:00
parent e54436e818
commit 67f66b72e8
1 changed files with 195 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
parsers/survex.py
View File

@ -2,6 +2,7 @@ import sys
import os
import re
import time
import copy
from datetime import datetime, timedelta
from subprocess import call, run
@ -35,6 +36,7 @@ class LoadingSurvex():
A 'scansfolder' is what we today call a "survey scans folder" or a "wallet".
"""
rx_flags = re.compile(r"not\s")
rx_linelen = re.compile(r"[\d\-+.]+$")
rx_team = re.compile(r"(?i)(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$")
rx_person = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$")
@ -54,13 +56,17 @@ class LoadingSurvex():
# This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g.
# *DATA normal from to length gradient bearing ignore ignore ignore ignore
stardatadefault = {"type":"normal", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
flagsdefault = {"duplicate":False, "surface":False, "splay":False, "any":False}
stardata ={}
starflags = {}
survexlegsalllength = 0.0
survexlegsnumber = 0
depthbegin = 0
depthinclude = 0
stackbegin =[]
stackflags =[]
stackdata =[]
stackinclude = []
stacksvxfiles = []
svxfileslist = []
@ -124,12 +130,30 @@ class LoadingSurvex():
survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date)
survexblock.save()
def LoadSurvexLineLeg(self, survexblock, svxline, sline, comment):
def LoadSurvexLineLeg(self, survexblock, sline, comment):
"""This reads compass, clino and tape data but only keeps the tape lengths,
the rest is discarded after error-checking.
"""
# Check first to see if we are in a splay and abort if so.
# TO DO splay abort
#print("! LEG stardata type:{}++{}\n{} ".format(self.stardata["type"].upper(), survexblock.survexfile.path, sline))
# SKIP PASSAGES *data passage
if self.stardata["type"] == "passage":
return
if self.stardata["type"] == "cartesian":
return
if self.stardata["type"] == "nosurvey":
return
if self.stardata["type"] == "diving":
return
if self.stardata["type"] == "cylpolar":
return
#print(" !! LEG data lineno:{}\n !! sline:'{}'\n !! stardata['tape']: {}".format(self.lineno, sline, self.stardata["tape"]))
# # For speed this should come first. But we are checking validity too.
# if self.starflags["any"]:
# survexleg.tape = invalid_tape
# #return
if self.stardata["type"] != "normal":
return
invalid_clino = 180.0
invalid_compass = 720.0
invalid_tape = 0.0
@ -137,10 +161,7 @@ class LoadingSurvex():
survexleg = SurvexLeg()
ls = sline.lower().split()
# this next fails for two surface survey svx files which use / for decimal point
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
if stardata["type"] == "normal": # should use current flags setting for this. May not be default order!
#print("! stardata {}++{}\n{} ".format(stardata, survexblock.survexfile.path, sline), file=sys.stderr)
try:
tape = ls[stardata["tape"]]
except:
@ -151,6 +172,8 @@ class LoadingSurvex():
models.DataIssue.objects.create(parser='survexleg', message=message)
survexleg.tape = invalid_tape
return
# this next fails for two surface survey svx files which use / for decimal point
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
tape = tape.replace("(","")
tape = tape.replace(")","")
tape = tape.replace("/",".")
@ -211,7 +234,13 @@ class LoadingSurvex():
models.DataIssue.objects.create(parser='survexleg', message=message)
survexleg.compass = invalid_compass
#print(" !! lineno '{}'\n !! svxline '{}'\n !! sline '{}'\n !! ls '{}'\n !! stardata {}".format(self.lineno, svxline, sline, ls,stardata))
# For speed this should come first. But we are checking validity too.
if self.starflags["any"]:
pass
# Comment out until we have the *data commands working!
#survexleg.tape = invalid_tape
#return
# delete the object to save memory
survexleg = None
@ -319,10 +348,12 @@ class LoadingSurvex():
if args == "":
# naked '*data' which is relevant only for passages. Ignore. Continue with previous settings.
return
# DEFAULT | NORMAL | CARTESIAN| NOSURVEY |PASSAGE | TOPOFIL | CYLPOLAR | DIVING
ls = args.lower().split()
if ls[0] == "normal":
if not (("from" in stardata and "to" in stardata) or "station" in stardata):
if ls[0] == "default":
self.stardata = self.stardatadefault
elif ls[0] == "normal" or ls[0] == "topofil":
if not ("from" in stardata and "to" in stardata):
message = " ! - Unrecognised *data normal statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath)
print(message)
print(message,file=sys.stderr)
@ -340,26 +371,48 @@ class LoadingSurvex():
stardata["tape"] = i-1
self.stardata = stardata
return
elif ls[0] == "default":
self.stardata = self.stardatadefault
elif ls[0] == "passage" or ls[0] == "nosurvey":
# we ignore everything else, such as '*data passage'
pass
elif ls[0] == "cartesian" or ls[0] == "nosurvey":
message = " ! - *data cartesian survey blocks are ignored. Length not calculated. '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath)
elif ls[0] == "cartesian" or ls[0] == "nosurvey" or ls[0] == "diving" or ls[0] == "cylpolar" or ls[0] == "passage":
message = " ! - *data {} blocks ignored. {}|{}" '{}' .format(ls[0].upper(), survexblock.name, survexblock.survexpath, args)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
#print(message,file=sys.stderr)
#models.DataIssue.objects.create(parser='survex', message=message)
self.stardata["type"] = ls[0]
else:
message = " ! - Unrecognised *data statement '{}'".format(args)
message = " ! - Unrecognised *data statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexFlags(self, line, cmd):
# Here we could set on/off 'splay', 'not splay', 'surface', 'not surface', or 'duplicate'
# but this data is only used for sense-checking not to actually calculate anything important
pass
def LoadSurvexFlags(self, args):
# Valid flags are DUPLICATE, SPLAY, and SURFACE, and a flag may be preceded with NOT to turn it off.
# Default values are NOT any of them
self.starflags = copy.deepcopy(self.flagsdefault)
flags = []
args = self.rx_flags.sub("not",args)
argslist = args.split()
for s in argslist:
flags.append(s)
if "duplicate" in flags:
self.starflags["duplicate"] = True
if "surface" in flags:
self.starflags["surface"] = True
if "splay" in flags:
self.starflags["splay"] = True
if "notduplicate" in flags:
self.starflags["duplicate"] = False
if "notsurface" in flags:
self.starflags["surface"] = False
if "notsplay" in flags:
self.starflags["splay"] = False
# if self.starflags["duplicate"] == True or self.starflags["surface"] == True or self.starflags["splay"] == True:
# actually we do want to count duplicates as this is for "effort expended in surveying underground"
if self.starflags["surface"] == True or self.starflags["splay"] == True:
self.starflags["any"] = True
def IdentifyCave(self, cavepath):
if cavepath.lower() in self.caveslist:
@ -412,7 +465,7 @@ class LoadingSurvex():
Creates a new current survexfile and valid .survexdirectory
The survexblock passed-in is not necessarily the parent. FIX THIS.
"""
self.stardata = self.stardatadefault
# self.stardata = self.stardatadefault
depth = " " * self.depthbegin
print("{:2}{} - NEW survexfile:'{}'".format(self.depthbegin, depth, svxid))
@ -518,8 +571,13 @@ class LoadingSurvex():
self.currentsurvexfile = survexblock.survexfile
self.currentsurvexfile.save() # django insists on this although it is already saved !?
self.stardata = copy.deepcopy(self.stardatadefault)
#self.stackdata.append(self.stardata) # and extra push will do it ?
self.starflags = copy.deepcopy(self.flagsdefault)
#self.stackflags.append(self.starflags)
blockcount = 0
lineno = 0
self.lineno = 0
def tickle():
nonlocal blockcount
blockcount +=1
@ -528,13 +586,15 @@ class LoadingSurvex():
if blockcount % 200 ==0 :
print("\n", file=sys.stderr,end='')
print(" - MEM:{:7.3f} MB in use".format(models.get_process_memory()),file=sys.stderr)
print(" ", file=sys.stderr,end='')
sys.stderr.flush()
for svxline in svxlines:
lineno += 1
self.lineno += 1
sline, comment = self.rx_comment.match(svxline).groups()
if comment:
self.LoadSurvexComment(survexblock, comment) # this catches the ;*include and ;*edulcni lines too
# this catches the ;*include NEWFILE and ;*edulcni ENDOFFILE lines too
self.LoadSurvexComment(survexblock, comment)
if not sline:
continue # skip blank lines
@ -547,18 +607,25 @@ class LoadingSurvex():
# ------------------------BEGIN
if re.match("begin$(?i)", cmd):
self.depthbegin += 1
depth = " " * self.depthbegin
blockid = args.lower()
self.stackbegin.append(blockid)
blkid = args.lower()
self.stackbegin.append(blkid)
# PUSH state ++++++++++++++
self.stackflags.append(copy.deepcopy(self.starflags))
self.stackdata.append(copy.deepcopy(self.stardata))
print(" # stackDATA after *begin 'type':", end="")
for dict in self.stackdata:
print("'{}' ".format(dict["type"].upper()), end="")
print("")
# PUSH state ++++++++++++++
previousnlegs = self.survexlegsnumber
print("{:2}{} - Begin for :'{}'".format(self.depthbegin,depth, blockid))
print("{:2}{} - Begin for :'{}'".format(self.depthbegin,depth, blkid))
pathlist = ""
for id in self.stackbegin:
if len(id) > 0:
pathlist += "." + id
newsurvexblock = models_survex.SurvexBlock(name=blockid, parent=survexblock,
newsurvexblock = models_survex.SurvexBlock(name=blkid, parent=survexblock,
survexpath=pathlist,
cave=self.currentcave, survexfile=self.currentsurvexfile,
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
@ -589,7 +656,18 @@ class LoadingSurvex():
raise
self.currentsurvexblock = survexblock.parent
survexblock = survexblock.parent
blockid = self.stackbegin.pop()
blkid = self.stackbegin.pop()
oldflags = self.starflags
# POP state ++++++++++++++
self.stardata = copy.deepcopy(self.stackdata.pop())
print(" # stackDATA at *end 'type':", end="")
for dict in self.stackdata:
print("'{}' ".format(dict["type"].upper()), end="")
print("")
self.starflags = copy.deepcopy(self.stackflags.pop())
if oldflags["any"] != self.starflags["any"]:
print(" # POP 'any' flag now:'{}' was:{} ".format(self.starflags["any"], oldflags["any"]))
# POP state ++++++++++++++
self.depthbegin -= 1
# -----------------------------
@ -598,7 +676,11 @@ class LoadingSurvex():
elif re.match("(?i)ref$", cmd):
self.LoadSurvexRef(survexblock, args)
elif re.match("(?i)flags$", cmd):
self.LoadSurvexFlags(args, cmd)
oldflags = self.starflags
self.LoadSurvexFlags(args)
if oldflags["any"] != self.starflags["any"]:
print(" # CHANGE 'any' flag now:'{}' was:{} ".format(self.starflags["any"], oldflags["any"]))
elif re.match("(?i)data$", cmd):
self.LoadSurvexDataCmd(survexblock, args)
elif re.match("(?i)date$", cmd):
@ -615,10 +697,8 @@ class LoadingSurvex():
else:
self.LoadSurvexIgnore(survexblock, args, cmd)
else: # not a *cmd so we are reading data OR rx_comment failed
if "from" in self.stardata: # only interested in survey legs
self.LoadSurvexLineLeg(survexblock, svxline, sline, comment)
else:
pass # ignore all other sorts of data
self.LoadSurvexLineLeg(survexblock, sline, comment)
def RecursiveScan(self, survexblock, path, fin, flinear, fcollate):
"""Follows the *include links in all the survex files from the root file 1623.svx
@ -631,7 +711,7 @@ class LoadingSurvex():
if self.callcount % 10 ==0 :
print(".", file=sys.stderr,end='')
if self.callcount % 500 ==0 :
print("\n", file=sys.stderr,end='')
print("\n ", file=sys.stderr,end='')
@ -726,13 +806,13 @@ class LoadingSurvex():
pass
def RunSurvexIfNeeded(self,fullpath):
cav_t = 0
log_t = 0
svx_t = 0
now = time.time()
cav_t = now - 365*24*3600
log_t = now - 365*24*3600
svx_t = now - 365*24*3600
def runcavern():
print(" - Regenerating stale cavern .log and .3d for '{}'\n days old: {:.1f} {:.1f} {:.1f}".
print(" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{}'\n days svx old: {:.1f} cav old:{:.1f} log old: {:.1f}".
format(fullpath, (svx_t - log_t)/(24*3600), (cav_t - log_t)/(24*3600), (now - log_t)/(24*3600)))
call([settings.CAVERN, "--log", "--output={}".format(fullpath), "{}.svx".format(fullpath)])
@ -761,7 +841,7 @@ class LoadingSurvex():
if cav_t - log_t > 0: # new version of cavern
runcavern()
return
if ChaosMonkey(30):
if ChaosMonkey(200):
runcavern()
def FindAndLoadSurvex(survexblockroot):
@ -789,6 +869,7 @@ def FindAndLoadSurvex(survexblockroot):
print(" - MEM:{:7.2f} MB START".format(mem0),file=sys.stderr)
flinear = open('svxlinear.log', 'w')
flinear.write(" - MEM:{:7.2f} MB START {}\n".format(mem0,survexfileroot.path))
print(" ", file=sys.stderr,end='')
finroot = survexfileroot.OpenFile()
fcollate.write(";*include {}\n".format(survexfileroot.path))
@ -905,7 +986,7 @@ def LoadPos():
svx_t = 0
d3d_t = 0
def runcavern3d():
print(" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{}'\n days old: {:.1f} {:.1f} {:.1f}".
print(" - Regenerating stale cavern .log and .3d for '{}'\n days old: {:.1f} {:.1f} {:.1f}".
format(topdata, (svx_t - d3d_t)/(24*3600), (cav_t - d3d_t)/(24*3600), (now - d3d_t)/(24*3600)))
call([settings.CAVERN, "--log", "--output={}".format(topdata), "{}.svx".format(topdata)])
call([settings.THREEDTOPOS, '{}.3d'.format(topdata)], cwd = settings.SURVEX_DATA)
@ -928,6 +1009,7 @@ def LoadPos():
svx_t = os.path.getmtime(svxpath)
if os.path.isfile(d3dpath):
# always fails to find log file if a double directory, e.g. caves-1623/B4/B4/B4.svx Why ?
d3d_t = os.path.getmtime(d3dpath)
now = time.time()