rename variables and leglength calc

This commit is contained in:
Philip Sargent 2020-07-04 01:10:17 +01:00
parent abbe8d467b
commit 5ed6271c08

View File

@ -48,7 +48,8 @@ class LoadingSurvex():
rx_comminc = re.compile(r'(?i)^\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include
rx_commcni = re.compile(r'(?i)^\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni
rx_include = re.compile(r'(?i)^\s*(\*include[\s].*)$')
rx_ref = re.compile(r'(?i)^\s*ref[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)')
rx_ref = re.compile(r'(?i)^\s*ref(erence)?[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)')
rx_ref_text= re.compile(r'(?i)^\s*\"[^"]*\"\s*$')
rx_star = re.compile(r'(?i)\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$')
rx_starref = re.compile(r'(?i)^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$')
rx_argsref = re.compile(r'(?i)^[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$')
@ -56,14 +57,16 @@ class LoadingSurvex():
# This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g.
# *DATA normal from to length gradient bearing ignore ignore ignore ignore
datastardefault = {"type":"normal", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
flagsdefault = {"duplicate":False, "surface":False, "splay":False, "any":False}
flagsdefault = {"duplicate":False, "surface":False, "splay":False, "skiplegs":False}
datastar ={}
flagsstar = {}
survexlegsalllength = 0.0
survexlegsnumber = 0
slength = 0.0
legsnumber = 0
depthbegin = 0
depthinclude = 0
legsnumberstack = []
slengthstack = []
stackbegin =[]
flagsstack =[]
datastack =[]
@ -134,6 +137,13 @@ class LoadingSurvex():
"""This reads compass, clino and tape data but only keeps the tape lengths,
the rest is discarded after error-checking.
"""
invalid_clino = 180.0
invalid_compass = 720.0
invalid_tape = 0.0
# if self.flagsstar["skiplegs"]:
# return
#print("! LEG datastar type:{}++{}\n{} ".format(self.datastar["type"].upper(), survexblock.survexfile.path, sline))
# SKIP PASSAGES *data passage
if self.datastar["type"] == "passage":
@ -146,20 +156,14 @@ class LoadingSurvex():
return
if self.datastar["type"] == "cylpolar":
return
#print(" !! LEG data lineno:{}\n !! sline:'{}'\n !! datastar['tape']: {}".format(self.lineno, sline, self.datastar["tape"]))
# # For speed this should come first. But we are checking validity too.
# if self.flagsstar["any"]:
# survexleg.tape = invalid_tape
# #return
# print(" !! LEG data lineno:{}\n !! sline:'{}'\n !! datastar['tape']: {}".format(self.lineno, sline, self.datastar["tape"]))
if self.datastar["type"] != "normal":
return
invalid_clino = 180.0
invalid_compass = 720.0
invalid_tape = 0.0
datastar = self.datastar # shallow copy: alias but the things inside are the same things
survexleg = SurvexLeg()
ls = sline.lower().split()
try:
@ -174,12 +178,12 @@ class LoadingSurvex():
return
# this next fails for two surface survey svx files which use / for decimal point
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
tape = tape.replace("(","")
tape = tape.replace(")","")
tape = tape.replace("/",".")
# tape = tape.replace("(","") # edited original file (only one) instead
# tape = tape.replace(")","") # edited original file (only one) instead
# tape = tape.replace("/",".") # edited original file (only one) instead.
try:
survexleg.tape = float(tape)
self.survexlegsnumber += 1
self.legsnumber += 1
except ValueError:
print(("! Tape misread in", survexblock.survexfile.path))
print((" datastar:", datastar))
@ -189,7 +193,7 @@ class LoadingSurvex():
survexleg.tape = invalid_tape
try:
survexblock.totalleglength += survexleg.tape
self.survexlegsalllength += survexleg.tape
self.slength += survexleg.tape
except ValueError:
message = ' ! Value Error: Tape length not added %s in %s' % (ls, survexblock.survexfile.path)
models.DataIssue.objects.create(parser='survexleg', message=message)
@ -234,20 +238,23 @@ class LoadingSurvex():
models.DataIssue.objects.create(parser='survexleg', message=message)
survexleg.compass = invalid_compass
# For speed this should come first. But we are checking validity too.
if self.flagsstar["any"]:
pass
# Comment out until we have the *data commands working!
#survexleg.tape = invalid_tape
#return
# delete the object to save memory
if self.flagsstar["skiplegs"]:
return
# delete the object to save memory
survexleg = None
def LoadSurvexRef(self, survexblock, args):
print(self.insp+ "*REF ---- '"+ args +"'")
# *REF but also ; Ref years from 1960 to 2039
refline = self.rx_ref_text.match(args)
if refline:
# a textual reference such as "1996-1999 Not-KH survey book pp 92-95"
print(self.insp+ "*REF quoted text so ignored:"+ args)
return
if len(args)< 4:
message = " ! Empty or BAD *REF command '{}' in '{}'".format(args, survexblock.survexfile.path)
message = " ! Empty or BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
return
@ -256,8 +263,8 @@ class LoadingSurvex():
if argsgps:
yr, letterx, wallet = argsgps.groups()
else:
message = " ! BAD *REF command '{}' in '{}'".format(args, survexblock.survexfile.path)
print((self.insp+message))
message = " ! BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path)
print(self.insp+message)
models.DataIssue.objects.create(parser='survex', message=message)
return
@ -372,10 +379,10 @@ class LoadingSurvex():
self.datastar = copy.deepcopy(datastar)
return
elif ls[0] == "cartesian" or ls[0] == "nosurvey" or ls[0] == "diving" or ls[0] == "cylpolar" or ls[0] == "passage":
message = " ! - *data {} blocks ignored. {}|{}" '{}' .format(ls[0].upper(), survexblock.name, survexblock.survexpath, args)
print(message)
#print(message,file=sys.stderr)
#models.DataIssue.objects.create(parser='survex', message=message)
# message = " ! - *data {} blocks ignored. {}|{}" '{}' .format(ls[0].upper(), survexblock.name, survexblock.survexpath, args)
# print(message)
# print(message,file=sys.stderr)
# models.DataIssue.objects.create(parser='survex', message=message)
self.datastar["type"] = ls[0]
else:
message = " ! - Unrecognised *data statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath)
@ -412,7 +419,7 @@ class LoadingSurvex():
# if self.flagsstar["duplicate"] == True or self.flagsstar["surface"] == True or self.flagsstar["splay"] == True:
# actually we do want to count duplicates as this is for "effort expended in surveying underground"
if self.flagsstar["surface"] == True or self.flagsstar["splay"] == True:
self.flagsstar["any"] = True
self.flagsstar["skiplegs"] = True
def IdentifyCave(self, cavepath):
if cavepath.lower() in self.caveslist:
@ -459,18 +466,17 @@ class LoadingSurvex():
print("\n"+message)
print("\n"+message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexFile(self, svxid):
"""Creates SurvexFile in the database, and SurvexDirectory if needed
with links to 'cave'
Creates a new current survexfile and valid .survexdirectory
The survexblock passed-in is not necessarily the parent. FIX THIS.
"""
# self.datastar = self.datastardefault
print(" # datastack in LoadSurvexFile:{} 'type':".format(svxid), end="")
for dict in self.datastack:
print("'{}' ".format(dict["type"].upper()), end="")
print("")
# print(" # datastack in LoadSurvexFile:{} 'type':".format(svxid), end="")
# for dict in self.datastack:
# print("'{}' ".format(dict["type"].upper()), end="")
# print("")
depth = " " * self.depthbegin
@ -512,10 +518,11 @@ class LoadingSurvex():
print(newdirectory.primarysurvexfile, file=sys.stderr)
raise
print(" # datastack end LoadSurvexFile:{} 'type':".format(svxid), end="")
for dict in self.datastack:
print("'{}' ".format(dict["type"].upper()), end="")
print("")
# print(" # datastack end LoadSurvexFile:{} 'type':".format(svxid), end="")
# for dict in self.datastack:
# print("'{}' ".format(dict["type"].upper()), end="")
# print("")
pass
def ProcessIncludeLine(self, included):
svxid = included.groups()[0]
@ -555,7 +562,7 @@ class LoadingSurvex():
def LoadSurvexSetup(self,survexblock, survexfile):
self.depthbegin = 0
self.datastar = self.datastardefault
blocklegs = self.survexlegsnumber
blocklegs = self.legsnumber
print(self.insp+" - MEM:{:.3f} Reading. parent:{} <> {} ".format(models.get_process_memory(),survexblock.survexfile.path, survexfile.path))
self.lineno = 0
sys.stderr.flush();
@ -576,6 +583,14 @@ class LoadingSurvex():
"""Loads a single survex file. Usually used to import all the survex files which have been collated
into a single file. Loads the begin/end blocks using a stack for labels.
"""
blkid = None
pathlist = None
args = None
oldflags = None
blockcount = 0
self.lineno = 0
slengthtotal = 0.0
nlegstotal = 0
self.relativefilename = path
cave = self.IdentifyCave(path) # this will produce null for survex files which are geographic collections
@ -584,13 +599,6 @@ class LoadingSurvex():
self.datastar = copy.deepcopy(self.datastardefault)
self.flagsstar = copy.deepcopy(self.flagsdefault)
blkid = None
pathlist = None
args = None
previousnlegs = None
oldflags = None
blockcount = 0
self.lineno = 0
def tickle():
nonlocal blockcount
@ -609,6 +617,7 @@ class LoadingSurvex():
nonlocal pathlist
depth = " " * self.depthbegin
self.insp = depth
print("{:2}{} - Begin for :'{}'".format(self.depthbegin,depth, blkid))
pathlist = ""
for id in self.stackbegin:
@ -617,31 +626,28 @@ class LoadingSurvex():
def printend():
nonlocal args
nonlocal previousnlegs
depth = " " * self.depthbegin
print("{:2}{} - End from:'{}'".format(self.depthbegin,depth,args))
legsinblock = self.survexlegsnumber - previousnlegs
print("{:2}{} - LEGS: {} (previous: {}, now:{})".format(self.depthbegin,
depth, legsinblock, previousnlegs, self.survexlegsnumber))
survexblock.legsall = legsinblock
print("{:2}{} - LEGS: {} (n: {}, length:{})".format(self.depthbegin,
depth, self.slength, self.slength, self.legsnumber))
def pushblock():
nonlocal blkid
print(" # datastack at 1 *begin {} 'type':".format(blkid), end="")
for dict in self.datastack:
print("'{}' ".format(dict["type"].upper()), end="")
print("")
print("'{}' self.datastar ".format(self.datastar["type"].upper()))
# print(" # datastack at 1 *begin {} 'type':".format(blkid), end="")
# for dict in self.datastack:
# print("'{}' ".format(dict["type"].upper()), end="")
# print("")
# print("'{}' self.datastar ".format(self.datastar["type"].upper()))
# ------------ * DATA
self.datastack.append(copy.deepcopy(self.datastar))
# ------------ * DATA
print(" # datastack at 2 *begin {} 'type':".format(blkid), end="")
for dict in self.datastack:
print("'{}' ".format(dict["type"].upper()), end="")
print("")
print("'{}' self.datastar ".format(self.datastar["type"].upper()))
# print(" # datastack at 2 *begin {} 'type':".format(blkid), end="")
# for dict in self.datastack:
# print("'{}' ".format(dict["type"].upper()), end="")
# print("")
# print("'{}' self.datastar ".format(self.datastar["type"].upper()))
# ------------ * FLAGS
self.flagsstack.append(copy.deepcopy(self.flagsstar))
# ------------ * FLAGS
@ -650,32 +656,33 @@ class LoadingSurvex():
nonlocal blkid
nonlocal oldflags
print(" # datastack at *end '{} 'type':".format(blkid), end="")
for dict in self.datastack:
print("'{}' ".format(dict["type"].upper()), end="")
print("")
print("'{}' self.datastar ".format(self.datastar["type"].upper()))
# print(" # datastack at *end '{} 'type':".format(blkid), end="")
# for dict in self.datastack:
# print("'{}' ".format(dict["type"].upper()), end="")
# print("")
# print("'{}' self.datastar ".format(self.datastar["type"].upper()))
# ------------ * DATA
self.datastar = copy.deepcopy(self.datastack.pop())
# ------------ * DATA
print(" # datastack after *end '{} 'type':".format(blkid), end="")
for dict in self.datastack:
print("'{}' ".format(dict["type"].upper()), end="")
print("")
print("'{}' self.datastar ".format(self.datastar["type"].upper()))
# print(" # datastack after *end '{} 'type':".format(blkid), end="")
# for dict in self.datastack:
# print("'{}' ".format(dict["type"].upper()), end="")
# print("")
# print("'{}' self.datastar ".format(self.datastar["type"].upper()))
# ------------ * FLAGS
self.flagsstar = copy.deepcopy(self.flagsstack.pop())
# ------------ * FLAGS
if oldflags["any"] != self.flagsstar["any"]:
print(" # POP 'any' flag now:'{}' was:{} ".format(self.flagsstar["any"], oldflags["any"]))
if oldflags["skiplegs"] != self.flagsstar["skiplegs"]:
print(" # POP 'any' flag now:'{}' was:{} ".format(self.flagsstar["skiplegs"], oldflags["skiplegs"]))
def starstatement(mstar):
nonlocal survexblock
nonlocal blkid
nonlocal pathlist
nonlocal args
nonlocal previousnlegs
nonlocal oldflags
nonlocal slengthtotal
nonlocal nlegstotal
cmd, args = mstar.groups()
cmd = cmd.lower()
@ -685,9 +692,12 @@ class LoadingSurvex():
blkid = args.lower()
# PUSH state ++++++++++++++
self.stackbegin.append(blkid)
self.legsnumberstack.append(self.legsnumber)
self.slengthstack.append(self.slength)
pushblock()
# PUSH state ++++++++++++++
previousnlegs = self.survexlegsnumber
self.legsnumber = 0
self.slength = 0.0
printbegin()
newsurvexblock = models_survex.SurvexBlock(name=blkid, parent=survexblock,
survexpath=pathlist,
@ -701,7 +711,12 @@ class LoadingSurvex():
# ---------------------------END
elif re.match("end$(?i)", cmd):
survexblock.legsall = self.legsnumber
survexblock.totalleglength = self.slength
printend()
slengthtotal += self.slength
nlegstotal += self.legsnumber
try:
survexblock.parent.save() # django insists on this although it is already saved !?
except:
@ -714,12 +729,14 @@ class LoadingSurvex():
raise
# POP state ++++++++++++++
popblock()
self.legsnumber = self.legsnumberstack.pop()
self.slength = self.slengthstack.pop()
blkid = self.stackbegin.pop()
# POP state ++++++++++++++
self.currentsurvexblock = survexblock.parent
survexblock = survexblock.parent
oldflags = self.flagsstar
self.depthbegin -= 1
# POP state ++++++++++++++
# -----------------------------
elif re.match("(?i)title$", cmd):
@ -729,8 +746,8 @@ class LoadingSurvex():
elif re.match("(?i)flags$", cmd):
oldflags = self.flagsstar
self.LoadSurvexFlags(args)
if oldflags["any"] != self.flagsstar["any"]:
print(" # CHANGE 'any' flag now:'{}' was:{} ".format(self.flagsstar["any"], oldflags["any"]))
if oldflags["skiplegs"] != self.flagsstar["skiplegs"]:
print(" # CHANGE 'any' flag now:'{}' was:{} ".format(self.flagsstar["skiplegs"], oldflags["skiplegs"]))
elif re.match("(?i)data$", cmd):
self.LoadSurvexDataCmd(survexblock, args)
@ -761,11 +778,14 @@ class LoadingSurvex():
# detect a star command
mstar = self.rx_star.match(sline)
if mstar: # yes we are reading a *cmd
if mstar:
# yes we are reading a *command
starstatement(mstar)
else: # not a *cmd so we are reading data OR rx_comment failed
else: # not a *cmd so we are reading data OR a ";" rx_comment failed
self.LoadSurvexLineLeg(survexblock, sline, comment)
self.legsnumber = slengthtotal
self.slength = nlegstotal
def RecursiveScan(self, survexblock, path, fin, flinear, fcollate):
"""Follows the *include links in all the survex files from the root file 1623.svx
@ -783,7 +803,7 @@ class LoadingSurvex():
if path in self.svxfileslist:
message = " * Warning. Duplicate in *include list at:{} depth:{} file:{}".format(self.callcount, self.depthinclude, path)
message = " * Warning. Duplicate detected in *include list at callcount:{} depth:{} file:{}".format(self.callcount, self.depthinclude, path)
print(message)
print(message,file=flinear)
print("\n"+message,file=sys.stderr)
@ -879,7 +899,7 @@ class LoadingSurvex():
svx_t = now - 365*24*3600
def runcavern():
print(" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{}'\n days svx old: {:.1f} cav old:{:.1f} log old: {:.1f}".
print(" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{}'\n days svx old: {:.1f} cav:{:.1f} log old: {:.1f}".
format(fullpath, (svx_t - log_t)/(24*3600), (cav_t - log_t)/(24*3600), (now - log_t)/(24*3600)))
call([settings.CAVERN, "--log", "--output={}".format(fullpath), "{}.svx".format(fullpath)])
@ -919,7 +939,7 @@ def FindAndLoadSurvex(survexblockroot):
# Redirect sys.stdout to the file
sys.stdout = open('svxblks.log', 'w')
print(' - SCANNING All Survex Blocks...',file=sys.stderr)
print(' - Scanning Survex Blocks tree from {}.svx ...'.format(settings.SURVEX_TOPNAME),file=sys.stderr)
survexfileroot = survexblockroot.survexfile # i.e. SURVEX_TOPNAME only
collatefilename = "_" + survexfileroot.path + ".svx"
@ -980,8 +1000,8 @@ def FindAndLoadSurvex(survexblockroot):
print("\n - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr)
print(" - MEM:{:7.3f} MB USED".format(mem1-mem0),file=sys.stderr)
survexlegsnumber = svx_load.survexlegsnumber
survexlegsalllength = svx_load.survexlegsalllength
legsnumber = svx_load.legsnumber
slength = svx_load.slength
mem1 = models.get_process_memory()
print(" - Number of SurvexDirectories: {}".format(len(svx_load.survexdict)))
@ -996,7 +1016,7 @@ def FindAndLoadSurvex(survexblockroot):
print("+", file=sys.stderr)
sys.stderr.flush();
sys.stdout = stdout_orig
return (survexlegsnumber, survexlegsalllength)
return (legsnumber, slength)
def MakeSurvexFileRoot():
"""Returns a file_object.path = SURVEX_TOPNAME associated with directory_object.path = SURVEX_DATA
@ -1030,22 +1050,23 @@ def LoadSurvexBlocks():
print(' - Loading Survex Blocks...')
memstart = models.get_process_memory()
#----------------------------------------------------------------
survexlegsnumber, survexlegsalllength = FindAndLoadSurvex(survexblockroot)
legsnumber, slength = FindAndLoadSurvex(survexblockroot)
#----------------------------------------------------------------
memend = models.get_process_memory()
print(" - MEMORY start:{:.3f} MB end:{:.3f} MB increase={:.3f} MB".format(memstart,memend, memend-memstart))
survexblockroot.totalleglength = survexlegsalllength
survexblockroot.legsall = survexlegsnumber
# Don't do this, it double-counts everything:
#survexblockroot.totalleglength = slength
#survexblockroot.legsall = legsnumber
survexblockroot.save()
print(" - total number of survex legs: {}".format(survexlegsnumber))
print(" - total leg lengths loaded: {}m".format(survexlegsalllength))
print(" - total number of survex legs: {}".format(legsnumber))
print(" - total leg lengths loaded: {}m".format(slength))
print(' - Loaded All Survex Blocks.')
poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
def LoadPos():
def LoadPositions():
"""First load the survex stations for entrances and fixed points (about 600) into the database.
Run cavern to produce a complete .3d file, then run 3dtopos to produce a table of
all survey point positions. Then lookup each position by name to see if we have it in the database
@ -1074,6 +1095,7 @@ def LoadPos():
svxpath = topdata + ".svx"
d3dpath = topdata + ".3d"
pospath = topdata + ".pos"
svx_t = os.path.getmtime(svxpath)
@ -1082,11 +1104,13 @@ def LoadPos():
d3d_t = os.path.getmtime(d3dpath)
now = time.time()
if not os.path.isfile(pospath):
runcavern3d()
if not os.path.isfile(d3dpath):
runcavern3d()
elif svx_t - d3d_t > 0: # stale, 3d older than svx file
elif d3d_t - svx_t > 0: # stale, 3d older than svx file
runcavern3d()
elif now - d3d_t > 60 *24*60*60: # >60 days old, re-run anyway
elif now - d3d_t> 60 *24*60*60: # >60 days old, re-run anyway
runcavern3d()
elif cav_t - d3d_t > 0: # new version of cavern
runcavern3d()