xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

all SurvexDirectory set OK, pending caves made

Browse Source
This commit is contained in:
Philip Sargent
2020-06-30 15:39:24 +01:00
parent 6b0275d035
commit 0a57ac3132
2 changed files with 75 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
parsers/survex.py
View File

@@ -68,6 +68,8 @@ class LoadingSurvex():
insp = ""
callcount = 0
stardata ={}
ignoreprefix = ["surface", "kataster", "fixedpts", "gpx"]
ignorenoncave = ["caves-1623", "caves-1623/2007-neu"]
includedfilename =""
currentsurvexblock = None
currentsurvexfile = None
@@ -308,7 +310,8 @@ class LoadingSurvex():
def IdentifyCave(self, cavepath):
if cavepath.lower() in self.caveslist:
return self.caveslist[cavepath.lower()]
# TO DO - some of this is already done in generating self.caveslist so simplify this
# esp. as it is in a loop.
path_match = self.rx_cave.search(cavepath)
if path_match:
sluggy = '{}-{}'.format(path_match.group(1), path_match.group(2))
@@ -329,6 +332,19 @@ class LoadingSurvex():
self.svxdirs[headpath.lower()] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile)
return self.svxdirs[headpath.lower()]
def ReportNonCaveIncludes(self, headpath, includelabel):
"""Ignore surface, kataser and gps *include survex files
"""
if headpath in self.ignorenoncave:
return
for i in self.ignoreprefix:
if headpath.startswith(i):
return
message = " ! {} is not a cave. (while creating {} sfile & sdirectory)".format(headpath, includelabel)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexFile(self, includelabel):
"""Creates SurvexFile in the database, and SurvexDirectory if needed
with links to 'cave'
@@ -338,28 +354,37 @@ class LoadingSurvex():
depth = " " * self.depthbegin
print("{:2}{} - NEW survexfile:'{}'".format(self.depthbegin, depth, includelabel))
newsurvexfile = models_survex.SurvexFile(path=includelabel)
newfile = models_survex.SurvexFile(path=includelabel)
headpath, tail = os.path.split(includelabel)
newsurvexdirectory = self.GetSurvexDirectory(headpath)
newsurvexfile.survexdirectory = newsurvexdirectory
newdirectory = self.GetSurvexDirectory(headpath)
if not newdirectory:
message = " ! 'None' SurvexDirectory returned from GetSurvexDirectory({})".format(headpath)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
newfile.survexdirectory = newdirectory
cave = self.IdentifyCave(headpath)
if cave:
newsurvexdirectory.cave = cave
newsurvexfile.cave = cave
# else:
# message = " ! Cannot identify cave from {} when creating sfile & sdirectory".format(headpath)
# print(message)
# print(message,file=sys.stderr)
# models.DataIssue.objects.create(parser='survex', message=message)
newdirectory.cave = cave
newfile.cave = cave
else:
self.ReportNonCaveIncludes(headpath, includelabel)
if not newfile.survexdirectory:
message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(includelabel)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
self.currentsurvexfile.save() # django insists on this although it is already saved !?
try:
newsurvexdirectory.save()
newdirectory.save()
except:
print(newsurvexdirectory, file=sys.stderr)
print(newsurvexdirectory.primarysurvexfile, file=sys.stderr)
print(newdirectory, file=sys.stderr)
print(newdirectory.primarysurvexfile, file=sys.stderr)
raise
self.currentsurvexfile = newsurvexfile
self.currentsurvexfile = newfile
def ProcessIncludeLine(self, included):
svxid = included.groups()[0]
@@ -418,129 +443,6 @@ class LoadingSurvex():
if cave:
survexfile.cave = cave
def RecursiveRecursiveLoad(self, survexblock, survexfile, fin):
"""Follows the *include links in all the survex files from the root file 1623.svx
and reads in the survex blocks, other data and the wallet references (scansfolder) as it
goes. This part of the data include process is where the maximum memory is used and where it
crashes on memory-constrained machines. Begin-end blocks may also be nested.
"""
# self.LoadSurvexSetup(survexblock, survexfile)
# insp =self.insp
# previousnlegs = 0
# svxlines = fin.read().splitlines()
# # cannot close file now as may be recursively called with the same fin if nested *begin-end
# for svxline in svxlines:
# self.lineno += 1
# sline, comment = self.rx_comment.match(svxline.strip()).groups()
# if comment:
# self.LoadSurvexComment(survexblock, comment)
# if not sline:
# continue # skip blank lines
# # detect the star command
# mstar = self.rx_star.match(sline)
# if mstar: # yes we are reading a *cmd
# cmd, args = mstar.groups()
# cmd = cmd.lower()
# if re.match("include$(?i)", cmd):
# cave = self.IdentifyCave(args)
# if cave:
# survexfile.cave = cave
# includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args)))
# print((insp+' - INCLUDE-go path found, including - ' + args))
# includesurvexfile = models_survex.SurvexFile(path=includepath)
# includesurvexfile.save()
# includesurvexfile.SetDirectory()
# if includesurvexfile.exists():
# survexblock.save()
# self.insp += "> "
# #--------------------------------------------------------
# fininclude = includesurvexfile.OpenFile()
# self.RecursiveRecursiveLoad(survexblock, includesurvexfile, fininclude)
# fininclude.close()
# #--------------------------------------------------------
# self.insp = self.insp[2:]
# insp = self.insp
# print((insp+' - INCLUDE-return from include - ' + includepath))
# else:
# print((insp+' ! ERROR *include file not found for %s' % includesurvexfile))
# elif re.match("begin$(?i)", cmd):
# # On a *begin statement we start a new survexblock.
# # There should not be any *include inside a begin-end block, so this is a simple
# # load not a recursive fileload. But there may be many blocks nested to any depth in one file.
# if args:
# newsvxpath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args))
# # Try to find the cave in the DB if not use the string as before
# path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", newsvxpath)
# if path_match:
# pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
# # print(insp+pos_cave)
# cave = models_caves.getCaveByReference(pos_cave)
# if cave:
# survexfile.cave = cave
# else:
# print((insp+' - No match (b) for %s' % newsvxpath))
# previousnlegs = self.survexlegsnumber
# name = args.lower()
# print(insp+' - Begin found for:{}, creating new SurvexBlock '.format(name))
# # the recursive call re-reads the entire file. This is wasteful. We should pass in only
# # the un-parsed part of the file.
# survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock,
# survexpath=survexblock.survexpath+"."+name,
# cave=survexfile.cave, survexfile=survexfile,
# legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
# survexblockdown.save()
# survexblock.save()
# survexblock = survexblockdown
# print(insp+" - BLOCK-enter nested *begin/*end block: '{}'".format(name))
# self.insp += "> "
# #--------------------------------------------------------
# self.RecursiveRecursiveLoad(survexblockdown, survexfile, fin)
# #--------------------------------------------------------
# # do not close the file as there may be more blocks in this one
# # and it is re-read afresh with every nested begin-end block.
# self.insp = self.insp[2:]
# insp = self.insp
# else:
# self.depthbegin += 1
# elif re.match("end$(?i)", cmd):
# if self.depthbegin:
# print(insp+" - BLOCK-return from nested *begin/*end block: '{}'".format(args))
# self.depthbegin -= 1
# else:
# legsinblock = self.survexlegsnumber - previousnlegs
# print(insp+" - LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,self.survexlegsnumber))
# survexblock.legsall = legsinblock
# survexblock.save()
# return
# elif cmd == "ref":
# self.LoadSurvexRef(survexblock, args)
# elif cmd == "flags":
# self.LoadSurvexFlags(args, cmd)
# elif cmd == "data":
# self.LoadSurvexDataCmd(survexblock, args)
# elif cmd == "set" and re.match("names(?i)", args):
# pass
# elif re.match("date$(?i)", cmd):
# self.LoadSurvexDate(survexblock, args)
# elif re.match("team$(?i)", cmd):
# self.LoadSurvexTeam(survexblock, args)
# else:
# self.LoadSurvexIgnore(survexblock, args, cmd)
# else: # not a *cmd so we are reading data OR rx_comment failed
# if "from" in self.stardata: # only interested in survey legs
# self.LoadSurvexLineLeg(survexblock, svxline, sline, comment)
# else:
# pass # ignore all other sorts of data
pass
def LinearRecursiveLoad(self, survexblock, path, svxlines):
"""Loads a single survex file. Usually used to import all the survex files which have been collated
into a single file. Loads the begin/end blocks recursively.
@@ -702,7 +604,9 @@ class LoadingSurvex():
flinear.write("{:2} {} *include {}\n".format(self.depthinclude, indent, includesurvexfile.path))
push = includesurvexfile.path.lower()
self.stackinclude.append(push)
#-----------------
self.RecursiveScan(survexblock, includesurvexfile, fininclude, flinear, fcollate)
#-----------------
pop = self.stackinclude.pop()
if pop != push:
message = "!!!!!!! ERROR pop != push {} != {} {}".format(pop, push, self.stackinclude)
@@ -710,8 +614,9 @@ class LoadingSurvex():
print(message,file=flinear)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, includesurvexfile.path))
fcollate.write(";*edulcni {}\n".format(includesurvexfile.path))
includesurvexfile = None
flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop))
fcollate.write(";*edulcni {}\n".format(pop))
fininclude.close()
self.depthinclude -= 1
#--------------------------------------------------------
@@ -820,16 +725,6 @@ def FindAndLoadSurvex(survexblockroot):
mem1 = models.get_process_memory()
svx_load = None
# print('\n - Loading All Survex Blocks (RecursiveRecursive)',file=sys.stderr)
# svxlrl = LoadingSurvex()
# finroot = survexfileroot.OpenFile()
# svxlrl.RecursiveRecursiveLoad(survexblockroot, survexfileroot, finroot)
# finroot.close()
# survexlegsnumber = svxlrl.survexlegsnumber
# survexlegsalllength = svxlrl.survexlegsalllength
# svxlrl = None
# Close the logging file, Restore sys.stdout to our old saved file handle
sys.stdout.close()
print("+", file=sys.stderr)