expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-12-16 12:27:10 +00:00
Code Issues 18 Projects 3 Activity

fix survexdirectories to ref files properly

Browse Source
This commit is contained in:
Philip Sargent
2020-07-01 17:41:09 +01:00
parent 5feb07e3f6
commit 8cc768e5b6
1 changed files with 39 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
parsers/survex.py
View File

@@ -64,6 +64,7 @@ class LoadingSurvex():
stacksvxfiles = [] stacksvxfiles = []
svxfileslist = [] svxfileslist = []
svxdirs = {} svxdirs = {}
survexdict = {} # each key is a directory, and its value is a list of files
lineno = 0 lineno = 0
insp = "" insp = ""
callcount = 0 callcount = 0
@@ -105,6 +106,7 @@ class LoadingSurvex():
if (personexpedition, tm) not in teammembers: if (personexpedition, tm) not in teammembers:
teammembers.append((personexpedition, tm)) teammembers.append((personexpedition, tm))
personrole = models_survex.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm) personrole = models_survex.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
personrole.save()
personrole.expeditionday = survexblock.expeditionday personrole.expeditionday = survexblock.expeditionday
if personexpedition: if personexpedition:
personrole.person=personexpedition.person personrole.person=personexpedition.person
@@ -326,10 +328,16 @@ class LoadingSurvex():
return None return None
def GetSurvexDirectory(self, headpath): def GetSurvexDirectory(self, headpath):
"""This creates a SurvexDirectory if it has not been seen before, and on creation
it sets the primarysurvexfile. This is correct as it should be set on the first file
in the directory, where first is defined by the *include ordering. Which is what we
are doing.
"""
if not headpath: if not headpath:
return self.svxdirs[""] return self.svxdirs[""]
if headpath.lower() not in self.svxdirs: if headpath.lower() not in self.svxdirs:
self.svxdirs[headpath.lower()] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile) self.svxdirs[headpath.lower()] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile)
self.svxdirs[headpath.lower()].save()
return self.svxdirs[headpath.lower()] return self.svxdirs[headpath.lower()]
def ReportNonCaveIncludes(self, headpath, includelabel): def ReportNonCaveIncludes(self, headpath, includelabel):
@@ -340,9 +348,9 @@ class LoadingSurvex():
for i in self.ignoreprefix: for i in self.ignoreprefix:
if headpath.startswith(i): if headpath.startswith(i):
return return
message = " ! {} is not a cave. (while creating {} sfile & sdirectory)".format(headpath, includelabel) message = " ! {} is not a cave. (while creating '{}' sfile & sdirectory)".format(headpath, includelabel)
print(message) print("\n"+message)
print(message,file=sys.stderr) print("\n"+message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message) models.DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexFile(self, includelabel): def LoadSurvexFile(self, includelabel):
@@ -353,25 +361,30 @@ class LoadingSurvex():
""" """
depth = " " * self.depthbegin depth = " " * self.depthbegin
print("{:2}{} - NEW survexfile:'{}'".format(self.depthbegin, depth, includelabel)) print("{:2}{} - NEW survexfile:'{}'".format(self.depthbegin, depth, includelabel))
headpath, tail = os.path.split(includelabel)
newfile = models_survex.SurvexFile(path=includelabel) newfile = models_survex.SurvexFile(path=includelabel)
headpath, tail = os.path.split(includelabel) newfile.save() # until we do this there is no internal id so no foreign key works
self.currentsurvexfile = newfile
newdirectory = self.GetSurvexDirectory(headpath) newdirectory = self.GetSurvexDirectory(headpath)
newdirectory.save()
newfile.survexdirectory = newdirectory
self.survexdict[newdirectory] = [newfile,]
cave = self.IdentifyCave(headpath) # cave already exists in db
if not newdirectory: if not newdirectory:
message = " ! 'None' SurvexDirectory returned from GetSurvexDirectory({})".format(headpath) message = " ! 'None' SurvexDirectory returned from GetSurvexDirectory({})".format(headpath)
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message) models.DataIssue.objects.create(parser='survex', message=message)
newfile.survexdirectory = newdirectory
cave = self.IdentifyCave(headpath)
if cave: if cave:
newdirectory.cave = cave newdirectory.cave = cave
newfile.cave = cave newfile.cave = cave
#print("\n"+str(newdirectory.cave),file=sys.stderr)
else: else:
self.ReportNonCaveIncludes(headpath, includelabel) self.ReportNonCaveIncludes(headpath, includelabel)
if not newfile.survexdirectory: if not newfile.survexdirectory:
message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(includelabel) message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(includelabel)
print(message) print(message)
@@ -384,7 +397,6 @@ class LoadingSurvex():
print(newdirectory, file=sys.stderr) print(newdirectory, file=sys.stderr)
print(newdirectory.primarysurvexfile, file=sys.stderr) print(newdirectory.primarysurvexfile, file=sys.stderr)
raise raise
self.currentsurvexfile = newfile
def ProcessIncludeLine(self, included): def ProcessIncludeLine(self, included):
svxid = included.groups()[0] svxid = included.groups()[0]
@@ -443,7 +455,7 @@ class LoadingSurvex():
if cave: if cave:
survexfile.cave = cave survexfile.cave = cave
def LinearRecursiveLoad(self, survexblock, path, svxlines): def LinearLoad(self, survexblock, path, svxlines):
"""Loads a single survex file. Usually used to import all the survex files which have been collated """Loads a single survex file. Usually used to import all the survex files which have been collated
into a single file. Loads the begin/end blocks recursively. into a single file. Loads the begin/end blocks recursively.
""" """
@@ -491,17 +503,18 @@ class LoadingSurvex():
pathlist += "." + id pathlist += "." + id
newsurvexblock = models_survex.SurvexBlock(name=blockid, parent=survexblock, newsurvexblock = models_survex.SurvexBlock(name=blockid, parent=survexblock,
survexpath=pathlist, survexpath=pathlist,
title = survexblock.title, # copy parent inititally
cave=self.currentcave, survexfile=self.currentsurvexfile, cave=self.currentcave, survexfile=self.currentsurvexfile,
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0) legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
newsurvexblock.save()
survexblock = newsurvexblock survexblock = newsurvexblock
survexblock.survexfile.save() # django insists on this although it is already saved !? # survexblock.survexfile.save()
survexblock.save() # django insists on this , but we want to save at the end ! survexblock.save() # django insists on this , but we want to save at the end !
tickle() tickle()
# ---------------------------END # ---------------------------END
elif re.match("end$(?i)", cmd): elif re.match("end$(?i)", cmd):
depth = " " * self.depthbegin depth = " " * self.depthbegin
self.currentsurvexblock = survexblock.parent
print("{:2}{} - End from:'{}'".format(self.depthbegin,depth,args)) print("{:2}{} - End from:'{}'".format(self.depthbegin,depth,args))
legsinblock = self.survexlegsnumber - previousnlegs legsinblock = self.survexlegsnumber - previousnlegs
@@ -518,12 +531,14 @@ class LoadingSurvex():
except: except:
print(survexblock, file=sys.stderr) print(survexblock, file=sys.stderr)
raise raise
self.currentsurvexblock = survexblock.parent
survexblock = survexblock.parent
blockid = self.stackbegin.pop() blockid = self.stackbegin.pop()
self.depthbegin -= 1 self.depthbegin -= 1
# ----------------------------- # -----------------------------
elif re.match("(?i)title$", cmd): elif re.match("(?i)title$", cmd):
survexblock.title = args # only apply to current survexblock survexblock.title = args # block has own title, overwrite that from parent
elif re.match("(?i)ref$", cmd): elif re.match("(?i)ref$", cmd):
self.LoadSurvexRef(survexblock, args) self.LoadSurvexRef(survexblock, args)
elif re.match("(?i)flags$", cmd): elif re.match("(?i)flags$", cmd):
@@ -594,6 +609,7 @@ class LoadingSurvex():
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath) path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
includesurvexfile = models_survex.SurvexFile(path=includepath) includesurvexfile = models_survex.SurvexFile(path=includepath)
includesurvexfile.save()
if includesurvexfile.exists(): if includesurvexfile.exists():
# do not create SurvexFile in DB here by doing includesurvexfile.save(). Do it when reading data. # do not create SurvexFile in DB here by doing includesurvexfile.save(). Do it when reading data.
@@ -614,6 +630,7 @@ class LoadingSurvex():
print(message,file=flinear) print(message,file=flinear)
print(message,file=sys.stderr) print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message) models.DataIssue.objects.create(parser='survex', message=message)
includesurvexfile.path += "-TEMP"
includesurvexfile = None includesurvexfile = None
flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop)) flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop))
fcollate.write(";*edulcni {}\n".format(pop)) fcollate.write(";*edulcni {}\n".format(pop))
@@ -621,7 +638,7 @@ class LoadingSurvex():
self.depthinclude -= 1 self.depthinclude -= 1
#-------------------------------------------------------- #--------------------------------------------------------
else: else:
message = " ! ERROR *include file not found for {}".format(includesurvexfile) message = " ! ERROR *include file not found for [{}]:'{}'".format(includesurvexfile, includepath)
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message) models.DataIssue.objects.create(parser='survex', message=message)
@@ -714,7 +731,7 @@ def FindAndLoadSurvex(survexblockroot):
with open(collatefilename, "r") as fcollate: with open(collatefilename, "r") as fcollate:
svxlines = fcollate.read().splitlines() svxlines = fcollate.read().splitlines()
#---------------------------------------------------------------- #----------------------------------------------------------------
svx_load.LinearRecursiveLoad(survexblockroot,survexfileroot.path, svxlines) svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines)
#---------------------------------------------------------------- #----------------------------------------------------------------
print("\n - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr) print("\n - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr)
@@ -723,6 +740,12 @@ def FindAndLoadSurvex(survexblockroot):
survexlegsnumber = svx_load.survexlegsnumber survexlegsnumber = svx_load.survexlegsnumber
survexlegsalllength = svx_load.survexlegsalllength survexlegsalllength = svx_load.survexlegsalllength
mem1 = models.get_process_memory() mem1 = models.get_process_memory()
print(" - Number of SurvexDirectories: {}".format(len(svx_load.survexdict)))
tf=0
for d in svx_load.survexdict:
tf += len(svx_load.survexdict[d])
print(" - Number of SurvexFiles: {}".format(tf))
svx_load = None svx_load = None
# Close the logging file, Restore sys.stdout to our old saved file handle # Close the logging file, Restore sys.stdout to our old saved file handle