xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

fixing parent blocks & titles

Browse Source
This commit is contained in:
Philip Sargent
2020-06-28 14:42:26 +01:00
parent 122cdd7fc8
commit bf1c683fd0
4 changed files with 204 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/TESTS/tests.py
View File

@@ -1,4 +1,6 @@
""" """
We are using unittest for troggle.
This file demonstrates two different styles of tests (one doctest and one This file demonstrates two different styles of tests (one doctest and one
unittest). These will both pass when you run "manage.py test". unittest). These will both pass when you run "manage.py test".
@@ -42,7 +44,7 @@ class SimpleTest(SimpleTestCase):
from django.template.defaultfilters import slugify from django.template.defaultfilters import slugify
from django.utils.timezone import get_current_timezone, make_aware from django.utils.timezone import get_current_timezone, make_aware
from troggle.core.models import DataIssue, Expedition from troggle.core.models import DataIssue, Expedition
from troggle.core.models_caves import Cave, OtherCaveName, getCaveByReference, LogbookEntry, PersonTrip from troggle.core.models_caves import Cave, OtherCaveName, LogbookEntry, PersonTrip
from parsers.people import GetPersonExpeditionNameLookup from parsers.people import GetPersonExpeditionNameLookup
def test_import_core_views_caves(self): def test_import_core_views_caves(self):
from django.http import HttpResponse, HttpResponseRedirect from django.http import HttpResponse, HttpResponseRedirect
@@ -52,6 +54,7 @@ class SimpleTest(SimpleTestCase):
from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
from troggle.helper import login_required_if_public from troggle.helper import login_required_if_public
def test_import_parses_mix(self): def test_import_parses_mix(self):
from troggle.parsers.logbooks import GetCaveLookup
import troggle.settings import troggle.settings
import troggle.flatpages.models import troggle.flatpages.models
import troggle.logbooksdump import troggle.logbooksdump

2
core/models_caves.py
View File

@@ -125,7 +125,7 @@ class Cave(TroggleModel):
return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,}))
def __str__(self, sep = ": "): def __str__(self, sep = ": "):
return str(self.slug()) return "[Cave:" + str(self.slug()) + "]"
def get_QMs(self): def get_QMs(self):
return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all()) return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all())

14
core/models_survex.py
View File

@@ -17,7 +17,7 @@ class SurvexDirectory(models.Model):
ordering = ('id',) ordering = ('id',)
def __str__(self): def __str__(self):
return str(self.path) + "-" + str(self.primarysurvexfile.path) return "[SurvexDirectory:"+str(self.path) + "-" + str(self.primarysurvexfile.path) + "-" + str(self.cave)+"]"
@@ -29,6 +29,9 @@ class SurvexFile(models.Model):
class Meta: class Meta:
ordering = ('id',) ordering = ('id',)
def __str__(self):
return "[SurvexFile:"+str(self.path) + "-" + str(self.survexdirectory) + "-" + str(self.cave)+"]"
def exists(self): def exists(self):
fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx") fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx")
return os.path.isfile(fname) return os.path.isfile(fname)
@@ -114,12 +117,15 @@ class SurvexBlock(models.Model):
class Meta: class Meta:
ordering = ('id',) ordering = ('id',)
def __str__(self):
return "[SurvexBlock:"+ str(self.name) + "-path:" + \
str(self.survexpath) + "-cave:" + \
str(self.cave) + "]"
def isSurvexBlock(self): # Function used in templates def isSurvexBlock(self): # Function used in templates
return True return True
def __str__(self):
return self.name and str(self.name) or 'no name'
def GetPersonroles(self): def GetPersonroles(self):
res = [ ] res = [ ]
for personrole in self.personrole_set.order_by('personexpedition'): for personrole in self.personrole_set.order_by('personexpedition'):

352
parsers/survex.py
View File

@@ -45,7 +45,7 @@ class LoadingSurvex():
rx_qm = re.compile(r'(?i)^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$') rx_qm = re.compile(r'(?i)^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')
# remember there is also QM_PATTERN used in views_other and set in settings.py # remember there is also QM_PATTERN used in views_other and set in settings.py
rx_cave = re.compile(r'caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/') rx_cave = re.compile(r'(?i)caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)')
rx_comment = re.compile(r'([^;]*?)\s*(?:;\s*(.*))?\n?$') rx_comment = re.compile(r'([^;]*?)\s*(?:;\s*(.*))?\n?$')
rx_comminc = re.compile(r'(?i)^\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include rx_comminc = re.compile(r'(?i)^\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include
rx_commcni = re.compile(r'(?i)^\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni rx_commcni = re.compile(r'(?i)^\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni
@@ -61,16 +61,15 @@ class LoadingSurvex():
depthinclude = 0 depthinclude = 0
stackbegin =[] stackbegin =[]
stackinclude = [] stackinclude = []
stacksvxfiles = []
svxfileslist = [] svxfileslist = []
svxdirs = {} svxdirs = {}
svxcaves = {} svxcaves = {}
svxfiletitle = {}
lineno = 0 lineno = 0
insp = "" insp = ""
callcount = 0 callcount = 0
stardata ={} stardata ={}
includedfilename ="" includedfilename =""
currenttitle =""
currentsurvexblock = None currentsurvexblock = None
currentsurvexfile = None currentsurvexfile = None
currentcave = None currentcave = None
@@ -79,9 +78,7 @@ class LoadingSurvex():
pass pass
def LoadSurvexIgnore(self, survexblock, line, cmd): def LoadSurvexIgnore(self, survexblock, line, cmd):
if cmd == "title": if cmd == "require":
pass # unused in troggle today - but will become text list on SurvexBlock
elif cmd == "require":
pass # should we check survex version available for processing? pass # should we check survex version available for processing?
elif cmd in ["equate", "fix", "alias", "calibrate", "cs","entrance", "export", "case", elif cmd in ["equate", "fix", "alias", "calibrate", "cs","entrance", "export", "case",
"declination", "infer","instrument", "sd", "units"]: "declination", "infer","instrument", "sd", "units"]:
@@ -314,9 +311,9 @@ class LoadingSurvex():
return self.svxcaves[cavepath] return self.svxcaves[cavepath]
path_match = self.rx_cave.search(cavepath) path_match = self.rx_cave.search(cavepath)
#print(' - Attempting cave match for %s' % cavepath)
if path_match: if path_match:
sluggy = '%s-%s'.format(path_match.group(1), path_match.group(2)) sluggy = '{}-{}'.format(path_match.group(1), path_match.group(2))
print(' - Attempting cave match for %s' % sluggy)
cave = GetCaveLookup().get(sluggy) cave = GetCaveLookup().get(sluggy)
# Below is how it has been done for years: very fuzzy & slow searches # Below is how it has been done for years: very fuzzy & slow searches
# ..and wrong! # ..and wrong!
@@ -327,59 +324,57 @@ class LoadingSurvex():
print(' - Cave matched for %s' % cavepath) print(' - Cave matched for %s' % cavepath)
return cave return cave
else: else:
print(' ! No cave match for %s' % cavepath) print(' ! Failed to set cave for {} or {}'.format(cavepath, sluggy))
else:
print(' ! No regex cave match for %s' % cavepath)
return None return None
def LoadSurvexFileBlock(self, survexblock, includelabel): def LoadSurvexFile(self, includelabel):
"""Creates SurvexDirectory and SurvexFile in the database """Creates SurvexFile in the database, and SurvexDirectory if needed
with links to 'cave' with links to 'cave'
Creates a new current survexblock with valid .survexfile and valid .survexdirectory Creates a new current survexblock with valid .survexfile and valid .survexdirectory
The survexblock passed-in is not necessarily the parent. FIX THIS.
""" """
depth = " " * self.depthbegin depth = " " * self.depthbegin
print("{:2}{} - NEW survexfile:'{}'".format(self.depthbegin, depth, includelabel)) print("{:2}{} - NEW survexfile:'{}'".format(self.depthbegin, depth, includelabel))
headpath, tail = os.path.split(includelabel) headpath, tail = os.path.split(includelabel)
if headpath not in self.svxdirs: if headpath not in self.svxdirs:
self.svxdirs[headpath] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=survexblock.survexfile) self.svxdirs[headpath] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile)
newsurvexdirectory = self.svxdirs[headpath] newsurvexdirectory = self.svxdirs[headpath]
newsurvexfile = models_survex.SurvexFile(path=includelabel) newsurvexfile = models_survex.SurvexFile(path=includelabel)
newsurvexfile.survexdirectory = newsurvexdirectory newsurvexfile.survexdirectory = newsurvexdirectory
# Do not create a survexblock. Yes, there is a virtual block before the *begin statement but
# only the *title is usually in that, so just inherit the *title into the blocks.
# name = includelabel
# newsurvexblock = models_survex.SurvexBlock(name=name, parent=survexblock,
# survexpath=survexblock.survexpath+"."+name,
# survexfile=newsurvexfile,
# legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
cave = self.IdentifyCave(headpath) cave = self.IdentifyCave(headpath)
if cave: if cave:
newsurvexdirectory.cave = cave newsurvexdirectory.cave = cave
newsurvexfile.cave = cave newsurvexfile.cave = cave
#newsurvexblock.cave = cave self.currentsurvexfile.save() # django insists on this although it is already saved !?
try:
newsurvexdirectory.save() newsurvexdirectory.save()
newsurvexfile.save() except:
#newsurvexblock.save print(newsurvexdirectory, file=sys.stderr)
print(newsurvexdirectory.primarysurvexfile, file=sys.stderr)
raise
self.currentsurvexfile = newsurvexfile self.currentsurvexfile = newsurvexfile
#self.currentsurvexblock = newsurvexblock
def ProcessIncludeLine(self, survexblock, included): def ProcessIncludeLine(self, included):
# should do some push stuff here
svxid = included.groups()[0] svxid = included.groups()[0]
#depth = " " * self.depthbegin #depth = " " * self.depthbegin
#print("{:2}{} - Include survexfile:'{}'".format(self.depthbegin, depth, svxid)) #print("{:2}{} - Include survexfile:'{}'".format(self.depthbegin, depth, svxid))
self.LoadSurvexFileBlock(survexblock, svxid) self.LoadSurvexFile(svxid)
self.stacksvxfiles.append(self.currentsurvexfile)
def ProcessEdulcniLine(self, survexblock, edulcni):
# should do some pop stuff here def ProcessEdulcniLine(self, edulcni):
"""Saves the current survexfile in the db
"""
svxid = edulcni.groups()[0] svxid = edulcni.groups()[0]
depth = " " * self.depthbegin #depth = " " * self.depthbegin
print("{:2}{} - Edulcni survexfile:'{}'".format(self.depthbegin, depth, svxid)) #print("{:2}{} - Edulcni survexfile:'{}'".format(self.depthbegin, depth, svxid))
self.currentsurvexblock = survexblock.parent self.currentsurvexfile.save()
self.currentsurvexfile = survexblock.parent.survexfile self.currentsurvexfile = self.stacksvxfiles.pop()
def LoadSurvexComment(self, survexblock, comment): def LoadSurvexComment(self, survexblock, comment):
# ignore all comments except ;ref and ;QM and ;*include (for collated survex file) # ignore all comments except ;ref and ;QM and ;*include (for collated survex file)
@@ -395,12 +390,12 @@ class LoadingSurvex():
included = self.rx_comminc.match(comment) included = self.rx_comminc.match(comment)
# ;*include means we have been included; not 'proceed to include' which *include means # ;*include means we have been included; not 'proceed to include' which *include means
if included: if included:
self.ProcessIncludeLine(survexblock,included) self.ProcessIncludeLine(included)
edulcni = self.rx_commcni.match(comment) edulcni = self.rx_commcni.match(comment)
# ;*edulcni means we are returning from an included file # ;*edulcni means we are returning from an included file
if edulcni: if edulcni:
self.ProcessEdulcniLine(survexblock,edulcni) self.ProcessEdulcniLine(edulcni)
def LoadSurvexSetup(self,survexblock, survexfile): def LoadSurvexSetup(self,survexblock, survexfile):
self.depthbegin = 0 self.depthbegin = 0
@@ -428,121 +423,122 @@ class LoadingSurvex():
goes. This part of the data include process is where the maximum memory is used and where it goes. This part of the data include process is where the maximum memory is used and where it
crashes on memory-constrained machines. Begin-end blocks may also be nested. crashes on memory-constrained machines. Begin-end blocks may also be nested.
""" """
self.LoadSurvexSetup(survexblock, survexfile) # self.LoadSurvexSetup(survexblock, survexfile)
insp =self.insp # insp =self.insp
previousnlegs = 0 # previousnlegs = 0
svxlines = fin.read().splitlines() # svxlines = fin.read().splitlines()
# cannot close file now as may be recursively called with the same fin if nested *begin-end # # cannot close file now as may be recursively called with the same fin if nested *begin-end
for svxline in svxlines: # for svxline in svxlines:
self.lineno += 1 # self.lineno += 1
sline, comment = self.rx_comment.match(svxline.strip()).groups() # sline, comment = self.rx_comment.match(svxline.strip()).groups()
if comment: # if comment:
self.LoadSurvexComment(survexblock, comment) # self.LoadSurvexComment(survexblock, comment)
if not sline: # if not sline:
continue # skip blank lines # continue # skip blank lines
# detect the star command # # detect the star command
mstar = self.rx_star.match(sline) # mstar = self.rx_star.match(sline)
if mstar: # yes we are reading a *cmd # if mstar: # yes we are reading a *cmd
cmd, args = mstar.groups() # cmd, args = mstar.groups()
cmd = cmd.lower() # cmd = cmd.lower()
if re.match("include$(?i)", cmd): # if re.match("include$(?i)", cmd):
cave = self.IdentifyCave(args) # cave = self.IdentifyCave(args)
if cave: # if cave:
survexfile.cave = cave # survexfile.cave = cave
includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args))) # includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args)))
print((insp+' - INCLUDE-go path found, including - ' + args)) # print((insp+' - INCLUDE-go path found, including - ' + args))
includesurvexfile = models_survex.SurvexFile(path=includepath) # includesurvexfile = models_survex.SurvexFile(path=includepath)
includesurvexfile.save() # includesurvexfile.save()
includesurvexfile.SetDirectory() # includesurvexfile.SetDirectory()
if includesurvexfile.exists(): # if includesurvexfile.exists():
survexblock.save() # survexblock.save()
self.insp += "> " # self.insp += "> "
#-------------------------------------------------------- # #--------------------------------------------------------
fininclude = includesurvexfile.OpenFile() # fininclude = includesurvexfile.OpenFile()
self.RecursiveRecursiveLoad(survexblock, includesurvexfile, fininclude) # self.RecursiveRecursiveLoad(survexblock, includesurvexfile, fininclude)
fininclude.close() # fininclude.close()
#-------------------------------------------------------- # #--------------------------------------------------------
self.insp = self.insp[2:] # self.insp = self.insp[2:]
insp = self.insp # insp = self.insp
print((insp+' - INCLUDE-return from include - ' + includepath)) # print((insp+' - INCLUDE-return from include - ' + includepath))
else: # else:
print((insp+' ! ERROR *include file not found for %s' % includesurvexfile)) # print((insp+' ! ERROR *include file not found for %s' % includesurvexfile))
elif re.match("begin$(?i)", cmd): # elif re.match("begin$(?i)", cmd):
# On a *begin statement we start a new survexblock. # # On a *begin statement we start a new survexblock.
# There should not be any *include inside a begin-end block, so this is a simple # # There should not be any *include inside a begin-end block, so this is a simple
# load not a recursive fileload. But there may be many blocks nested to any depth in one file. # # load not a recursive fileload. But there may be many blocks nested to any depth in one file.
if args: # if args:
newsvxpath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args)) # newsvxpath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args))
# Try to find the cave in the DB if not use the string as before # # Try to find the cave in the DB if not use the string as before
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", newsvxpath) # path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", newsvxpath)
if path_match: # if path_match:
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2)) # pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
# print(insp+pos_cave) # # print(insp+pos_cave)
cave = models_caves.getCaveByReference(pos_cave) # cave = models_caves.getCaveByReference(pos_cave)
if cave: # if cave:
survexfile.cave = cave # survexfile.cave = cave
else: # else:
print((insp+' - No match (b) for %s' % newsvxpath)) # print((insp+' - No match (b) for %s' % newsvxpath))
previousnlegs = self.survexlegsnumber # previousnlegs = self.survexlegsnumber
name = args.lower() # name = args.lower()
print(insp+' - Begin found for:{}, creating new SurvexBlock '.format(name)) # print(insp+' - Begin found for:{}, creating new SurvexBlock '.format(name))
# the recursive call re-reads the entire file. This is wasteful. We should pass in only # # the recursive call re-reads the entire file. This is wasteful. We should pass in only
# the un-parsed part of the file. # # the un-parsed part of the file.
survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock, # survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock,
survexpath=survexblock.survexpath+"."+name, # survexpath=survexblock.survexpath+"."+name,
cave=survexfile.cave, survexfile=survexfile, # cave=survexfile.cave, survexfile=survexfile,
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0) # legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
survexblockdown.save() # survexblockdown.save()
survexblock.save() # survexblock.save()
survexblock = survexblockdown # survexblock = survexblockdown
print(insp+" - BLOCK-enter nested *begin/*end block: '{}'".format(name)) # print(insp+" - BLOCK-enter nested *begin/*end block: '{}'".format(name))
self.insp += "> " # self.insp += "> "
#-------------------------------------------------------- # #--------------------------------------------------------
self.RecursiveRecursiveLoad(survexblockdown, survexfile, fin) # self.RecursiveRecursiveLoad(survexblockdown, survexfile, fin)
#-------------------------------------------------------- # #--------------------------------------------------------
# do not close the file as there may be more blocks in this one # # do not close the file as there may be more blocks in this one
# and it is re-read afresh with every nested begin-end block. # # and it is re-read afresh with every nested begin-end block.
self.insp = self.insp[2:] # self.insp = self.insp[2:]
insp = self.insp # insp = self.insp
else: # else:
self.depthbegin += 1 # self.depthbegin += 1
elif re.match("end$(?i)", cmd): # elif re.match("end$(?i)", cmd):
if self.depthbegin: # if self.depthbegin:
print(insp+" - BLOCK-return from nested *begin/*end block: '{}'".format(args)) # print(insp+" - BLOCK-return from nested *begin/*end block: '{}'".format(args))
self.depthbegin -= 1 # self.depthbegin -= 1
else: # else:
legsinblock = self.survexlegsnumber - previousnlegs # legsinblock = self.survexlegsnumber - previousnlegs
print(insp+" - LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,self.survexlegsnumber)) # print(insp+" - LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,self.survexlegsnumber))
survexblock.legsall = legsinblock # survexblock.legsall = legsinblock
survexblock.save() # survexblock.save()
return # return
elif cmd == "ref": # elif cmd == "ref":
self.LoadSurvexRef(survexblock, args) # self.LoadSurvexRef(survexblock, args)
elif cmd == "flags": # elif cmd == "flags":
self.LoadSurvexFlags(args, cmd) # self.LoadSurvexFlags(args, cmd)
elif cmd == "data": # elif cmd == "data":
self.LoadSurvexDataCmd(survexblock, args) # self.LoadSurvexDataCmd(survexblock, args)
elif cmd == "set" and re.match("names(?i)", args): # elif cmd == "set" and re.match("names(?i)", args):
# pass
# elif re.match("date$(?i)", cmd):
# self.LoadSurvexDate(survexblock, args)
# elif re.match("team$(?i)", cmd):
# self.LoadSurvexTeam(survexblock, args)
# else:
# self.LoadSurvexIgnore(survexblock, args, cmd)
# else: # not a *cmd so we are reading data OR rx_comment failed
# if "from" in self.stardata: # only interested in survey legs
# self.LoadSurvexLineLeg(survexblock, svxline, sline, comment)
# else:
# pass # ignore all other sorts of data
pass pass
elif re.match("date$(?i)", cmd):
self.LoadSurvexDate(survexblock, args)
elif re.match("team$(?i)", cmd):
self.LoadSurvexTeam(survexblock, args)
else:
self.LoadSurvexIgnore(survexblock, args, cmd)
else: # not a *cmd so we are reading data OR rx_comment failed
if "from" in self.stardata: # only interested in survey legs
self.LoadSurvexLineLeg(survexblock, svxline, sline, comment)
else:
pass # ignore all other sorts of data
def LinearRecursiveLoad(self, survexblock, path, svxlines): def LinearRecursiveLoad(self, survexblock, path, svxlines):
"""Loads a single survex file. Usually used to import all the survex files which have been collated """Loads a single survex file. Usually used to import all the survex files which have been collated
@@ -551,7 +547,19 @@ class LoadingSurvex():
self.relativefilename = path self.relativefilename = path
cave = self.IdentifyCave(path) # this will produce null for survex files which are geographic collections cave = self.IdentifyCave(path) # this will produce null for survex files which are geographic collections
self.currentsurvexfile = survexblock.survexfile
self.currentsurvexfile.save() # django insists on this although it is already saved !?
blockcount = 0 blockcount = 0
def tickle():
nonlocal blockcount
blockcount +=1
if blockcount % 10 ==0 :
print(".", file=sys.stderr,end='')
if blockcount % 500 ==0 :
print("\n", file=sys.stderr,end='')
sys.stderr.flush();
for svxline in svxlines: for svxline in svxlines:
sline, comment = self.rx_comment.match(svxline.strip()).groups() sline, comment = self.rx_comment.match(svxline.strip()).groups()
if comment: if comment:
@@ -569,43 +577,50 @@ class LoadingSurvex():
if re.match("begin$(?i)", cmd): if re.match("begin$(?i)", cmd):
self.depthbegin += 1 self.depthbegin += 1
depth = " " * self.depthbegin depth = " " * self.depthbegin
self.stackbegin.append(args.lower()) blockid = args.lower()
self.stackbegin.append(blockid)
previousnlegs = self.survexlegsnumber previousnlegs = self.survexlegsnumber
name = args.lower() print("{:2}{} - Begin for :'{}'".format(self.depthbegin,depth, blockid))
print("{:2}{} - Begin for :'{}'".format(self.depthbegin,depth, name)) pathlist = ""
survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock, for id in self.stackbegin:
survexpath=survexblock.survexpath+"."+name, if len(id) > 0:
pathlist += "." + id
newsurvexblock = models_survex.SurvexBlock(name=blockid, parent=survexblock,
survexpath=pathlist,
cave=self.currentcave, survexfile=self.currentsurvexfile, cave=self.currentcave, survexfile=self.currentsurvexfile,
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0) legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
survexblockdown.save() survexblock = newsurvexblock
survexblock.save() survexblock.survexfile.save() # django insists on this although it is already saved !?
survexblock = survexblockdown survexblock.save() # django insists on this , but we want to save at the end !
tickle()
blockcount +=1
if blockcount % 10 ==0 :
print(".", file=sys.stderr,end='')
if blockcount % 500 ==0 :
print("\n", file=sys.stderr,end='')
sys.stderr.flush();
# ---------------------------END # ---------------------------END
elif re.match("end$(?i)", cmd): elif re.match("end$(?i)", cmd):
depth = " " * self.depthbegin depth = " " * self.depthbegin
self.currentsurvexblock = survexblock.parent self.currentsurvexblock = survexblock.parent
self.currentsurvexfile = survexblock.parent.survexfile
print("{:2}{} - End from:'{}'".format(self.depthbegin,depth,args)) print("{:2}{} - End from:'{}'".format(self.depthbegin,depth,args))
legsinblock = self.survexlegsnumber - previousnlegs legsinblock = self.survexlegsnumber - previousnlegs
print("{:2}{} - LEGS: {} (previous: {}, now:{})".format(self.depthbegin, print("{:2}{} - LEGS: {} (previous: {}, now:{})".format(self.depthbegin,
depth,legsinblock,previousnlegs,self.survexlegsnumber)) depth,legsinblock,previousnlegs,self.survexlegsnumber))
survexblock.legsall = legsinblock survexblock.legsall = legsinblock
survexblock.save() try:
survexblock.parent.save() # django insists on this although it is already saved !?
except:
print(survexblock.parent, file=sys.stderr)
raise
try:
survexblock.save() # save to db at end of block
except:
print(survexblock, file=sys.stderr)
raise
blockid = self.stackbegin.pop()
self.depthbegin -= 1 self.depthbegin -= 1
# ----------------------------- # -----------------------------
elif re.match("(?i)title$", cmd): elif re.match("(?i)title$", cmd):
self.currenttitle = args survexblock.title = args # only apply to current survexblock
elif re.match("(?i)ref$", cmd): elif re.match("(?i)ref$", cmd):
self.LoadSurvexRef(survexblock, args) self.LoadSurvexRef(survexblock, args)
elif re.match("(?i)flags$", cmd): elif re.match("(?i)flags$", cmd):
@@ -671,7 +686,7 @@ class LoadingSurvex():
if mstar: # yes we are reading a *cmd if mstar: # yes we are reading a *cmd
cmd, args = mstar.groups() cmd, args = mstar.groups()
cmd = cmd.lower() cmd = cmd.lower()
if re.match("include$(?i)", cmd): if re.match("(?i)include$", cmd):
includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args))) includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args)))
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath) path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
@@ -704,7 +719,7 @@ class LoadingSurvex():
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message) models.DataIssue.objects.create(parser='survex', message=message)
elif re.match("begin$(?i)", cmd): elif re.match("(?i)begin$", cmd):
self.depthbegin += 1 self.depthbegin += 1
depth = " " * self.depthbegin depth = " " * self.depthbegin
if args: if args:
@@ -714,7 +729,7 @@ class LoadingSurvex():
self.stackbegin.append(pushargs.lower()) self.stackbegin.append(pushargs.lower())
flinear.write(" {:2} {} *begin {}\n".format(self.depthbegin, depth, args)) flinear.write(" {:2} {} *begin {}\n".format(self.depthbegin, depth, args))
pass pass
elif re.match("end$(?i)", cmd): elif re.match("(?i)end$", cmd):
depth = " " * self.depthbegin depth = " " * self.depthbegin
flinear.write(" {:2} {} *end {}\n".format(self.depthbegin, depth, args)) flinear.write(" {:2} {} *end {}\n".format(self.depthbegin, depth, args))
if not args: if not args:
@@ -729,7 +744,10 @@ class LoadingSurvex():
self.depthbegin -= 1 self.depthbegin -= 1
pass pass
elif re.match("(?i)title$", cmd):
depth = " " * self.depthbegin
flinear.write(" {:2} {} *title {}\n".format(self.depthbegin, depth, args))
pass
def FindAndLoadSurvex(survexblockroot): def FindAndLoadSurvex(survexblockroot):
@@ -765,7 +783,7 @@ def FindAndLoadSurvex(survexblockroot):
flinear.write("{:2} {} *edulcni {}\n".format(svx_scan.depthinclude, indent, survexfileroot.path)) flinear.write("{:2} {} *edulcni {}\n".format(svx_scan.depthinclude, indent, survexfileroot.path))
fcollate.write(";*edulcni {}\n".format(survexfileroot.path)) fcollate.write(";*edulcni {}\n".format(survexfileroot.path))
mem1 = models.get_process_memory() mem1 = models.get_process_memory()
flinear.write(" - MEM:{:.2f} MB STOP {}\n".format(mem1,survexfileroot.path)) flinear.write("\n - MEM:{:.2f} MB STOP {}\n".format(mem1,survexfileroot.path))
flinear.write(" - MEM:{:.3f} MB USED\n".format(mem1-mem0)) flinear.write(" - MEM:{:.3f} MB USED\n".format(mem1-mem0))
svxfileslist = svx_scan.svxfileslist svxfileslist = svx_scan.svxfileslist
flinear.write(" - {:,} survex files in linear include list \n".format(len(svxfileslist))) flinear.write(" - {:,} survex files in linear include list \n".format(len(svxfileslist)))
@@ -791,7 +809,7 @@ def FindAndLoadSurvex(survexblockroot):
svx_load.LinearRecursiveLoad(survexblockroot,survexfileroot.path, svxlines) svx_load.LinearRecursiveLoad(survexblockroot,survexfileroot.path, svxlines)
#---------------------------------------------------------------- #----------------------------------------------------------------
print(" - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr) print("\n - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr)
print(" - MEM:{:7.3f} MB USED".format(mem1-mem0),file=sys.stderr) print(" - MEM:{:7.3f} MB USED".format(mem1-mem0),file=sys.stderr)
survexlegsnumber = svx_load.survexlegsnumber survexlegsnumber = svx_load.survexlegsnumber
@@ -799,7 +817,7 @@ def FindAndLoadSurvex(survexblockroot):
mem1 = models.get_process_memory() mem1 = models.get_process_memory()
svx_load = None svx_load = None
print('\n - Loading All Survex Blocks (RecursiveRecursive)',file=sys.stderr) # print('\n - Loading All Survex Blocks (RecursiveRecursive)',file=sys.stderr)
# svxlrl = LoadingSurvex() # svxlrl = LoadingSurvex()
# finroot = survexfileroot.OpenFile() # finroot = survexfileroot.OpenFile()