expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-12-14 21:37:13 +00:00
Code Issues 18 Projects 3 Activity

Understanding and speeding up LoadPos

Browse Source
This commit is contained in:
Philip Sargent
2020-04-28 18:26:08 +01:00
parent 4be8c81291
commit b4c0c4d219
3 changed files with 80 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
parsers/survex.py
View File

@@ -232,7 +232,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
if cave:
survexfile.cave = cave
else:
print('No match for %s' % includepath)
print(' - No match (i) for %s' % includepath)
includesurvexfile = models.SurvexFile(path=includepath)
includesurvexfile.save()
includesurvexfile.SetDirectory()
@@ -253,7 +253,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
if cave:
survexfile.cave = cave
else:
print('No match for %s' % newsvxpath)
print(' - No match (b) for %s' % newsvxpath)
name = line.lower()
print(' - Begin found for: ' + name)
@@ -391,21 +391,71 @@ def LoadPos():
and if we do, then save the x/y/z coordinates.
If we don't have it in the database, print an error message and discard it.
"""
print(' - Generating a list of Pos and then loading them....')
topdata = settings.SURVEX_DATA + settings.SURVEX_TOPNAME
print(' - Generating a list of Pos from %s.svx and then loading...' % (topdata))
# Be careful with the cache file.
# If LoadPos has been run before,
# but without cave import being run before,
# then *everything* may be in the fresh 'not found' cache file.
cachefile = settings.SURVEX_DATA + "posnotfound"
notfoundbefore = {}
if os.path.isfile(cachefile):
updtsvx = os.path.getmtime(topdata + ".svx")
updtcache = os.path.getmtime(cachefile)
age = updtcache - updtsvx
print(' svx: %s cache: %s cache age: %s' % (updtsvx, updtcache, age ))
if age < 0 :
print " cache is stale."
os.remove(cachefile)
else:
print " cache is fresh."
try:
f = open(cachefile, "r")
for line in f:
notfoundbefore[line] +=1 # should not be duplicates
except:
print " FAILURE READ opening cache file %s" % (cachefile)
f.close()
call([settings.CAVERN, "--output=%s%s.3d" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME), "%s%s.svx" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME)])
call([settings.THREEDTOPOS, '%s%s.3d' % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME)], cwd = settings.SURVEX_DATA)
posfile = open("%s%s.pos" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME))
notfoundnow =[]
found = 0
skip = {}
print "\n" # extra line because cavern overwrites the text buffer somehow
# cavern defaults to using same cwd as supplied input file
call([settings.CAVERN, "--output=%s.3d" % (topdata), "%s.svx" % (topdata)])
call([settings.THREEDTOPOS, '%s.3d' % (topdata)], cwd = settings.SURVEX_DATA)
posfile = open("%s.pos" % (topdata))
posfile.readline() #Drop header
for line in posfile.readlines():
r = poslineregex.match(line)
if r:
x, y, z, name = r.groups()
try:
ss = models.SurvexStation.objects.lookup(name)
ss.x = float(x)
ss.y = float(y)
ss.z = float(z)
ss.save()
except:
print "%s in %s.pos not found in lookup of SurvexStation.objects" % (name, settings.SURVEX_TOPNAME)
x, y, z, name = r.groups() # easting, northing, altitude
if name in notfoundbefore:
skip[name] += 1
else:
try:
ss = models.SurvexStation.objects.lookup(name)
ss.x = float(x)
ss.y = float(y)
ss.z = float(z)
ss.save()
found += 1
except:
#print "%s in %s.pos not found in lookup of SurvexStation.objects" % (name, settings.SURVEX_TOPNAME)
notfoundnow.append(name)
print " - %s stations NOT found in lookup of SurvexStation.objects. %s found. %s skipper." % (len(notfoundnow),found, skip)
if found > 10: # i.e. a previous cave import has been done
try:
with open(cachefile, "w") as f:
print " cache file opened"
for i in notfoundnow:
f.write("%s\n" % i)
for j in skip:
f.write("%s\n" % j) # NB skip not notfoundbefore
except:
print " FAILURE WRITE opening cache file %s" % (cachefile)