expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-25 08:41:51 +00:00
Code Issues 19 Projects 3 Activity

clean up survexlegs

Browse Source
This commit is contained in:
Philip Sargent 2020-06-16 19:27:32 +01:00
parent 8fc0ba136f
commit 94e5a06a15
6 changed files with 134 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
core/models_survex.py
View File

@ -81,14 +81,18 @@ class SurvexStation(models.Model):
else: else:
return r return r
class SurvexLeg(models.Model): # class SurvexLeg(models.Model):
block = models.ForeignKey('SurvexBlock') # block = models.ForeignKey('SurvexBlock')
#title = models.ForeignKey('SurvexTitle') # #title = models.ForeignKey('SurvexTitle')
stationfrom = models.ForeignKey('SurvexStation', related_name='stationfrom') # stationfrom = models.ForeignKey('SurvexStation', related_name='stationfrom')
stationto = models.ForeignKey('SurvexStation', related_name='stationto') # stationto = models.ForeignKey('SurvexStation', related_name='stationto')
tape = models.FloatField() # tape = models.FloatField()
compass = models.FloatField() # compass = models.FloatField()
clino = models.FloatField() # clino = models.FloatField()
class SurvexLeg():
tape = 0.0
compass = 0.0
clino = 0.0
# #
# Single SurvexBlock # Single SurvexBlock
@ -108,7 +112,7 @@ class SurvexBlock(models.Model):
objects = SurvexBlockLookUpManager() objects = SurvexBlockLookUpManager()
name = models.CharField(max_length=100) name = models.CharField(max_length=100)
parent = models.ForeignKey('SurvexBlock', blank=True, null=True) parent = models.ForeignKey('SurvexBlock', blank=True, null=True)
text = models.TextField() # text = models.TextField()
cave = models.ForeignKey('Cave', blank=True, null=True) cave = models.ForeignKey('Cave', blank=True, null=True)
date = models.DateField(blank=True, null=True) date = models.DateField(blank=True, null=True)
@ -116,13 +120,16 @@ class SurvexBlock(models.Model):
expedition = models.ForeignKey('Expedition', blank=True, null=True) expedition = models.ForeignKey('Expedition', blank=True, null=True)
survexfile = models.ForeignKey("SurvexFile", blank=True, null=True) survexfile = models.ForeignKey("SurvexFile", blank=True, null=True)
begin_char = models.IntegerField() # code for where in the survex data files this block sits # begin_char = models.IntegerField() # code for where in the survex data files this block sits
survexpath = models.CharField(max_length=200) # the path for the survex stations survexpath = models.CharField(max_length=200) # the path for the survex stations
survexscansfolder = models.ForeignKey("SurvexScansFolder", null=True) survexscansfolder = models.ForeignKey("SurvexScansFolder", null=True)
#refscandir = models.CharField(max_length=100) #refscandir = models.CharField(max_length=100)
totalleglength = models.FloatField() legsall = models.IntegerField(null=True) # summary data for this block
legssplay = models.IntegerField(null=True) # summary data for this block
legssurfc = models.IntegerField(null=True) # summary data for this block
totalleglength = models.FloatField(null=True)
class Meta: class Meta:
ordering = ('id',) ordering = ('id',)

2
core/views_other.py
View File

@ -11,7 +11,7 @@ from django.template import Context, loader
import troggle.parsers.imports import troggle.parsers.imports
from troggle.core.models import Expedition, Person, PersonExpedition from troggle.core.models import Expedition, Person, PersonExpedition
from troggle.core.models_caves import LogbookEntry, QM, Cave, PersonTrip from troggle.core.models_caves import LogbookEntry, QM, Cave, PersonTrip
from troggle.core.models_survex import SurvexLeg #from troggle.core.models_survex import SurvexLeg
from troggle.helper import login_required_if_public from troggle.helper import login_required_if_public
from troggle.core.forms import UploadFileForm from troggle.core.forms import UploadFileForm

29
core/views_statistics.py
View File

@ -82,19 +82,24 @@ def stats(request):
statsDict['caveCount'] = "{:,}".format(Cave.objects.count()) statsDict['caveCount'] = "{:,}".format(Cave.objects.count())
statsDict['personCount'] = "{:,}".format(Person.objects.count()) statsDict['personCount'] = "{:,}".format(Person.objects.count())
statsDict['logbookEntryCount'] = "{:,}".format(LogbookEntry.objects.count()) statsDict['logbookEntryCount'] = "{:,}".format(LogbookEntry.objects.count())
blockroots = SurvexBlock.objects.filter(name="rootblock")
if len(blockroots)>1:
print(" ! more than one root survexblock {}".format(len(blockroots)))
for sbr in blockroots:
print("{} {} {} {}".format(sbr.id, sbr.name, sbr.text, sbr.date))
sbr = blockroots[0]
totalsurvexlength = sbr.totalleglength
try: try:
nimportlegs = int(sbr.text) blockroots = SurvexBlock.objects.filter(name="rootblock")
if len(blockroots)>1:
print(" ! more than one root survexblock {}".format(len(blockroots)))
for sbr in blockroots:
print("{} {} {} {}".format(sbr.id, sbr.name, sbr.legsall, sbr.date))
sbr = blockroots[0]
totalsurvexlength = sbr.totalleglength
nimportlegs = sbr.legsall
except: except:
print("{} {} {} {}".format(sbr.id, sbr.name, sbr.text, sbr.date)) # if no files yet imported into database
#survexfile = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
#survexblockdummy = models_survex.SurvexBlock(name="dummy", survexpath="", cave=None, survexfile=survexfile,
#legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
#sbr = survexblockdummy
totalsurvexlength = 0.0
nimportlegs = -1 nimportlegs = -1
print("{} {} {} {}".format(sbr.id, sbr.name, sbr.legsall, sbr.date))
legsbyexpo = [ ] legsbyexpo = [ ]
addupsurvexlength = 0 addupsurvexlength = 0
@ -105,14 +110,14 @@ def stats(request):
for survexblock in survexblocks: for survexblock in survexblocks:
survexleglength += survexblock.totalleglength survexleglength += survexblock.totalleglength
try: try:
legsyear += int(survexblock.text) legsyear += int(survexblock.legsall)
except: except:
pass pass
addupsurvexlength += survexleglength addupsurvexlength += survexleglength
legsbyexpo.append((expedition, {"nsurvexlegs": "{:,}".format(legsyear), legsbyexpo.append((expedition, {"nsurvexlegs": "{:,}".format(legsyear),
"survexleglength":"{:,.0f}".format(survexleglength)})) "survexleglength":"{:,.0f}".format(survexleglength)}))
legsbyexpo.reverse() legsbyexpo.reverse()
survexlegs = SurvexLeg.objects.all() #survexlegs = SurvexLeg.objects.all()
renderDict = {**statsDict, **{ "nsurvexlegs": "{:,}".format(nimportlegs), "totalsurvexlength":totalsurvexlength/1000, "addupsurvexlength":addupsurvexlength/1000, "legsbyexpo":legsbyexpo }} # new syntax renderDict = {**statsDict, **{ "nsurvexlegs": "{:,}".format(nimportlegs), "totalsurvexlength":totalsurvexlength/1000, "addupsurvexlength":addupsurvexlength/1000, "legsbyexpo":legsbyexpo }} # new syntax
return render(request,'statistics.html', renderDict) return render(request,'statistics.html', renderDict)

7
databaseReset.py
View File

@ -22,7 +22,7 @@ import troggle.flatpages.models
import troggle.logbooksdump import troggle.logbooksdump
from troggle.parsers.imports import import_caves, import_people, import_surveyscans, \ from troggle.parsers.imports import import_caves, import_people, import_surveyscans, \
import_logbooks, import_QMs, import_survex, import_survexpos, import_tunnelfiles import_logbooks, import_QMs, import_survex, import_tunnelfiles
import troggle.core.models import troggle.core.models
import troggle.core.models_survex import troggle.core.models_survex
@ -389,7 +389,6 @@ def usage():
autologbooks - Not used. read in autologbooks (what are these?) autologbooks - Not used. read in autologbooks (what are these?)
dumplogbooks - Not used. write out autologbooks (not working?) dumplogbooks - Not used. write out autologbooks (not working?)
surveyimgs - Not used. read in scans by-expo, must run after "people".
and [runlabel] is an optional string identifying this run of the script and [runlabel] is an optional string identifying this run of the script
in the stored profiling data 'import-profile.json' in the stored profiling data 'import-profile.json'
@ -442,16 +441,12 @@ if __name__ == "__main__":
jq.enq("QMs",import_QMs) jq.enq("QMs",import_QMs)
jq.enq("tunnel",import_tunnelfiles) jq.enq("tunnel",import_tunnelfiles)
jq.enq("survex",import_survex) jq.enq("survex",import_survex)
jq.enq("survexpos",import_survexpos)
elif "scans" in sys.argv: elif "scans" in sys.argv:
jq.enq("scans",import_surveyscans) jq.enq("scans",import_surveyscans)
elif "survex" in sys.argv: elif "survex" in sys.argv:
jq.enq("survex",import_survex) jq.enq("survex",import_survex)
jq.enq("survexpos",import_survexpos)
elif "tunnel" in sys.argv: elif "tunnel" in sys.argv:
jq.enq("tunnel",import_tunnelfiles) jq.enq("tunnel",import_tunnelfiles)
elif "surveyimgs" in sys.argv:
jq.enq("surveyimgs",import_surveyimgs) # imports into tables which are never read
elif "autologbooks" in sys.argv: # untested in 2020 elif "autologbooks" in sys.argv: # untested in 2020
import_auto_logbooks() import_auto_logbooks()
elif "dumplogbooks" in sys.argv: # untested in 2020 elif "dumplogbooks" in sys.argv: # untested in 2020

6
parsers/imports.py
View File

@ -41,11 +41,9 @@ def import_survex():
# when this import is moved to the top with the rest it all crashes horribly # when this import is moved to the top with the rest it all crashes horribly
import troggle.parsers.survex import troggle.parsers.survex
print("Importing Survex Blocks") print("Importing Survex Blocks")
print(" - Survex Blocks")
troggle.parsers.survex.LoadAllSurvexBlocks() troggle.parsers.survex.LoadAllSurvexBlocks()
print(" - Survex entrances x/y/z Positions")
def import_survexpos():
import troggle.parsers.survex
print("Importing Survex x/y/z Positions")
troggle.parsers.survex.LoadPos() troggle.parsers.survex.LoadPos()
def import_tunnelfiles(): def import_tunnelfiles():

185
parsers/survex.py
View File

@ -31,10 +31,11 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
global survexlegsnumber global survexlegsnumber
# The try catches here need replacing as they are relatively expensive # The try catches here need replacing as they are relatively expensive
ls = sline.lower().split() ls = sline.lower().split()
ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]]) #ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]])
ssto = survexblock.MakeSurvexStation(ls[stardata["to"]]) #ssto = survexblock.MakeSurvexStation(ls[stardata["to"]])
survexleg = models_survex.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto) # survexleg = models_survex.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto)
survexleg = models_survex.SurvexLeg()
# this next fails for two surface survey svx files which use / for decimal point # this next fails for two surface survey svx files which use / for decimal point
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05) # e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
if stardata["type"] == "normal": if stardata["type"] == "normal":
@ -326,7 +327,9 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
previousnlegs = survexlegsnumber previousnlegs = survexlegsnumber
name = line.lower() name = line.lower()
print((insp+' - Begin found for: ' + name)) print((insp+' - Begin found for: ' + name))
survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0) # survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0)
survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock, survexpath=survexblock.survexpath+"."+name,
cave=survexfile.cave, survexfile=survexfile, legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
survexblockdown.save() survexblockdown.save()
survexblock.save() survexblock.save()
survexblock = survexblockdown survexblock = survexblockdown
@ -344,7 +347,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
# .text not used, using it for number of legs per block # .text not used, using it for number of legs per block
legsinblock = survexlegsnumber - previousnlegs legsinblock = survexlegsnumber - previousnlegs
print(insp+"LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber)) print(insp+"LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber))
survexblock.text = str(legsinblock) survexblock.legsall = legsinblock
survexblock.save() survexblock.save()
endstamp = datetime.now() endstamp = datetime.now()
timetaken = endstamp - stamp timetaken = endstamp - stamp
@ -436,7 +439,7 @@ def LoadAllSurvexBlocks():
models_survex.SurvexFile.objects.all().delete() models_survex.SurvexFile.objects.all().delete()
models_survex.SurvexDirectory.objects.all().delete() models_survex.SurvexDirectory.objects.all().delete()
models_survex.SurvexEquate.objects.all().delete() models_survex.SurvexEquate.objects.all().delete()
models_survex.SurvexLeg.objects.all().delete() #models_survex.SurvexLeg.objects.all().delete()
models_survex.SurvexTitle.objects.all().delete() models_survex.SurvexTitle.objects.all().delete()
models_survex.SurvexPersonRole.objects.all().delete() models_survex.SurvexPersonRole.objects.all().delete()
models_survex.SurvexStation.objects.all().delete() models_survex.SurvexStation.objects.all().delete()
@ -457,7 +460,8 @@ def LoadAllSurvexBlocks():
#Load all #Load all
# this is the first so id=1 # this is the first so id=1
survexblockroot = models_survex.SurvexBlock(name="rootblock", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0) survexblockroot = models_survex.SurvexBlock(name="rootblock", survexpath="", cave=None, survexfile=survexfile,
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
survexblockroot.save() survexblockroot.save()
fin = survexfile.OpenFile() fin = survexfile.OpenFile()
textlines = [ ] textlines = [ ]
@ -465,7 +469,7 @@ def LoadAllSurvexBlocks():
RecursiveLoad(survexblockroot, survexfile, fin, textlines) RecursiveLoad(survexblockroot, survexfile, fin, textlines)
fin.close() fin.close()
survexblockroot.totalleglength = survexlegsalllength survexblockroot.totalleglength = survexlegsalllength
survexblockroot.text = str(survexlegsnumber) survexblockroot.legsall = survexlegsnumber
#survexblockroot.text = "".join(textlines) these are all blank #survexblockroot.text = "".join(textlines) these are all blank
survexblockroot.save() survexblockroot.save()
@ -501,41 +505,41 @@ def LoadPos():
# but without cave import being run before, # but without cave import being run before,
# then *everything* may be in the fresh 'not found' cache file. # then *everything* may be in the fresh 'not found' cache file.
cachefile = settings.SURVEX_DATA + "posnotfound.cache" # cachefile = settings.SURVEX_DATA + "posnotfound.cache"
notfoundbefore = {} # notfoundbefore = {}
if os.path.isfile(cachefile): # if os.path.isfile(cachefile):
# this is not a good test. 1623.svx may never change but *included files may have done. # # this is not a good test. 1623.svx may never change but *included files may have done.
# When the *include is unrolled, we will be able to get a proper timestamp to use # # When the *include is unrolled, we will be able to get a proper timestamp to use
# and can increase the timeout from 3 days to 30 days. # # and can increase the timeout from 3 days to 30 days.
updtsvx = os.path.getmtime(topdata + ".svx") # updtsvx = os.path.getmtime(topdata + ".svx")
updtcache = os.path.getmtime(cachefile) # updtcache = os.path.getmtime(cachefile)
age = updtcache - updtsvx # age = updtcache - updtsvx
print((' svx: %s cache: %s not-found cache is fresher by: %s' % (updtsvx, updtcache, str(timedelta(seconds=age) )))) # print((' svx: %s cache: %s not-found cache is fresher by: %s' % (updtsvx, updtcache, str(timedelta(seconds=age) ))))
now = time.time() # now = time.time()
if now - updtcache > 3*24*60*60: # if now - updtcache > 3*24*60*60:
print(" cache is more than 3 days old. Deleting.") # print(" cache is more than 3 days old. Deleting.")
os.remove(cachefile) # os.remove(cachefile)
elif age < 0 : # elif age < 0 :
print(" cache is stale. Deleting.") # print(" cache is stale. Deleting.")
os.remove(cachefile) # os.remove(cachefile)
else: # else:
print(" cache is fresh. Reading...") # print(" cache is fresh. Reading...")
try: # try:
with open(cachefile, "r") as f: # with open(cachefile, "r") as f:
for line in f: # for line in f:
l = line.rstrip() # l = line.rstrip()
if l in notfoundbefore: # if l in notfoundbefore:
notfoundbefore[l] +=1 # should not be duplicates # notfoundbefore[l] +=1 # should not be duplicates
print(" DUPLICATE ", line, notfoundbefore[l]) # print(" DUPLICATE ", line, notfoundbefore[l])
else: # else:
notfoundbefore[l] =1 # notfoundbefore[l] =1
except: # except:
print(" FAILURE READ opening cache file %s" % (cachefile)) # print(" FAILURE READ opening cache file %s" % (cachefile))
raise # raise
notfoundnow =[] # notfoundnow =[]
found = 0 found = 0
skip = {} skip = {}
print("\n") # extra line because cavern overwrites the text buffer somehow print("\n") # extra line because cavern overwrites the text buffer somehow
@ -557,60 +561,61 @@ def LoadPos():
r = poslineregex.match(line) r = poslineregex.match(line)
if r: if r:
x, y, z, id = r.groups() x, y, z, id = r.groups()
if id in notfoundbefore: # if id in notfoundbefore:
skip[id] = 1 # skip[id] = 1
else: # else:
for sid in mappoints: for sid in mappoints:
if id.endswith(sid): if id.endswith(sid):
notfoundnow.append(id) # notfoundnow.append(id)
# Now that we don't import any stations, we create it rather than look it up # Now that we don't import any stations, we create it rather than look it up
# ss = models_survex.SurvexStation.objects.lookup(id) # ss = models_survex.SurvexStation.objects.lookup(id)
# need to set block_id which means doing a search on all the survex blocks.. # need to set block_id which means doing a search on all the survex blocks..
# remove dot at end and add one at beginning # remove dot at end and add one at beginning
blockpath = "." + id[:-len(sid)].strip(".") blockpath = "." + id[:-len(sid)].strip(".")
try: try:
sbqs = models_survex.SurvexBlock.objects.filter(survexpath=blockpath) sbqs = models_survex.SurvexBlock.objects.filter(survexpath=blockpath)
if len(sbqs)==1: if len(sbqs)==1:
sb = sbqs[0] sb = sbqs[0]
if len(sbqs)>1: if len(sbqs)>1:
message = ' ! MULTIPLE SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid) message = ' ! MULTIPLE SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
print(message)
models.DataIssue.objects.create(parser='survex', message=message)
sb = sbqs[0]
elif len(sbqs)<=0:
message = ' ! ZERO SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
print(message)
models.DataIssue.objects.create(parser='survex', message=message)
sb = survexblockroot
except:
message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid)
print(message) print(message)
models.DataIssue.objects.create(parser='survex', message=message) models.DataIssue.objects.create(parser='survex', message=message)
try: sb = sbqs[0]
ss = models_survex.SurvexStation(name=id, block=sb) elif len(sbqs)<=0:
ss.x = float(x) message = ' ! ZERO SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
ss.y = float(y)
ss.z = float(z)
ss.save()
found += 1
except:
message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid)
print(message) print(message)
models.DataIssue.objects.create(parser='survex', message=message) models.DataIssue.objects.create(parser='survex', message=message)
raise sb = survexblockroot
except:
message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid)
print(message)
models.DataIssue.objects.create(parser='survex', message=message)
try:
ss = models_survex.SurvexStation(name=id, block=sb)
ss.x = float(x)
ss.y = float(y)
ss.z = float(z)
ss.save()
found += 1
except:
message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid)
print(message)
models.DataIssue.objects.create(parser='survex', message=message)
raise
#print(" - %s failed lookups of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip))) #print(" - %s failed lookups of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip)))
print(" - {} SurvexStation entrances found.".format(found))
if found > 10: # i.e. a previous cave import has been done # if found > 10: # i.e. a previous cave import has been done
try: # try:
with open(cachefile, "w") as f: # with open(cachefile, "w") as f:
c = len(notfoundnow)+len(skip) # c = len(notfoundnow)+len(skip)
for i in notfoundnow: # for i in notfoundnow:
pass #f.write("%s\n" % i) # pass #f.write("%s\n" % i)
for j in skip: # for j in skip:
pass #f.write("%s\n" % j) # NB skip not notfoundbefore # pass #f.write("%s\n" % j) # NB skip not notfoundbefore
print((' Not-found cache file written: %s entries' % c)) # print((' Not-found cache file written: %s entries' % c))
except: # except:
print(" FAILURE WRITE opening cache file %s" % (cachefile)) # print(" FAILURE WRITE opening cache file %s" % (cachefile))
raise # raise