mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2024-11-21 23:01:52 +00:00
clean up survexlegs
This commit is contained in:
Philip Sargent
parent
8fc0ba136f
commit
94e5a06a15
@ -81,14 +81,18 @@ class SurvexStation(models.Model):
|
||||
else:
|
||||
return r
|
||||
|
||||
class SurvexLeg(models.Model):
|
||||
block = models.ForeignKey('SurvexBlock')
|
||||
#title = models.ForeignKey('SurvexTitle')
|
||||
stationfrom = models.ForeignKey('SurvexStation', related_name='stationfrom')
|
||||
stationto = models.ForeignKey('SurvexStation', related_name='stationto')
|
||||
tape = models.FloatField()
|
||||
compass = models.FloatField()
|
||||
clino = models.FloatField()
|
||||
# class SurvexLeg(models.Model):
|
||||
# block = models.ForeignKey('SurvexBlock')
|
||||
# #title = models.ForeignKey('SurvexTitle')
|
||||
# stationfrom = models.ForeignKey('SurvexStation', related_name='stationfrom')
|
||||
# stationto = models.ForeignKey('SurvexStation', related_name='stationto')
|
||||
# tape = models.FloatField()
|
||||
# compass = models.FloatField()
|
||||
# clino = models.FloatField()
|
||||
class SurvexLeg():
|
||||
tape = 0.0
|
||||
compass = 0.0
|
||||
clino = 0.0
|
||||
|
||||
#
|
||||
# Single SurvexBlock
|
||||
@ -108,7 +112,7 @@ class SurvexBlock(models.Model):
|
||||
objects = SurvexBlockLookUpManager()
|
||||
name = models.CharField(max_length=100)
|
||||
parent = models.ForeignKey('SurvexBlock', blank=True, null=True)
|
||||
text = models.TextField()
|
||||
# text = models.TextField()
|
||||
cave = models.ForeignKey('Cave', blank=True, null=True)
|
||||
|
||||
date = models.DateField(blank=True, null=True)
|
||||
@ -116,13 +120,16 @@ class SurvexBlock(models.Model):
|
||||
expedition = models.ForeignKey('Expedition', blank=True, null=True)
|
||||
|
||||
survexfile = models.ForeignKey("SurvexFile", blank=True, null=True)
|
||||
begin_char = models.IntegerField() # code for where in the survex data files this block sits
|
||||
# begin_char = models.IntegerField() # code for where in the survex data files this block sits
|
||||
survexpath = models.CharField(max_length=200) # the path for the survex stations
|
||||
|
||||
survexscansfolder = models.ForeignKey("SurvexScansFolder", null=True)
|
||||
#refscandir = models.CharField(max_length=100)
|
||||
|
||||
totalleglength = models.FloatField()
|
||||
legsall = models.IntegerField(null=True) # summary data for this block
|
||||
legssplay = models.IntegerField(null=True) # summary data for this block
|
||||
legssurfc = models.IntegerField(null=True) # summary data for this block
|
||||
totalleglength = models.FloatField(null=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ('id',)
|
||||
|
||||
@ -11,7 +11,7 @@ from django.template import Context, loader
|
||||
import troggle.parsers.imports
|
||||
from troggle.core.models import Expedition, Person, PersonExpedition
|
||||
from troggle.core.models_caves import LogbookEntry, QM, Cave, PersonTrip
|
||||
from troggle.core.models_survex import SurvexLeg
|
||||
#from troggle.core.models_survex import SurvexLeg
|
||||
from troggle.helper import login_required_if_public
|
||||
from troggle.core.forms import UploadFileForm
|
||||
|
||||
|
||||
@ -82,19 +82,24 @@ def stats(request):
|
||||
statsDict['caveCount'] = "{:,}".format(Cave.objects.count())
|
||||
statsDict['personCount'] = "{:,}".format(Person.objects.count())
|
||||
statsDict['logbookEntryCount'] = "{:,}".format(LogbookEntry.objects.count())
|
||||
|
||||
blockroots = SurvexBlock.objects.filter(name="rootblock")
|
||||
if len(blockroots)>1:
|
||||
print(" ! more than one root survexblock {}".format(len(blockroots)))
|
||||
for sbr in blockroots:
|
||||
print("{} {} {} {}".format(sbr.id, sbr.name, sbr.text, sbr.date))
|
||||
sbr = blockroots[0]
|
||||
totalsurvexlength = sbr.totalleglength
|
||||
try:
|
||||
nimportlegs = int(sbr.text)
|
||||
blockroots = SurvexBlock.objects.filter(name="rootblock")
|
||||
if len(blockroots)>1:
|
||||
print(" ! more than one root survexblock {}".format(len(blockroots)))
|
||||
for sbr in blockroots:
|
||||
print("{} {} {} {}".format(sbr.id, sbr.name, sbr.legsall, sbr.date))
|
||||
sbr = blockroots[0]
|
||||
totalsurvexlength = sbr.totalleglength
|
||||
nimportlegs = sbr.legsall
|
||||
except:
|
||||
print("{} {} {} {}".format(sbr.id, sbr.name, sbr.text, sbr.date))
|
||||
# if no files yet imported into database
|
||||
#survexfile = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
|
||||
#survexblockdummy = models_survex.SurvexBlock(name="dummy", survexpath="", cave=None, survexfile=survexfile,
|
||||
#legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
|
||||
#sbr = survexblockdummy
|
||||
totalsurvexlength = 0.0
|
||||
nimportlegs = -1
|
||||
print("{} {} {} {}".format(sbr.id, sbr.name, sbr.legsall, sbr.date))
|
||||
|
||||
legsbyexpo = [ ]
|
||||
addupsurvexlength = 0
|
||||
@ -105,14 +110,14 @@ def stats(request):
|
||||
for survexblock in survexblocks:
|
||||
survexleglength += survexblock.totalleglength
|
||||
try:
|
||||
legsyear += int(survexblock.text)
|
||||
legsyear += int(survexblock.legsall)
|
||||
except:
|
||||
pass
|
||||
addupsurvexlength += survexleglength
|
||||
legsbyexpo.append((expedition, {"nsurvexlegs": "{:,}".format(legsyear),
|
||||
"survexleglength":"{:,.0f}".format(survexleglength)}))
|
||||
legsbyexpo.reverse()
|
||||
survexlegs = SurvexLeg.objects.all()
|
||||
#survexlegs = SurvexLeg.objects.all()
|
||||
|
||||
renderDict = {**statsDict, **{ "nsurvexlegs": "{:,}".format(nimportlegs), "totalsurvexlength":totalsurvexlength/1000, "addupsurvexlength":addupsurvexlength/1000, "legsbyexpo":legsbyexpo }} # new syntax
|
||||
return render(request,'statistics.html', renderDict)
|
||||
|
||||
@ -22,7 +22,7 @@ import troggle.flatpages.models
|
||||
import troggle.logbooksdump
|
||||
|
||||
from troggle.parsers.imports import import_caves, import_people, import_surveyscans, \
|
||||
import_logbooks, import_QMs, import_survex, import_survexpos, import_tunnelfiles
|
||||
import_logbooks, import_QMs, import_survex, import_tunnelfiles
|
||||
|
||||
import troggle.core.models
|
||||
import troggle.core.models_survex
|
||||
@ -389,7 +389,6 @@ def usage():
|
||||
|
||||
autologbooks - Not used. read in autologbooks (what are these?)
|
||||
dumplogbooks - Not used. write out autologbooks (not working?)
|
||||
surveyimgs - Not used. read in scans by-expo, must run after "people".
|
||||
|
||||
and [runlabel] is an optional string identifying this run of the script
|
||||
in the stored profiling data 'import-profile.json'
|
||||
@ -442,16 +441,12 @@ if __name__ == "__main__":
|
||||
jq.enq("QMs",import_QMs)
|
||||
jq.enq("tunnel",import_tunnelfiles)
|
||||
jq.enq("survex",import_survex)
|
||||
jq.enq("survexpos",import_survexpos)
|
||||
elif "scans" in sys.argv:
|
||||
jq.enq("scans",import_surveyscans)
|
||||
elif "survex" in sys.argv:
|
||||
jq.enq("survex",import_survex)
|
||||
jq.enq("survexpos",import_survexpos)
|
||||
elif "tunnel" in sys.argv:
|
||||
jq.enq("tunnel",import_tunnelfiles)
|
||||
elif "surveyimgs" in sys.argv:
|
||||
jq.enq("surveyimgs",import_surveyimgs) # imports into tables which are never read
|
||||
elif "autologbooks" in sys.argv: # untested in 2020
|
||||
import_auto_logbooks()
|
||||
elif "dumplogbooks" in sys.argv: # untested in 2020
|
||||
|
||||
@ -41,11 +41,9 @@ def import_survex():
|
||||
# when this import is moved to the top with the rest it all crashes horribly
|
||||
import troggle.parsers.survex
|
||||
print("Importing Survex Blocks")
|
||||
print(" - Survex Blocks")
|
||||
troggle.parsers.survex.LoadAllSurvexBlocks()
|
||||
|
||||
def import_survexpos():
|
||||
import troggle.parsers.survex
|
||||
print("Importing Survex x/y/z Positions")
|
||||
print(" - Survex entrances x/y/z Positions")
|
||||
troggle.parsers.survex.LoadPos()
|
||||
|
||||
def import_tunnelfiles():
|
||||
|
||||
@ -31,10 +31,11 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
|
||||
global survexlegsnumber
|
||||
# The try catches here need replacing as they are relatively expensive
|
||||
ls = sline.lower().split()
|
||||
ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]])
|
||||
ssto = survexblock.MakeSurvexStation(ls[stardata["to"]])
|
||||
#ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]])
|
||||
#ssto = survexblock.MakeSurvexStation(ls[stardata["to"]])
|
||||
|
||||
survexleg = models_survex.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto)
|
||||
# survexleg = models_survex.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto)
|
||||
survexleg = models_survex.SurvexLeg()
|
||||
# this next fails for two surface survey svx files which use / for decimal point
|
||||
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
|
||||
if stardata["type"] == "normal":
|
||||
@ -326,7 +327,9 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
|
||||
previousnlegs = survexlegsnumber
|
||||
name = line.lower()
|
||||
print((insp+' - Begin found for: ' + name))
|
||||
survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0)
|
||||
# survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0)
|
||||
survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock, survexpath=survexblock.survexpath+"."+name,
|
||||
cave=survexfile.cave, survexfile=survexfile, legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
|
||||
survexblockdown.save()
|
||||
survexblock.save()
|
||||
survexblock = survexblockdown
|
||||
@ -344,7 +347,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
|
||||
# .text not used, using it for number of legs per block
|
||||
legsinblock = survexlegsnumber - previousnlegs
|
||||
print(insp+"LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber))
|
||||
survexblock.text = str(legsinblock)
|
||||
survexblock.legsall = legsinblock
|
||||
survexblock.save()
|
||||
endstamp = datetime.now()
|
||||
timetaken = endstamp - stamp
|
||||
@ -436,7 +439,7 @@ def LoadAllSurvexBlocks():
|
||||
models_survex.SurvexFile.objects.all().delete()
|
||||
models_survex.SurvexDirectory.objects.all().delete()
|
||||
models_survex.SurvexEquate.objects.all().delete()
|
||||
models_survex.SurvexLeg.objects.all().delete()
|
||||
#models_survex.SurvexLeg.objects.all().delete()
|
||||
models_survex.SurvexTitle.objects.all().delete()
|
||||
models_survex.SurvexPersonRole.objects.all().delete()
|
||||
models_survex.SurvexStation.objects.all().delete()
|
||||
@ -457,7 +460,8 @@ def LoadAllSurvexBlocks():
|
||||
|
||||
#Load all
|
||||
# this is the first so id=1
|
||||
survexblockroot = models_survex.SurvexBlock(name="rootblock", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0)
|
||||
survexblockroot = models_survex.SurvexBlock(name="rootblock", survexpath="", cave=None, survexfile=survexfile,
|
||||
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
|
||||
survexblockroot.save()
|
||||
fin = survexfile.OpenFile()
|
||||
textlines = [ ]
|
||||
@ -465,7 +469,7 @@ def LoadAllSurvexBlocks():
|
||||
RecursiveLoad(survexblockroot, survexfile, fin, textlines)
|
||||
fin.close()
|
||||
survexblockroot.totalleglength = survexlegsalllength
|
||||
survexblockroot.text = str(survexlegsnumber)
|
||||
survexblockroot.legsall = survexlegsnumber
|
||||
#survexblockroot.text = "".join(textlines) these are all blank
|
||||
survexblockroot.save()
|
||||
|
||||
@ -501,41 +505,41 @@ def LoadPos():
|
||||
# but without cave import being run before,
|
||||
# then *everything* may be in the fresh 'not found' cache file.
|
||||
|
||||
cachefile = settings.SURVEX_DATA + "posnotfound.cache"
|
||||
notfoundbefore = {}
|
||||
if os.path.isfile(cachefile):
|
||||
# this is not a good test. 1623.svx may never change but *included files may have done.
|
||||
# When the *include is unrolled, we will be able to get a proper timestamp to use
|
||||
# and can increase the timeout from 3 days to 30 days.
|
||||
updtsvx = os.path.getmtime(topdata + ".svx")
|
||||
updtcache = os.path.getmtime(cachefile)
|
||||
age = updtcache - updtsvx
|
||||
print((' svx: %s cache: %s not-found cache is fresher by: %s' % (updtsvx, updtcache, str(timedelta(seconds=age) ))))
|
||||
# cachefile = settings.SURVEX_DATA + "posnotfound.cache"
|
||||
# notfoundbefore = {}
|
||||
# if os.path.isfile(cachefile):
|
||||
# # this is not a good test. 1623.svx may never change but *included files may have done.
|
||||
# # When the *include is unrolled, we will be able to get a proper timestamp to use
|
||||
# # and can increase the timeout from 3 days to 30 days.
|
||||
# updtsvx = os.path.getmtime(topdata + ".svx")
|
||||
# updtcache = os.path.getmtime(cachefile)
|
||||
# age = updtcache - updtsvx
|
||||
# print((' svx: %s cache: %s not-found cache is fresher by: %s' % (updtsvx, updtcache, str(timedelta(seconds=age) ))))
|
||||
|
||||
now = time.time()
|
||||
if now - updtcache > 3*24*60*60:
|
||||
print(" cache is more than 3 days old. Deleting.")
|
||||
os.remove(cachefile)
|
||||
elif age < 0 :
|
||||
print(" cache is stale. Deleting.")
|
||||
os.remove(cachefile)
|
||||
else:
|
||||
print(" cache is fresh. Reading...")
|
||||
try:
|
||||
with open(cachefile, "r") as f:
|
||||
for line in f:
|
||||
l = line.rstrip()
|
||||
if l in notfoundbefore:
|
||||
notfoundbefore[l] +=1 # should not be duplicates
|
||||
print(" DUPLICATE ", line, notfoundbefore[l])
|
||||
else:
|
||||
notfoundbefore[l] =1
|
||||
except:
|
||||
print(" FAILURE READ opening cache file %s" % (cachefile))
|
||||
raise
|
||||
# now = time.time()
|
||||
# if now - updtcache > 3*24*60*60:
|
||||
# print(" cache is more than 3 days old. Deleting.")
|
||||
# os.remove(cachefile)
|
||||
# elif age < 0 :
|
||||
# print(" cache is stale. Deleting.")
|
||||
# os.remove(cachefile)
|
||||
# else:
|
||||
# print(" cache is fresh. Reading...")
|
||||
# try:
|
||||
# with open(cachefile, "r") as f:
|
||||
# for line in f:
|
||||
# l = line.rstrip()
|
||||
# if l in notfoundbefore:
|
||||
# notfoundbefore[l] +=1 # should not be duplicates
|
||||
# print(" DUPLICATE ", line, notfoundbefore[l])
|
||||
# else:
|
||||
# notfoundbefore[l] =1
|
||||
# except:
|
||||
# print(" FAILURE READ opening cache file %s" % (cachefile))
|
||||
# raise
|
||||
|
||||
|
||||
notfoundnow =[]
|
||||
# notfoundnow =[]
|
||||
found = 0
|
||||
skip = {}
|
||||
print("\n") # extra line because cavern overwrites the text buffer somehow
|
||||
@ -557,60 +561,61 @@ def LoadPos():
|
||||
r = poslineregex.match(line)
|
||||
if r:
|
||||
x, y, z, id = r.groups()
|
||||
if id in notfoundbefore:
|
||||
skip[id] = 1
|
||||
else:
|
||||
for sid in mappoints:
|
||||
if id.endswith(sid):
|
||||
notfoundnow.append(id)
|
||||
# Now that we don't import any stations, we create it rather than look it up
|
||||
# ss = models_survex.SurvexStation.objects.lookup(id)
|
||||
|
||||
# need to set block_id which means doing a search on all the survex blocks..
|
||||
# remove dot at end and add one at beginning
|
||||
blockpath = "." + id[:-len(sid)].strip(".")
|
||||
try:
|
||||
sbqs = models_survex.SurvexBlock.objects.filter(survexpath=blockpath)
|
||||
if len(sbqs)==1:
|
||||
sb = sbqs[0]
|
||||
if len(sbqs)>1:
|
||||
message = ' ! MULTIPLE SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
|
||||
print(message)
|
||||
models.DataIssue.objects.create(parser='survex', message=message)
|
||||
sb = sbqs[0]
|
||||
elif len(sbqs)<=0:
|
||||
message = ' ! ZERO SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
|
||||
print(message)
|
||||
models.DataIssue.objects.create(parser='survex', message=message)
|
||||
sb = survexblockroot
|
||||
except:
|
||||
message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid)
|
||||
# if id in notfoundbefore:
|
||||
# skip[id] = 1
|
||||
# else:
|
||||
for sid in mappoints:
|
||||
if id.endswith(sid):
|
||||
# notfoundnow.append(id)
|
||||
# Now that we don't import any stations, we create it rather than look it up
|
||||
# ss = models_survex.SurvexStation.objects.lookup(id)
|
||||
|
||||
# need to set block_id which means doing a search on all the survex blocks..
|
||||
# remove dot at end and add one at beginning
|
||||
blockpath = "." + id[:-len(sid)].strip(".")
|
||||
try:
|
||||
sbqs = models_survex.SurvexBlock.objects.filter(survexpath=blockpath)
|
||||
if len(sbqs)==1:
|
||||
sb = sbqs[0]
|
||||
if len(sbqs)>1:
|
||||
message = ' ! MULTIPLE SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
|
||||
print(message)
|
||||
models.DataIssue.objects.create(parser='survex', message=message)
|
||||
try:
|
||||
ss = models_survex.SurvexStation(name=id, block=sb)
|
||||
ss.x = float(x)
|
||||
ss.y = float(y)
|
||||
ss.z = float(z)
|
||||
ss.save()
|
||||
found += 1
|
||||
except:
|
||||
message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid)
|
||||
sb = sbqs[0]
|
||||
elif len(sbqs)<=0:
|
||||
message = ' ! ZERO SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
|
||||
print(message)
|
||||
models.DataIssue.objects.create(parser='survex', message=message)
|
||||
raise
|
||||
sb = survexblockroot
|
||||
except:
|
||||
message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid)
|
||||
print(message)
|
||||
models.DataIssue.objects.create(parser='survex', message=message)
|
||||
try:
|
||||
ss = models_survex.SurvexStation(name=id, block=sb)
|
||||
ss.x = float(x)
|
||||
ss.y = float(y)
|
||||
ss.z = float(z)
|
||||
ss.save()
|
||||
found += 1
|
||||
except:
|
||||
message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid)
|
||||
print(message)
|
||||
models.DataIssue.objects.create(parser='survex', message=message)
|
||||
raise
|
||||
|
||||
#print(" - %s failed lookups of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip)))
|
||||
print(" - {} SurvexStation entrances found.".format(found))
|
||||
|
||||
if found > 10: # i.e. a previous cave import has been done
|
||||
try:
|
||||
with open(cachefile, "w") as f:
|
||||
c = len(notfoundnow)+len(skip)
|
||||
for i in notfoundnow:
|
||||
pass #f.write("%s\n" % i)
|
||||
for j in skip:
|
||||
pass #f.write("%s\n" % j) # NB skip not notfoundbefore
|
||||
print((' Not-found cache file written: %s entries' % c))
|
||||
except:
|
||||
print(" FAILURE WRITE opening cache file %s" % (cachefile))
|
||||
raise
|
||||
# if found > 10: # i.e. a previous cave import has been done
|
||||
# try:
|
||||
# with open(cachefile, "w") as f:
|
||||
# c = len(notfoundnow)+len(skip)
|
||||
# for i in notfoundnow:
|
||||
# pass #f.write("%s\n" % i)
|
||||
# for j in skip:
|
||||
# pass #f.write("%s\n" % j) # NB skip not notfoundbefore
|
||||
# print((' Not-found cache file written: %s entries' % c))
|
||||
# except:
|
||||
# print(" FAILURE WRITE opening cache file %s" % (cachefile))
|
||||
# raise
|
||||
Loading…
Reference in New Issue
Block a user