mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2024-11-26 01:01:53 +00:00
691 lines
34 KiB
Python
691 lines
34 KiB
Python
import sys
|
|
import os
|
|
import re
|
|
import time
|
|
|
|
from datetime import datetime, timedelta
|
|
from subprocess import call, Popen, PIPE
|
|
|
|
from django.utils.timezone import get_current_timezone
|
|
from django.utils.timezone import make_aware
|
|
|
|
import troggle.settings as settings
|
|
import troggle.core.models as models
|
|
import troggle.core.models_caves as models_caves
|
|
import troggle.core.models_survex as models_survex
|
|
from troggle.parsers.people import GetPersonExpeditionNameLookup
|
|
from troggle.core.views_caves import MapLocations
|
|
|
|
survexblockroot = None
|
|
ROOTBLOCK = "rootblock"
|
|
|
|
class SurvexLeg():
|
|
"""No longer a models.Model subclass, so no longer a database table
|
|
"""
|
|
tape = 0.0
|
|
compass = 0.0
|
|
clino = 0.0
|
|
|
|
class LoadSurvex():
|
|
"""A 'survex block' is a *begin...*end set of cave data.
|
|
A survex file can contain many begin-end blocks, which can be nested, and which can *include
|
|
other survex files.
|
|
A 'scansfolder' is what we today call a "survey scans folder" or a "wallet".
|
|
"""
|
|
|
|
# This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g.
|
|
# *DATA normal from to length gradient bearing ignore ignore ignore ignore
|
|
stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
|
|
stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"}
|
|
|
|
rx_linelen = re.compile(r"[\d\-+.]+$")
|
|
rx_team = re.compile(r"(?i)(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$")
|
|
rx_person = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$")
|
|
rx_qm = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')
|
|
# remember there is also QM_PATTERN used in views_other and set in settings.py
|
|
|
|
rx_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$")
|
|
rx_ref = re.compile(r'^\s*ref[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)')
|
|
rx_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$')
|
|
rx_starref = re.compile(r'(?i)^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$')
|
|
rx_argsref = re.compile(r'(?i)^[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$')
|
|
|
|
survexlegsalllength = 0.0
|
|
survexlegsnumber = 0
|
|
depthbegin = 0
|
|
depthimport = 0
|
|
stackbegin =[]
|
|
stackimport = []
|
|
svxfileslist =[]
|
|
lineno = 0
|
|
insp = ""
|
|
callcount = 0
|
|
stardata ={}
|
|
|
|
def __init__(self):
|
|
pass
|
|
|
|
def LoadSurvexIgnore(self, survexblock, line, cmd):
|
|
if cmd == "title":
|
|
pass # unused in troggle today - but will become text list on SurvexBlock
|
|
elif cmd == "require":
|
|
pass # should we check survex version available for processing?
|
|
elif cmd in ["equate", "fix", "alias", "calibrate", "cs","entrance", "export", "case",
|
|
"declination", "infer","instrument", "sd", "units"]:
|
|
pass # we ignore all these, which is fine.
|
|
else:
|
|
if cmd in ["include", "data", "flags", "title", "set", "ref"]:
|
|
message = "! Unparsed [*{}]: '{}' {}".format(cmd, line, survexblock.survexfile.path)
|
|
print((self.insp+message))
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
else:
|
|
message = "! Bad svx command: [*{}] {} ({}) {}".format(cmd, line, survexblock, survexblock.survexfile.path)
|
|
print((self.insp+message))
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
|
|
def LoadSurvexTeam(self, survexblock, line):
|
|
teammembers = [ ]
|
|
mteammember = self.rx_team.match(line)
|
|
if mteammember:
|
|
for tm in self.rx_person.split(mteammember.group(2)):
|
|
if tm:
|
|
personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
|
|
if (personexpedition, tm) not in teammembers:
|
|
teammembers.append((personexpedition, tm))
|
|
personrole = models_survex.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
|
|
personrole.expeditionday = survexblock.expeditionday
|
|
if personexpedition:
|
|
personrole.person=personexpedition.person
|
|
personrole.save()
|
|
|
|
def LoadSurvexDate(self, survexblock, line):
|
|
# we should make this a date range for everything
|
|
if len(line) == 10:
|
|
survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone())
|
|
expeditions = models.Expedition.objects.filter(year=line[:4])
|
|
if expeditions:
|
|
assert len(expeditions) == 1
|
|
survexblock.expedition = expeditions[0]
|
|
survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date)
|
|
survexblock.save()
|
|
|
|
def LoadSurvexLineLeg(self, survexblock, svxline, sline, comment):
|
|
"""This reads compass, clino and tape data but only keeps the tape lengths,
|
|
the rest is discarded after error-checking.
|
|
"""
|
|
stardata = self.stardata
|
|
survexleg = SurvexLeg()
|
|
|
|
ls = sline.lower().split()
|
|
# this next fails for two surface survey svx files which use / for decimal point
|
|
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
|
|
if stardata["type"] == "normal": # should use current flags setting for this
|
|
# print(" !! lineno '{}'\n !! svxline '{}'\n !! sline '{}'\n !! ls '{}'\n !! stardata {}".format(self.lineno, svxline, sline, ls,stardata))
|
|
tape = ls[stardata["tape"]]
|
|
tape = tape.replace("(","")
|
|
tape = tape.replace(")","")
|
|
tape = tape.replace("/",".")
|
|
try:
|
|
survexleg.tape = float(tape)
|
|
self.survexlegsnumber += 1
|
|
except ValueError:
|
|
print(("! Tape misread in", survexblock.survexfile.path))
|
|
print((" Stardata:", stardata))
|
|
print((" Line:", ls))
|
|
message = ' ! Value Error: Tape misread in line %s in %s' % (ls, survexblock.survexfile.path)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
survexleg.tape = 0
|
|
try:
|
|
survexblock.totalleglength += survexleg.tape
|
|
self.survexlegsalllength += survexleg.tape
|
|
except ValueError:
|
|
message = ' ! Value Error: Tape length not added %s in %s' % (ls, survexblock.survexfile.path)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
|
|
try:
|
|
lclino = ls[stardata["clino"]]
|
|
except:
|
|
print(("! Clino misread in", survexblock.survexfile.path))
|
|
print((" Stardata:", stardata))
|
|
print((" Line:", ls))
|
|
message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
lclino = error
|
|
|
|
try:
|
|
lcompass = ls[stardata["compass"]]
|
|
except:
|
|
print(("! Compass misread in", survexblock.survexfile.path))
|
|
print((" Stardata:", stardata))
|
|
print((" Line:", ls))
|
|
message = ' ! Value Error: Compass misread in line %s in %s' % (ls, survexblock.survexfile.path)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
lcompass = error
|
|
|
|
if lclino == "up":
|
|
survexleg.compass = 0.0
|
|
survexleg.clino = 90.0
|
|
elif lclino == "down":
|
|
survexleg.compass = 0.0
|
|
survexleg.clino = -90.0
|
|
elif lclino == "-" or lclino == "level":
|
|
try:
|
|
survexleg.compass = float(lcompass)
|
|
except ValueError:
|
|
print(("! Compass misread in", survexblock.survexfile.path))
|
|
print((" Stardata:", stardata))
|
|
print((" Line:", ls))
|
|
message = ' ! Value Error: line %s in %s' % (ls, survexblock.survexfile.path)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
survexleg.compass = 1000
|
|
survexleg.clino = -90.0
|
|
else:
|
|
assert self.rx_linelen.match(lcompass), ls
|
|
assert self.rx_linelen.match(lclino) and lclino != "-", ls
|
|
survexleg.compass = float(lcompass)
|
|
survexleg.clino = float(lclino)
|
|
# delete the object so that django autosaving doesn't save it.
|
|
survexleg = None
|
|
|
|
def LoadSurvexRef(self, survexblock, args):
|
|
# *REF but also ; Ref years from 1960 to 2039
|
|
if len(args)< 4:
|
|
message = " ! Empty or BAD *REF command '{}' in '{}'".format(args, survexblock.survexfile.path)
|
|
print((self.insp+message))
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
return
|
|
|
|
argsgps = self.rx_argsref.match(args)
|
|
if argsgps:
|
|
yr, letterx, wallet = argsgps.groups()
|
|
else:
|
|
message = " ! BAD *REF command '{}' in '{}'".format(args, survexblock.survexfile.path)
|
|
print((self.insp+message))
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
return
|
|
|
|
if not letterx:
|
|
letterx = ""
|
|
else:
|
|
letterx = "X"
|
|
if len(wallet)<2:
|
|
wallet = "0" + wallet
|
|
assert (int(yr)>1960 and int(yr)<2039), "Wallet year out of bounds: %s" % yr
|
|
refscan = "%s#%s%s" % (yr, letterx, wallet)
|
|
try:
|
|
if int(wallet)>100:
|
|
message = " ! Wallet *REF {} - too big in '{}'".format(refscan, survexblock.survexfile.path)
|
|
print((self.insp+message))
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
except:
|
|
message = " ! Wallet *REF {} - not numeric in '{}'".format(refscan, survexblock.survexfile.path)
|
|
print((self.insp+message))
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
manyscansfolders = models_survex.ScansFolder.objects.filter(walletname=refscan)
|
|
if manyscansfolders:
|
|
survexblock.scansfolder = manyscansfolders[0]
|
|
survexblock.save()
|
|
if len(manyscansfolders) > 1:
|
|
message = " ! Wallet *REF {} - {} scan folders from DB search in {}".format(refscan, len(manyscansfolders), survexblock.survexfile.path)
|
|
print((self.insp+message))
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
else:
|
|
message = " ! Wallet *REF '{}' - NOT found in DB search '{}'".format(refscan, survexblock.survexfile.path)
|
|
print((self.insp+message))
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
|
|
|
|
def LoadSurvexQM(self, survexblock, qmline):
|
|
insp = self.insp
|
|
qm_no = qmline.group(1)
|
|
qm_grade = qmline.group(2)
|
|
if qmline.group(3): # usual closest survey station
|
|
qm_nearest = qmline.group(3)
|
|
if qmline.group(4):
|
|
qm_nearest = qm_nearest +"."+ qmline.group(4)
|
|
|
|
if qmline.group(6) and qmline.group(6) != '-':
|
|
qm_resolve_station = qmline.group(6)
|
|
if qmline.group(7):
|
|
qm_resolve_station = qm_resolve_station +"."+ qmline.group(7)
|
|
else:
|
|
qm_resolve_station = ""
|
|
qm_notes = qmline.group(8)
|
|
# Spec of QM in SVX files:
|
|
# ;Serial number grade(A/B/C/D/X) nearest-station resolution-station description
|
|
# ;QM1 a hobnob_hallway_2.42 hobnob-hallway_3.42 junction of keyhole passage
|
|
# ;QM1 a hobnob_hallway_2.42 - junction of keyhole passage
|
|
|
|
# NB none of the SurveyStations are in the DB now, so if we want to link to aSurvexStation
|
|
# we would have to create one. But that is not obligatory and no QMs loaded from CSVs have one
|
|
try:
|
|
qm = models_caves.QM.objects.create(number=qm_no,
|
|
# nearest_station=a_survex_station_object, # can be null
|
|
nearest_station_description=qm_resolve_station,
|
|
nearest_station_name=qm_nearest,
|
|
grade=qm_grade.upper(),
|
|
location_description=qm_notes)
|
|
qm.save
|
|
# message = " ! QM{} '{}' CREATED in DB in '{}'".format(qm_no, qm_nearest,survexblock.survexfile.path)
|
|
# print(insp+message)
|
|
# models.DataIssue.objects.create(parser='survex', message=message)
|
|
except:
|
|
message = " ! QM{} FAIL to create {} in'{}'".format(qm_no, qm_nearest,survexblock.survexfile.path)
|
|
print(insp+message)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
|
|
|
|
def LoadSurvexComment(self, survexblock, comment):
|
|
# ignore all comments except ;ref and ;QM
|
|
refline = self.rx_ref.match(comment)
|
|
if refline:
|
|
comment = comment.replace("ref","").strip()
|
|
self.LoadSurvexRef(survexblock, comment)
|
|
|
|
qmline = self.rx_qm.match(comment)
|
|
if qmline:
|
|
self.LoadSurvexQM(survexblock, qmline)
|
|
|
|
def LoadSurvexFlags(self, line, cmd):
|
|
# Here we could set on/off 'splay', 'not splay', 'surface', 'not surface', or 'duplicate'
|
|
# but this data is only used for sense-checking not to actually calculate anything important
|
|
pass
|
|
|
|
def LoadSurvexSetup(self,survexblock, survexfile):
|
|
self.depthbegin = 0
|
|
self.stardata = self.stardatadefault
|
|
blocklegs = self.survexlegsnumber
|
|
print(self.insp+" - MEM:{:.3f} Reading. parent:{} <> {} ".format(models.get_process_memory(),survexblock.survexfile.path, survexfile.path))
|
|
self.lineno = 0
|
|
sys.stderr.flush();
|
|
self.callcount +=1
|
|
if self.callcount % 10 ==0 :
|
|
print(".", file=sys.stderr,end='')
|
|
if self.callcount % 500 ==0 :
|
|
print("\n", file=sys.stderr,end='')
|
|
# Try to find the cave in the DB if not use the string as before
|
|
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", survexblock.survexfile.path)
|
|
if path_match:
|
|
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
|
|
cave = models_caves.getCaveByReference(pos_cave)
|
|
if cave:
|
|
survexfile.cave = cave
|
|
|
|
def RecursiveScan(self, survexblock, survexfile, fin, flinear):
|
|
"""Follows the *include links in all the survex files from the root file 1623.svx
|
|
and reads only the *import and *begin and *end statements. It produces a linearised
|
|
list of the import tree
|
|
"""
|
|
indent = " " * self.depthimport
|
|
sys.stderr.flush();
|
|
self.callcount +=1
|
|
if self.callcount % 10 ==0 :
|
|
print(".", file=sys.stderr,end='')
|
|
if self.callcount % 500 ==0 :
|
|
print("\n", file=sys.stderr,end='')
|
|
|
|
self.svxfileslist.append(survexfile)
|
|
|
|
svxlines = fin.read().splitlines()
|
|
for svxline in svxlines:
|
|
self.lineno += 1
|
|
sline, comment = self.rx_comment.match(svxline.strip()).groups()
|
|
mstar = self.rx_star.match(sline)
|
|
if mstar: # yes we are reading a *cmd
|
|
cmd, args = mstar.groups()
|
|
cmd = cmd.lower()
|
|
if re.match("include$(?i)", cmd):
|
|
includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args)))
|
|
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
|
|
|
|
includesurvexfile = models_survex.SurvexFile(path=includepath)
|
|
|
|
if includesurvexfile.exists():
|
|
#--------------------------------------------------------
|
|
self.depthimport += 1
|
|
fininclude = includesurvexfile.OpenFile()
|
|
flinear.write("{:2} {} *import {}\n".format(self.depthimport, indent, includesurvexfile.path))
|
|
push = includesurvexfile.path.lower()
|
|
self.stackimport.append(push)
|
|
self.RecursiveScan(survexblock, includesurvexfile, fininclude, flinear)
|
|
pop = self.stackimport.pop()
|
|
if pop != push:
|
|
print("!!!!!!! ERROR pop != push {} != {} {}".format(pop, push, self.stackimport))
|
|
print("!!!!!!! ERROR pop != push {} != {} {}\n".format(pop, push, self.stackimport),file=flinear)
|
|
print("!!!!!!! ERROR pop != push {} != {} {}".format(pop, push, self.stackimport),file=sys.stderr)
|
|
flinear.write("{:2} {} *tropmi {}\n".format(self.depthimport, indent, includesurvexfile.path))
|
|
fininclude.close()
|
|
self.depthimport -= 1
|
|
#--------------------------------------------------------
|
|
else:
|
|
print(" ! ERROR *include file not found for {}".format(includesurvexfile))
|
|
elif re.match("begin$(?i)", cmd):
|
|
self.depthbegin += 1
|
|
depth = " " * self.depthbegin
|
|
if args:
|
|
pushargs = args
|
|
else:
|
|
pushargs = " "
|
|
self.stackbegin.append(pushargs.lower())
|
|
flinear.write(" {:2} {} *begin {}\n".format(self.depthbegin, depth, args))
|
|
pass
|
|
elif re.match("end$(?i)", cmd):
|
|
depth = " " * self.depthbegin
|
|
flinear.write(" {:2} {} *end {}\n".format(self.depthbegin, depth, args))
|
|
if not args:
|
|
args = " "
|
|
popargs = self.stackbegin.pop()
|
|
if popargs != args.lower():
|
|
print("!!!!!!! ERROR BEGIN/END pop != push {} != {}\n{}".format(popargs, args, self. stackbegin))
|
|
print("!!!!!!! ERROR BEGIN/END pop != push {} != {}\n{}\n".format(popargs, args, self. stackbegin), file=flinear)
|
|
print(" !!!!!!! ERROR BEGIN/END pop != push {} != {}\n{}".format(popargs, args,self. stackbegin), file=sys.stderr,)
|
|
|
|
self.depthbegin -= 1
|
|
pass
|
|
|
|
|
|
def RecursiveLoad(self, survexblock, survexfile, fin):
|
|
"""Follows the *include links in all the survex files from the root file 1623.svx
|
|
and reads in the survex blocks, other data and the wallet references (scansfolder) as it
|
|
goes. This part of the data import process is where the maximum memory is used and where it
|
|
crashes on memory-constrained machines. Begin-end blocks may also be nested.
|
|
"""
|
|
self.LoadSurvexSetup(survexblock, survexfile)
|
|
insp =self.insp
|
|
previousnlegs = 0
|
|
|
|
svxlines = fin.read().splitlines()
|
|
# cannot close file now as may be recursively called with the same fin if nested *begin-end
|
|
|
|
for svxline in svxlines:
|
|
self.lineno += 1
|
|
sline, comment = self.rx_comment.match(svxline.strip()).groups()
|
|
if comment:
|
|
self.LoadSurvexComment(survexblock, comment)
|
|
if not sline:
|
|
continue # skip blank lines
|
|
|
|
# detect the star command
|
|
mstar = self.rx_star.match(sline)
|
|
if mstar: # yes we are reading a *cmd
|
|
cmd, args = mstar.groups()
|
|
cmd = cmd.lower()
|
|
if re.match("include$(?i)", cmd):
|
|
includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args)))
|
|
print((insp+' - INCLUDE-go path found, including - ' + includepath))
|
|
# Try to find the cave in the DB. if not, use the string as before
|
|
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
|
|
if path_match:
|
|
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
|
|
print((insp+' - Match in DB (i) for cave {}.'.format(pos_cave)))
|
|
cave = models_caves.getCaveByReference(pos_cave)
|
|
if cave:
|
|
survexfile.cave = cave
|
|
else:
|
|
print((insp+' - NO Match in DB (i) for a cave for {}'.format(includepath)))
|
|
|
|
includesurvexfile = models_survex.SurvexFile(path=includepath)
|
|
includesurvexfile.save()
|
|
includesurvexfile.SetDirectory()
|
|
if includesurvexfile.exists():
|
|
survexblock.save()
|
|
self.insp += "> "
|
|
#--------------------------------------------------------
|
|
fininclude = includesurvexfile.OpenFile()
|
|
self.RecursiveLoad(survexblock, includesurvexfile, fininclude)
|
|
fininclude.close()
|
|
#--------------------------------------------------------
|
|
self.insp = self.insp[2:]
|
|
insp = self.insp
|
|
print((insp+' - INCLUDE-return from include - ' + includepath))
|
|
else:
|
|
print((insp+' ! ERROR *include file not found for %s' % includesurvexfile))
|
|
|
|
elif re.match("begin$(?i)", cmd):
|
|
# On a *begin statement we start a new survexblock.
|
|
# There should not be any *include inside a begin-end block, so this is a simple
|
|
# load not a recursive fileload. But there may be many blocks nested to any depth in one file.
|
|
if args:
|
|
newsvxpath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args))
|
|
# Try to find the cave in the DB if not use the string as before
|
|
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", newsvxpath)
|
|
if path_match:
|
|
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
|
|
# print(insp+pos_cave)
|
|
cave = models_caves.getCaveByReference(pos_cave)
|
|
if cave:
|
|
survexfile.cave = cave
|
|
else:
|
|
print((insp+' - No match (b) for %s' % newsvxpath))
|
|
|
|
previousnlegs = self.survexlegsnumber
|
|
name = args.lower()
|
|
print(insp+' - Begin found for:{}, creating new SurvexBlock '.format(name))
|
|
# the recursive call re-reads the entire file. This is wasteful. We should pass in only
|
|
# the un-parsed part of the file.
|
|
survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock,
|
|
survexpath=survexblock.survexpath+"."+name,
|
|
cave=survexfile.cave, survexfile=survexfile,
|
|
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
|
|
survexblockdown.save()
|
|
survexblock.save()
|
|
survexblock = survexblockdown
|
|
print(insp+" - BLOCK-enter nested *begin/*end block: '{}'".format(name))
|
|
self.insp += "> "
|
|
#--------------------------------------------------------
|
|
self.RecursiveLoad(survexblockdown, survexfile, fin)
|
|
#--------------------------------------------------------
|
|
# do not close the file as there may be more blocks in this one
|
|
# and it is re-read afresh with every nested begin-end block.
|
|
self.insp = self.insp[2:]
|
|
insp = self.insp
|
|
else:
|
|
self.depthbegin += 1
|
|
|
|
elif re.match("end$(?i)", cmd):
|
|
if self.depthbegin:
|
|
print(insp+" - BLOCK-return from nested *begin/*end block: '{}'".format(args))
|
|
self.depthbegin -= 1
|
|
else:
|
|
legsinblock = self.survexlegsnumber - previousnlegs
|
|
print(insp+" - LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,self.survexlegsnumber))
|
|
survexblock.legsall = legsinblock
|
|
survexblock.save()
|
|
return
|
|
elif cmd == "ref":
|
|
self.LoadSurvexRef(survexblock, args)
|
|
elif cmd == "flags":
|
|
self.LoadSurvexFlags(args, cmd)
|
|
elif cmd == "data":
|
|
ls = args.lower().split()
|
|
stardata = { "type":ls[0] }
|
|
for i in range(0, len(ls)):
|
|
stardata[self.stardataparamconvert.get(ls[i], ls[i])] = i - 1
|
|
self.stardata = stardata
|
|
if ls[0] in ["normal", "cartesian", "nosurvey"]:
|
|
assert (("from" in stardata and "to" in stardata) or "station" in stardata), args
|
|
elif ls[0] == "default":
|
|
stardata = self.stardatadefault
|
|
else:
|
|
assert ls[0] == "passage", args
|
|
elif cmd == "set" and re.match("names(?i)", args):
|
|
pass
|
|
elif re.match("date$(?i)", cmd):
|
|
self.LoadSurvexDate(survexblock, args)
|
|
elif re.match("team$(?i)", cmd):
|
|
self.LoadSurvexTeam(survexblock, args)
|
|
else:
|
|
self.LoadSurvexIgnore(survexblock, args, cmd)
|
|
else: # not a *cmd so we are reading data OR rx_comment failed
|
|
if "from" in self.stardata: # only interested in survey legs
|
|
self.LoadSurvexLineLeg(survexblock, svxline, sline, comment)
|
|
else:
|
|
pass # ignore all other sorts of data
|
|
|
|
|
|
def FindAndLoadAllSurvex(survexblockroot, survexfileroot):
|
|
"""Follows the *include links recursively to find files
|
|
"""
|
|
print(' - redirecting stdout to svxblks.log...')
|
|
stdout_orig = sys.stdout
|
|
# Redirect sys.stdout to the file
|
|
sys.stdout = open('svxblks.log', 'w')
|
|
|
|
print(' - SCANNING All Survex Blocks...',file=sys.stderr)
|
|
|
|
svxl0 = LoadSurvex()
|
|
svxl0.callcount = 0
|
|
svxl0.depthimport = 0
|
|
indent=""
|
|
|
|
mem0 = models.get_process_memory()
|
|
flinear = open('svxlinear.log', 'w')
|
|
flinear.write(" - MEM:{:.2f} MB START {}\n".format(mem0,survexfileroot.path))
|
|
|
|
finroot = survexfileroot.OpenFile()
|
|
flinear.write("{:2} {} *import {}\n".format(svxl0.depthimport, indent, survexfileroot.path))
|
|
svxl0.RecursiveScan(survexblockroot, survexfileroot, finroot, flinear)
|
|
flinear.write("{:2} {} *tropmi {}\n".format(svxl0.depthimport, indent, survexfileroot.path))
|
|
mem1 = models.get_process_memory()
|
|
flinear.write(" - MEM:{:.2f} MB STOP {}\n".format(mem1,survexfileroot.path))
|
|
flinear.write(" - MEM:{:.3f} MB USED\n".format(mem1-mem0))
|
|
svxfileslist = svxl0.svxfileslist
|
|
flinear.write(" - {:,} survex files in linear import list \n".format(len(svxfileslist)))
|
|
flinear.close()
|
|
svxl0 = None
|
|
print("\n - {:,} survex files in linear import list \n".format(len(svxfileslist)),file=sys.stderr)
|
|
|
|
# INSERT IN HERE linear, not recursive, wrt import loading of all the data using [svxfileslist] #
|
|
for f in svxfileslist:
|
|
# Load legs etc. recursive only in BEGIN / END
|
|
pass
|
|
|
|
print('\n - Loading All Survex Blocks...',file=sys.stderr)
|
|
svxlrl = LoadSurvex()
|
|
|
|
finroot = survexfileroot.OpenFile()
|
|
svxlrl.RecursiveLoad(survexblockroot, survexfileroot, finroot)
|
|
finroot.close()
|
|
|
|
survexlegsnumber = svxlrl.survexlegsnumber
|
|
survexlegsalllength = svxlrl.survexlegsalllength
|
|
svxlrl = None
|
|
# Close the logging file, Restore sys.stdout to our old saved file handle
|
|
sys.stdout.close()
|
|
print("+", file=sys.stderr)
|
|
sys.stderr.flush();
|
|
sys.stdout = stdout_orig
|
|
return (survexlegsnumber, survexlegsalllength)
|
|
|
|
|
|
def LoadAllSurvexBlocks():
|
|
|
|
print(' - Flushing All Survex Blocks...')
|
|
models_survex.SurvexBlock.objects.all().delete()
|
|
models_survex.SurvexFile.objects.all().delete()
|
|
models_survex.SurvexDirectory.objects.all().delete()
|
|
models_survex.SurvexPersonRole.objects.all().delete()
|
|
models_survex.SurvexStation.objects.all().delete()
|
|
print(" - survex Data Issues flushed")
|
|
models.DataIssue.objects.filter(parser='survex').delete()
|
|
|
|
survexfileroot = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
|
|
survexfileroot.save()
|
|
survexfileroot.SetDirectory()
|
|
survexblockroot = models_survex.SurvexBlock(name=ROOTBLOCK, survexpath="", cave=None, survexfile=survexfileroot,
|
|
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
|
|
# this is the first so id=1
|
|
survexblockroot.save()
|
|
|
|
print(' - Loading All Survex Blocks...')
|
|
memstart = models.get_process_memory()
|
|
survexlegsnumber, survexlegsalllength = FindAndLoadAllSurvex(survexblockroot, survexfileroot)
|
|
memend = models.get_process_memory()
|
|
print(" - MEMORY start:{:.3f} MB end:{:.3f} MB increase={:.3f} MB".format(memstart,memend, memend-memstart))
|
|
|
|
survexblockroot.totalleglength = survexlegsalllength
|
|
survexblockroot.legsall = survexlegsnumber
|
|
survexblockroot.save()
|
|
|
|
print(" - total number of survex legs: {}m".format(survexlegsnumber))
|
|
print(" - total leg lengths loaded: {}m".format(survexlegsalllength))
|
|
print(' - Loaded All Survex Blocks.')
|
|
|
|
|
|
poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
|
|
|
|
def LoadPos():
|
|
"""Run cavern to produce a complete .3d file, then run 3dtopos to produce a table of
|
|
all survey point positions. Then lookup each position by name to see if we have it in the database
|
|
and if we do, then save the x/y/z coordinates.
|
|
If we don't have it in the database, print an error message and discard it.
|
|
This is ONLY ever used for entrance and fixedpts locations for the prospecting map:
|
|
about 600 points out of 32,000.
|
|
"""
|
|
topdata = settings.SURVEX_DATA + settings.SURVEX_TOPNAME
|
|
print((' - Generating a list of Pos from %s.svx and then loading...' % (topdata)))
|
|
|
|
found = 0
|
|
skip = {}
|
|
print("\n") # extra line because cavern overwrites the text buffer somehow
|
|
# cavern defaults to using same cwd as supplied input file
|
|
call([settings.CAVERN, "--output=%s.3d" % (topdata), "%s.svx" % (topdata)])
|
|
call([settings.THREEDTOPOS, '%s.3d' % (topdata)], cwd = settings.SURVEX_DATA)
|
|
|
|
mappoints = {}
|
|
for pt in MapLocations().points():
|
|
svxid, number, point_type, label = pt
|
|
mappoints[svxid]=True
|
|
|
|
posfile = open("%s.pos" % (topdata))
|
|
posfile.readline() #Drop header
|
|
try:
|
|
survexblockroot = models_survex.SurvexBlock.objects.get(name=ROOTBLOCK)
|
|
except:
|
|
try:
|
|
survexblockroot = models_survex.SurvexBlock.objects.get(id=1)
|
|
except:
|
|
message = ' ! FAILED to find root SurvexBlock'
|
|
print(message)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
raise
|
|
for line in posfile.readlines():
|
|
r = poslineregex.match(line)
|
|
if r:
|
|
x, y, z, id = r.groups()
|
|
for sid in mappoints:
|
|
if id.endswith(sid):
|
|
blockpath = "." + id[:-len(sid)].strip(".")
|
|
try:
|
|
sbqs = models_survex.SurvexBlock.objects.filter(survexpath=blockpath)
|
|
if len(sbqs)==1:
|
|
sb = sbqs[0]
|
|
if len(sbqs)>1:
|
|
message = ' ! MULTIPLE SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
|
|
print(message)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
sb = sbqs[0]
|
|
elif len(sbqs)<=0:
|
|
message = ' ! ZERO SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
|
|
print(message)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
sb = survexblockroot
|
|
except:
|
|
message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid)
|
|
print(message)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
try:
|
|
ss = models_survex.SurvexStation(name=id, block=sb)
|
|
ss.x = float(x)
|
|
ss.y = float(y)
|
|
ss.z = float(z)
|
|
ss.save()
|
|
found += 1
|
|
except:
|
|
message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid)
|
|
print(message)
|
|
models.DataIssue.objects.create(parser='survex', message=message)
|
|
raise
|
|
print(" - {} SurvexStation entrances found.".format(found))
|
|
|