mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2025-04-07 03:11:44 +01:00
start refactor survex import
This commit is contained in:
Philip Sargent
parent
dc5a53376d
commit
bb69cc073a
@ -3,6 +3,7 @@ import os
|
|||||||
import datetime
|
import datetime
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
import resource
|
||||||
from subprocess import call
|
from subprocess import call
|
||||||
|
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
@ -22,6 +23,11 @@ from django.template import Context, loader
|
|||||||
|
|
||||||
import troggle.core.models_survex
|
import troggle.core.models_survex
|
||||||
|
|
||||||
|
def get_process_memory():
|
||||||
|
usage=resource.getrusage(resource.RUSAGE_SELF)
|
||||||
|
return usage[2]/1024.0
|
||||||
|
|
||||||
|
|
||||||
def get_related_by_wikilinks(wiki_text):
|
def get_related_by_wikilinks(wiki_text):
|
||||||
found=re.findall(settings.QM_PATTERN,wiki_text)
|
found=re.findall(settings.QM_PATTERN,wiki_text)
|
||||||
res=[]
|
res=[]
|
||||||
|
|||||||
@ -81,13 +81,6 @@ class SurvexStation(models.Model):
|
|||||||
else:
|
else:
|
||||||
return r
|
return r
|
||||||
|
|
||||||
class SurvexLeg():
|
|
||||||
"""No longer a models.Model subclass, so no longer a database table
|
|
||||||
"""
|
|
||||||
tape = 0.0
|
|
||||||
compass = 0.0
|
|
||||||
clino = 0.0
|
|
||||||
|
|
||||||
#
|
#
|
||||||
# Single SurvexBlock
|
# Single SurvexBlock
|
||||||
#
|
#
|
||||||
|
|||||||
@ -12,7 +12,7 @@ from django.views.generic.list import ListView
|
|||||||
|
|
||||||
from troggle.core.models import Expedition, Person, PersonExpedition
|
from troggle.core.models import Expedition, Person, PersonExpedition
|
||||||
from troggle.core.models_caves import Cave, LogbookEntry
|
from troggle.core.models_caves import Cave, LogbookEntry
|
||||||
from troggle.core.models_survex import SurvexLeg, SurvexBlock
|
from troggle.core.models_survex import SurvexBlock
|
||||||
|
|
||||||
import troggle.settings as settings
|
import troggle.settings as settings
|
||||||
|
|
||||||
@ -106,7 +106,6 @@ def stats(request):
|
|||||||
legsbyexpo.append((expedition, {"nsurvexlegs": "{:,}".format(legsyear),
|
legsbyexpo.append((expedition, {"nsurvexlegs": "{:,}".format(legsyear),
|
||||||
"survexleglength":"{:,.0f}".format(survexleglength)}))
|
"survexleglength":"{:,.0f}".format(survexleglength)}))
|
||||||
legsbyexpo.reverse()
|
legsbyexpo.reverse()
|
||||||
#survexlegs = SurvexLeg.objects.all()
|
|
||||||
|
|
||||||
renderDict = {**statsDict, **{ "nsurvexlegs": "{:,}".format(nimportlegs), "totalsurvexlength":totalsurvexlength/1000, "addupsurvexlength":addupsurvexlength/1000, "legsbyexpo":legsbyexpo }} # new syntax
|
renderDict = {**statsDict, **{ "nsurvexlegs": "{:,}".format(nimportlegs), "totalsurvexlength":totalsurvexlength/1000, "addupsurvexlength":addupsurvexlength/1000, "legsbyexpo":legsbyexpo }} # new syntax
|
||||||
return render(request,'statistics.html', renderDict)
|
return render(request,'statistics.html', renderDict)
|
||||||
|
|||||||
@ -2,7 +2,6 @@ import sys
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
import resource
|
|
||||||
|
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from subprocess import call, Popen, PIPE
|
from subprocess import call, Popen, PIPE
|
||||||
@ -17,471 +16,467 @@ import troggle.core.models_survex as models_survex
|
|||||||
from troggle.parsers.people import GetPersonExpeditionNameLookup
|
from troggle.parsers.people import GetPersonExpeditionNameLookup
|
||||||
from troggle.core.views_caves import MapLocations
|
from troggle.core.views_caves import MapLocations
|
||||||
|
|
||||||
|
|
||||||
"""A 'survex block' is a *begin...*end set of cave data.
|
|
||||||
A 'scansfolder' is what we today call a "survey scans folder" or a "wallet".
|
|
||||||
"""
|
|
||||||
|
|
||||||
rx_braskets= re.compile(r"[()]")
|
|
||||||
rx_line_length = re.compile(r"[\d\-+.]+$")
|
|
||||||
survexlegsalllength = 0.0
|
|
||||||
survexlegsnumber = 0
|
|
||||||
survexblockroot = None
|
survexblockroot = None
|
||||||
ROOTBLOCK = "rootblock"
|
ROOTBLOCK = "rootblock"
|
||||||
|
|
||||||
|
class SurvexLeg():
|
||||||
def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
|
"""No longer a models.Model subclass, so no longer a database table
|
||||||
"""This reads compass, clino and tape data but only keeps the tape lengths,
|
|
||||||
the rest is discarded after error-checking.
|
|
||||||
"""
|
"""
|
||||||
global survexlegsalllength
|
tape = 0.0
|
||||||
global survexlegsnumber
|
compass = 0.0
|
||||||
ls = sline.lower().split()
|
clino = 0.0
|
||||||
survexleg = models_survex.SurvexLeg()
|
|
||||||
# this next fails for two surface survey svx files which use / for decimal point
|
class LoadSurvex():
|
||||||
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
|
"""A 'survex block' is a *begin...*end set of cave data.
|
||||||
if stardata["type"] == "normal":
|
A survex file can contain many begin-end blocks, which can be nested, and which can *include
|
||||||
tape = rx_braskets.sub("",ls[stardata["tape"]])
|
other survex files.
|
||||||
tape = tape.replace("/",".")
|
A 'scansfolder' is what we today call a "survey scans folder" or a "wallet".
|
||||||
try:
|
"""
|
||||||
survexleg.tape = float(tape)
|
|
||||||
survexlegsnumber += 1
|
# This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g.
|
||||||
except ValueError:
|
# *DATA normal from to length gradient bearing ignore ignore ignore ignore
|
||||||
print(("! Tape misread in", survexblock.survexfile.path))
|
stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
|
||||||
print((" Stardata:", stardata))
|
stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"}
|
||||||
print((" Line:", ls))
|
|
||||||
message = ' ! Value Error: Tape misread in line %s in %s' % (ls, survexblock.survexfile.path)
|
rx_braskets= re.compile(r"[()]")
|
||||||
models.DataIssue.objects.create(parser='survex', message=message)
|
rx_linelen = re.compile(r"[\d\-+.]+$")
|
||||||
survexleg.tape = 0
|
rx_team = re.compile(r"(?i)(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$")
|
||||||
try:
|
rx_person = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$")
|
||||||
lclino = ls[stardata["clino"]]
|
rx_qm = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')
|
||||||
except:
|
# remember there is also QM_PATTERN used in views_other and set in settings.py
|
||||||
print(("! Clino misread in", survexblock.survexfile.path))
|
|
||||||
print((" Stardata:", stardata))
|
rx_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$")
|
||||||
print((" Line:", ls))
|
rx_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(X)?\s*(\d+)')
|
||||||
message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path)
|
rx_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$')
|
||||||
models.DataIssue.objects.create(parser='survex', message=message)
|
# years from 1960 to 2039
|
||||||
lclino = error
|
rx_starref = re.compile(r'(?i)^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$')
|
||||||
try:
|
# rx_starref = re.compile("""?x # VERBOSE mode - can't get this to work
|
||||||
lcompass = ls[stardata["compass"]]
|
# ^\s*\*ref # look for *ref at start of line
|
||||||
except:
|
# [\s.:]* # some spaces, stops or colons
|
||||||
print(("! Compass misread in", survexblock.survexfile.path))
|
# ((?:19[6789]\d)|(?:20[0123]\d)) # a date from 1960 to 2039 - captured as one field
|
||||||
print((" Stardata:", stardata))
|
# \s*# # spaces then hash separator
|
||||||
print((" Line:", ls))
|
# ?\s*(X) # optional X - captured
|
||||||
message = ' ! Value Error: Compass misread in line %s in %s' % (ls, survexblock.survexfile.path)
|
# ?\s*(.*?\d+.*?) # maybe a space, then at least one digit in the string - captured
|
||||||
models.DataIssue.objects.create(parser='survex', message=message)
|
# $(?i)""", re.X) # the end (do the whole thing case insensitively)
|
||||||
lcompass = error
|
|
||||||
if lclino == "up":
|
survexlegsalllength = 0.0
|
||||||
survexleg.compass = 0.0
|
survexlegsnumber = 0
|
||||||
survexleg.clino = 90.0
|
insp = ""
|
||||||
elif lclino == "down":
|
callcount = 0
|
||||||
survexleg.compass = 0.0
|
|
||||||
survexleg.clino = -90.0
|
def __init__(self):
|
||||||
elif lclino == "-" or lclino == "level":
|
pass
|
||||||
|
|
||||||
|
def LoadSurvexLineLeg(self,survexblock, stardata, sline, comment):
|
||||||
|
"""This reads compass, clino and tape data but only keeps the tape lengths,
|
||||||
|
the rest is discarded after error-checking.
|
||||||
|
"""
|
||||||
|
ls = sline.lower().split()
|
||||||
|
survexleg = SurvexLeg()
|
||||||
|
# this next fails for two surface survey svx files which use / for decimal point
|
||||||
|
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
|
||||||
|
if stardata["type"] == "normal":
|
||||||
|
tape = self.rx_braskets.sub("",ls[stardata["tape"]])
|
||||||
|
tape = tape.replace("/",".")
|
||||||
try:
|
try:
|
||||||
survexleg.compass = float(lcompass)
|
survexleg.tape = float(tape)
|
||||||
|
self.survexlegsnumber += 1
|
||||||
except ValueError:
|
except ValueError:
|
||||||
|
print(("! Tape misread in", survexblock.survexfile.path))
|
||||||
|
print((" Stardata:", stardata))
|
||||||
|
print((" Line:", ls))
|
||||||
|
message = ' ! Value Error: Tape misread in line %s in %s' % (ls, survexblock.survexfile.path)
|
||||||
|
models.DataIssue.objects.create(parser='survex', message=message)
|
||||||
|
survexleg.tape = 0
|
||||||
|
try:
|
||||||
|
lclino = ls[stardata["clino"]]
|
||||||
|
except:
|
||||||
|
print(("! Clino misread in", survexblock.survexfile.path))
|
||||||
|
print((" Stardata:", stardata))
|
||||||
|
print((" Line:", ls))
|
||||||
|
message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path)
|
||||||
|
models.DataIssue.objects.create(parser='survex', message=message)
|
||||||
|
lclino = error
|
||||||
|
try:
|
||||||
|
lcompass = ls[stardata["compass"]]
|
||||||
|
except:
|
||||||
print(("! Compass misread in", survexblock.survexfile.path))
|
print(("! Compass misread in", survexblock.survexfile.path))
|
||||||
print((" Stardata:", stardata))
|
print((" Stardata:", stardata))
|
||||||
print((" Line:", ls))
|
print((" Line:", ls))
|
||||||
message = ' ! Value Error: line %s in %s' % (ls, survexblock.survexfile.path)
|
message = ' ! Value Error: Compass misread in line %s in %s' % (ls, survexblock.survexfile.path)
|
||||||
models.DataIssue.objects.create(parser='survex', message=message)
|
models.DataIssue.objects.create(parser='survex', message=message)
|
||||||
survexleg.compass = 1000
|
lcompass = error
|
||||||
survexleg.clino = -90.0
|
if lclino == "up":
|
||||||
else:
|
survexleg.compass = 0.0
|
||||||
assert rx_line_length.match(lcompass), ls
|
survexleg.clino = 90.0
|
||||||
assert rx_line_length.match(lclino) and lclino != "-", ls
|
elif lclino == "down":
|
||||||
survexleg.compass = float(lcompass)
|
survexleg.compass = 0.0
|
||||||
survexleg.clino = float(lclino)
|
survexleg.clino = -90.0
|
||||||
|
elif lclino == "-" or lclino == "level":
|
||||||
if cave:
|
try:
|
||||||
survexleg.cave = cave
|
survexleg.compass = float(lcompass)
|
||||||
|
except ValueError:
|
||||||
# No need to save as we are measuring lengths only on parsing now.
|
print(("! Compass misread in", survexblock.survexfile.path))
|
||||||
# delete the object so that django autosaving doesn't save it.
|
print((" Stardata:", stardata))
|
||||||
survexleg = None
|
print((" Line:", ls))
|
||||||
|
message = ' ! Value Error: line %s in %s' % (ls, survexblock.survexfile.path)
|
||||||
itape = stardata.get("tape")
|
models.DataIssue.objects.create(parser='survex', message=message)
|
||||||
if itape:
|
survexleg.compass = 1000
|
||||||
try:
|
survexleg.clino = -90.0
|
||||||
survexblock.totalleglength += float(ls[itape])
|
|
||||||
survexlegsalllength += float(ls[itape])
|
|
||||||
except ValueError:
|
|
||||||
print("! Length not added")
|
|
||||||
# No need to save as we are measuring lengths only on parsing now.
|
|
||||||
|
|
||||||
|
|
||||||
# def LoadSurvexEquate(survexblock, sline):
|
|
||||||
# #print sline #
|
|
||||||
# stations = sline.split()
|
|
||||||
# assert len(stations) > 1
|
|
||||||
# for station in stations:
|
|
||||||
# survexblock.MakeSurvexStation(station)
|
|
||||||
|
|
||||||
|
|
||||||
def LoadSurvexLinePassage(survexblock, stardata, sline, comment):
|
|
||||||
# do not import this: *data passage.. data which is LRUD not tape/compass/clino
|
|
||||||
pass
|
|
||||||
|
|
||||||
# This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g.
|
|
||||||
# *DATA normal from to length gradient bearing ignore ignore ignore ignore
|
|
||||||
stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
|
|
||||||
stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"}
|
|
||||||
|
|
||||||
rx_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$")
|
|
||||||
rx_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(X)?\s*(\d+)')
|
|
||||||
rx_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$')
|
|
||||||
# years from 1960 to 2039
|
|
||||||
rx_starref = re.compile(r'(?i)^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*#?\s*(X)?\s*(.*?\d+.*?)$')
|
|
||||||
# rx_starref = re.compile("""?x # VERBOSE mode - can't get this to work
|
|
||||||
# ^\s*\*ref # look for *ref at start of line
|
|
||||||
# [\s.:]* # some spaces, stops or colons
|
|
||||||
# ((?:19[6789]\d)|(?:20[0123]\d)) # a date from 1960 to 2039 - captured as one field
|
|
||||||
# \s*# # spaces then hash separator
|
|
||||||
# ?\s*(X) # optional X - captured
|
|
||||||
# ?\s*(.*?\d+.*?) # maybe a space, then at least one digit in the string - captured
|
|
||||||
# $(?i)""", re.X) # the end (do the whole thing case insensitively)
|
|
||||||
|
|
||||||
rx_team = re.compile(r"(?i)(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$")
|
|
||||||
rx_person = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$")
|
|
||||||
rx_qm = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')
|
|
||||||
|
|
||||||
insp = ""
|
|
||||||
callcount = 0
|
|
||||||
def RecursiveLoad(survexblock, survexfile, fin):
|
|
||||||
"""Follows the *include links in all the survex files from the root file 1623.svx
|
|
||||||
and reads in the survex blocks, other data and the wallet references (scansfolder) as it
|
|
||||||
goes. This part of the data import process is where the maximum memory is used and where it
|
|
||||||
crashes on memory-constrained machines.
|
|
||||||
"""
|
|
||||||
iblankbegins = 0
|
|
||||||
text = [ ]
|
|
||||||
stardata = stardatadefault
|
|
||||||
teammembers = [ ]
|
|
||||||
global insp
|
|
||||||
global callcount
|
|
||||||
global survexlegsnumber
|
|
||||||
|
|
||||||
print(insp+" - MEM:{:.3f} Reading. parent:{} <> {} ".format(get_process_memory(),survexblock.survexfile.path,survexfile.path))
|
|
||||||
stamp = datetime.now()
|
|
||||||
lineno = 0
|
|
||||||
|
|
||||||
sys.stderr.flush();
|
|
||||||
callcount +=1
|
|
||||||
if callcount >=10:
|
|
||||||
callcount=0
|
|
||||||
print(".", file=sys.stderr,end='')
|
|
||||||
|
|
||||||
# Try to find the cave in the DB if not use the string as before
|
|
||||||
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", survexblock.survexfile.path)
|
|
||||||
if path_match:
|
|
||||||
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
|
|
||||||
cave = models_caves.getCaveByReference(pos_cave)
|
|
||||||
if cave:
|
|
||||||
survexfile.cave = cave
|
|
||||||
svxlines = ''
|
|
||||||
svxlines = fin.read().splitlines()
|
|
||||||
# cannot close file now as it may be recursively called with the same file id fin if nested *begin
|
|
||||||
# occurs.
|
|
||||||
for svxline in svxlines:
|
|
||||||
lineno += 1
|
|
||||||
# break the line at the comment
|
|
||||||
sline, comment = rx_comment.match(svxline.strip()).groups()
|
|
||||||
# detect ref line pointing to the scans directory
|
|
||||||
mref = comment and rx_ref.match(comment)
|
|
||||||
if mref:
|
|
||||||
yr, letterx, wallet = mref.groups()
|
|
||||||
if not letterx:
|
|
||||||
letterx = ""
|
|
||||||
else:
|
else:
|
||||||
letterx = "X"
|
assert self.rx_linelen.match(lcompass), ls
|
||||||
if len(wallet)<2:
|
assert self.rx_linelen.match(lclino) and lclino != "-", ls
|
||||||
wallet = "0" + wallet
|
survexleg.compass = float(lcompass)
|
||||||
refscan = "%s#%s%s" % (yr, letterx, wallet )
|
survexleg.clino = float(lclino)
|
||||||
manyscansfolders = models_survex.ScansFolder.objects.filter(walletname=refscan)
|
|
||||||
if manyscansfolders:
|
|
||||||
survexblock.scansfolder = manyscansfolders[0]
|
|
||||||
survexblock.save()
|
|
||||||
else:
|
|
||||||
message = ' ! Wallet ; ref {} - NOT found in manyscansfolders {}'.format(refscan, survexblock.survexfile.path)
|
|
||||||
print((insp+message))
|
|
||||||
models.DataIssue.objects.create(parser='survex', message=message)
|
|
||||||
|
|
||||||
# This whole section should be moved if we can have *QM become a proper survex command
|
# No need to save as we are measuring lengths only on parsing now.
|
||||||
# Spec of QM in SVX files, currently commented out need to add to survex
|
# delete the object so that django autosaving doesn't save it.
|
||||||
# needs to match rx_qm
|
survexleg = None
|
||||||
# ;Serial number grade(A/B/C/D/X) nearest-station resolution-station description
|
|
||||||
# ;QM1 a hobnob_hallway_2.42 hobnob-hallway_3.42 junction of keyhole passage
|
|
||||||
# ;QM1 a hobnob_hallway_2.42 - junction of keyhole passage
|
|
||||||
qmline = comment and rx_qm.match(comment)
|
|
||||||
if qmline:
|
|
||||||
qm_no = qmline.group(1)
|
|
||||||
qm_grade = qmline.group(2)
|
|
||||||
qm_from_section = qmline.group(3)
|
|
||||||
qm_from_station = qmline.group(4)
|
|
||||||
qm_resolve_section = qmline.group(6)
|
|
||||||
qm_resolve_station = qmline.group(7)
|
|
||||||
qm_notes = qmline.group(8)
|
|
||||||
|
|
||||||
# print(insp+'Cave - %s' % survexfile.cave)
|
itape = stardata.get("tape")
|
||||||
# print(insp+'QM no %d' % int(qm_no))
|
if itape:
|
||||||
# print(insp+'QM grade %s' % qm_grade)
|
try:
|
||||||
# print(insp+'QM section %s' % qm_from_section)
|
survexblock.totalleglength += float(ls[itape])
|
||||||
# print(insp+'QM station %s' % qm_from_station)
|
self.survexlegsalllength += float(ls[itape])
|
||||||
# print(insp+'QM res section %s' % qm_resolve_section)
|
except ValueError:
|
||||||
# print(insp+'QM res station %s' % qm_resolve_station)
|
print("! Length not added")
|
||||||
# print(insp+'QM notes %s' % qm_notes)
|
# No need to save as we are measuring lengths only on parsing now.
|
||||||
|
|
||||||
# If the QM isn't resolved (has a resolving station) then load it
|
|
||||||
if not qm_resolve_section or qm_resolve_section != '-' or qm_resolve_section != 'None':
|
|
||||||
from_section = models_survex.SurvexBlock.objects.filter(name=qm_from_section)
|
|
||||||
# If we can find a section (survex note chunck, named)
|
|
||||||
if len(from_section) > 0:
|
|
||||||
from_station = models_survex.SurvexStation.objects.filter(block=from_section[0], name=qm_from_station)
|
|
||||||
# If we can find a from station then we have the nearest station and can import it
|
|
||||||
if len(from_station) > 0:
|
|
||||||
qm = models_caves.QM.objects.create(number=qm_no,
|
|
||||||
nearest_station=from_station[0],
|
|
||||||
grade=qm_grade.upper(),
|
|
||||||
location_description=qm_notes)
|
|
||||||
else:
|
|
||||||
# print(insp+' - QM found but resolved')
|
|
||||||
pass
|
|
||||||
|
|
||||||
if not sline:
|
def LoadSurvexLinePassage(self,survexblock, stardata, sline, comment):
|
||||||
continue
|
# do not import this: *data passage.. data which is LRUD not tape/compass/clino
|
||||||
|
pass
|
||||||
|
|
||||||
# detect the star ref command
|
def RecursiveLoad(self,survexblock, survexfile, fin):
|
||||||
mstar = rx_starref.match(sline)
|
"""Follows the *include links in all the survex files from the root file 1623.svx
|
||||||
if mstar:
|
and reads in the survex blocks, other data and the wallet references (scansfolder) as it
|
||||||
yr,letterx,wallet = mstar.groups()
|
goes. This part of the data import process is where the maximum memory is used and where it
|
||||||
if not letterx:
|
crashes on memory-constrained machines.
|
||||||
letterx = ""
|
"""
|
||||||
else:
|
iblankbegins = 0
|
||||||
letterx = "X"
|
text = [ ]
|
||||||
if len(wallet)<2:
|
stardata = self.stardatadefault
|
||||||
wallet = "0" + wallet
|
teammembers = [ ]
|
||||||
assert (int(yr)>1960 and int(yr)<2039), "Wallet year out of bounds: %s" % yr
|
insp =self.insp
|
||||||
assert (int(wallet)<100), "Wallet number more than 100: %s" % wallet
|
blocklegs = self.survexlegsnumber
|
||||||
refscan = "%s#%s%s" % (yr, letterx, wallet)
|
|
||||||
manyscansfolders = models_survex.ScansFolder.objects.filter(walletname=refscan)
|
|
||||||
if manyscansfolders:
|
|
||||||
survexblock.scansfolder = manyscansfolders[0]
|
|
||||||
survexblock.save()
|
|
||||||
else:
|
|
||||||
message = ' ! Wallet *REF {} - NOT found in manyscansfolders {}'.format(refscan, survexblock.survexfile.path)
|
|
||||||
print((insp+message))
|
|
||||||
models.DataIssue.objects.create(parser='survex', message=message)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# detect the star command
|
print(insp+" - MEM:{:.3f} Reading. parent:{} <> {} ".format(models.get_process_memory(),survexblock.survexfile.path,survexfile.path))
|
||||||
mstar = rx_star.match(sline)
|
stamp = datetime.now()
|
||||||
if not mstar:
|
lineno = 0
|
||||||
if "from" in stardata:
|
|
||||||
LoadSurvexLineLeg(survexblock, stardata, sline, comment, survexfile.cave)
|
|
||||||
pass
|
|
||||||
elif stardata["type"] == "passage":
|
|
||||||
LoadSurvexLinePassage(survexblock, stardata, sline, comment)
|
|
||||||
#Missing "station" in stardata.
|
|
||||||
continue
|
|
||||||
|
|
||||||
# detect the star command
|
sys.stderr.flush();
|
||||||
cmd, line = mstar.groups()
|
self.callcount +=1
|
||||||
cmd = cmd.lower()
|
if self.callcount >=10:
|
||||||
if re.match("include$(?i)", cmd):
|
self.callcount=0
|
||||||
includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line)))
|
print(".", file=sys.stderr,end='')
|
||||||
print((insp+' - Include path found, including - ' + includepath))
|
|
||||||
# Try to find the cave in the DB. if not, use the string as before
|
|
||||||
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
|
|
||||||
if path_match:
|
|
||||||
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
|
|
||||||
print((insp+' - Match in DB (i) for cave {}.'.format(pos_cave)))
|
|
||||||
cave = models_caves.getCaveByReference(pos_cave)
|
|
||||||
if cave:
|
|
||||||
survexfile.cave = cave
|
|
||||||
else:
|
|
||||||
print((insp+' - NO Match in DB (i) for a cave for {}'.format(includepath)))
|
|
||||||
|
|
||||||
includesurvexfile = models_survex.SurvexFile(path=includepath)
|
# Try to find the cave in the DB if not use the string as before
|
||||||
includesurvexfile.save()
|
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", survexblock.survexfile.path)
|
||||||
includesurvexfile.SetDirectory()
|
if path_match:
|
||||||
if includesurvexfile.exists():
|
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
|
||||||
survexblock.save()
|
cave = models_caves.getCaveByReference(pos_cave)
|
||||||
fininclude = includesurvexfile.OpenFile()
|
if cave:
|
||||||
insp += "> "
|
survexfile.cave = cave
|
||||||
RecursiveLoad(survexblock, includesurvexfile, fininclude)
|
svxlines = ''
|
||||||
#--------------------------------------------------------
|
svxlines = fin.read().splitlines()
|
||||||
fininclude.close()
|
# cannot close file now as it may be recursively called with the same file id fin if nested *begin
|
||||||
insp = insp[2:]
|
# occurs.
|
||||||
else:
|
for svxline in svxlines:
|
||||||
print((insp+' ! ERROR *include file not found for %s' % includesurvexfile))
|
lineno += 1
|
||||||
|
# break the line at the comment
|
||||||
|
sline, comment = self.rx_comment.match(svxline.strip()).groups()
|
||||||
|
# detect ref line pointing to the scans directory
|
||||||
|
mref = comment and self.rx_ref.match(comment)
|
||||||
|
if mref:
|
||||||
|
yr, letterx, wallet = mref.groups()
|
||||||
|
if not letterx:
|
||||||
|
letterx = ""
|
||||||
|
else:
|
||||||
|
letterx = "X"
|
||||||
|
if len(wallet)<2:
|
||||||
|
wallet = "0" + wallet
|
||||||
|
refscan = "%s#%s%s" % (yr, letterx, wallet )
|
||||||
|
manyscansfolders = models_survex.ScansFolder.objects.filter(walletname=refscan)
|
||||||
|
if manyscansfolders:
|
||||||
|
survexblock.scansfolder = manyscansfolders[0]
|
||||||
|
survexblock.save()
|
||||||
|
else:
|
||||||
|
message = ' ! Wallet ; ref {} - NOT found in manyscansfolders {}'.format(refscan, survexblock.survexfile.path)
|
||||||
|
print((insp+message))
|
||||||
|
models.DataIssue.objects.create(parser='survex', message=message)
|
||||||
|
|
||||||
elif re.match("begin$(?i)", cmd):
|
# This whole section should be moved if we can have *QM become a proper survex command
|
||||||
# On a *begin statement we start a new survexblock.
|
# Spec of QM in SVX files, currently commented out need to add to survex
|
||||||
# There should not be any *include inside a begin-end block, so this is a simple
|
# needs to match self.rx_qm
|
||||||
# load not a recursive fileload. But there may be many blocks nested to any depth in one file.
|
# ;Serial number grade(A/B/C/D/X) nearest-station resolution-station description
|
||||||
if line:
|
# ;QM1 a hobnob_hallway_2.42 hobnob-hallway_3.42 junction of keyhole passage
|
||||||
newsvxpath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line))
|
# ;QM1 a hobnob_hallway_2.42 - junction of keyhole passage
|
||||||
# Try to find the cave in the DB if not use the string as before
|
qmline = comment and self.rx_qm.match(comment)
|
||||||
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", newsvxpath)
|
if qmline:
|
||||||
|
qm_no = qmline.group(1)
|
||||||
|
qm_grade = qmline.group(2)
|
||||||
|
qm_from_section = qmline.group(3)
|
||||||
|
qm_from_station = qmline.group(4)
|
||||||
|
qm_resolve_section = qmline.group(6)
|
||||||
|
qm_resolve_station = qmline.group(7)
|
||||||
|
qm_notes = qmline.group(8)
|
||||||
|
|
||||||
|
# print(insp+'Cave - %s' % survexfile.cave)
|
||||||
|
# print(insp+'QM no %d' % int(qm_no))
|
||||||
|
# print(insp+'QM grade %s' % qm_grade)
|
||||||
|
# print(insp+'QM section %s' % qm_from_section)
|
||||||
|
# print(insp+'QM station %s' % qm_from_station)
|
||||||
|
# print(insp+'QM res section %s' % qm_resolve_section)
|
||||||
|
# print(insp+'QM res station %s' % qm_resolve_station)
|
||||||
|
# print(insp+'QM notes %s' % qm_notes)
|
||||||
|
|
||||||
|
# If the QM isn't resolved (has a resolving station) then load it
|
||||||
|
if not qm_resolve_section or qm_resolve_section != '-' or qm_resolve_section != 'None':
|
||||||
|
from_section = models_survex.SurvexBlock.objects.filter(name=qm_from_section)
|
||||||
|
# If we can find a section (survex note chunck, named)
|
||||||
|
if len(from_section) > 0:
|
||||||
|
from_station = models_survex.SurvexStation.objects.filter(block=from_section[0], name=qm_from_station)
|
||||||
|
# If we can find a from station then we have the nearest station and can import it
|
||||||
|
if len(from_station) > 0:
|
||||||
|
qm = models_caves.QM.objects.create(number=qm_no,
|
||||||
|
nearest_station=from_station[0],
|
||||||
|
grade=qm_grade.upper(),
|
||||||
|
location_description=qm_notes)
|
||||||
|
else:
|
||||||
|
# print(insp+' - QM found but resolved')
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not sline:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# detect the star ref command
|
||||||
|
mstar = self.rx_starref.match(sline)
|
||||||
|
if mstar:
|
||||||
|
yr,letterx,wallet = mstar.groups()
|
||||||
|
if not letterx:
|
||||||
|
letterx = ""
|
||||||
|
else:
|
||||||
|
letterx = "X"
|
||||||
|
if len(wallet)<2:
|
||||||
|
wallet = "0" + wallet
|
||||||
|
assert (int(yr)>1960 and int(yr)<2039), "Wallet year out of bounds: %s" % yr
|
||||||
|
assert (int(wallet)<100), "Wallet number more than 100: %s" % wallet
|
||||||
|
refscan = "%s#%s%s" % (yr, letterx, wallet)
|
||||||
|
manyscansfolders = models_survex.ScansFolder.objects.filter(walletname=refscan)
|
||||||
|
if manyscansfolders:
|
||||||
|
survexblock.scansfolder = manyscansfolders[0]
|
||||||
|
survexblock.save()
|
||||||
|
else:
|
||||||
|
message = ' ! Wallet *REF {} - NOT found in manyscansfolders {}'.format(refscan, survexblock.survexfile.path)
|
||||||
|
print((insp+message))
|
||||||
|
models.DataIssue.objects.create(parser='survex', message=message)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# detect the star command
|
||||||
|
mstar = self.rx_star.match(sline)
|
||||||
|
if not mstar:
|
||||||
|
if "from" in stardata:
|
||||||
|
self.LoadSurvexLineLeg(survexblock, stardata, sline, comment)
|
||||||
|
pass
|
||||||
|
elif stardata["type"] == "passage":
|
||||||
|
pass
|
||||||
|
#self.LoadSurvexLinePassage(survexblock, stardata, sline, comment)
|
||||||
|
#Missing "station" in stardata.
|
||||||
|
continue
|
||||||
|
|
||||||
|
# detect the star command
|
||||||
|
cmd, line = mstar.groups()
|
||||||
|
cmd = cmd.lower()
|
||||||
|
if re.match("include$(?i)", cmd):
|
||||||
|
includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line)))
|
||||||
|
print((insp+' - Include path found, including - ' + includepath))
|
||||||
|
# Try to find the cave in the DB. if not, use the string as before
|
||||||
|
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
|
||||||
if path_match:
|
if path_match:
|
||||||
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
|
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
|
||||||
# print(insp+pos_cave)
|
print((insp+' - Match in DB (i) for cave {}.'.format(pos_cave)))
|
||||||
cave = models_caves.getCaveByReference(pos_cave)
|
cave = models_caves.getCaveByReference(pos_cave)
|
||||||
if cave:
|
if cave:
|
||||||
survexfile.cave = cave
|
survexfile.cave = cave
|
||||||
else:
|
else:
|
||||||
print((insp+' - No match (b) for %s' % newsvxpath))
|
print((insp+' - NO Match in DB (i) for a cave for {}'.format(includepath)))
|
||||||
|
|
||||||
previousnlegs = survexlegsnumber
|
includesurvexfile = models_survex.SurvexFile(path=includepath)
|
||||||
name = line.lower()
|
includesurvexfile.save()
|
||||||
print(insp+' - Begin found for:{}, creating new SurvexBlock '.format(name))
|
includesurvexfile.SetDirectory()
|
||||||
survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock,
|
if includesurvexfile.exists():
|
||||||
survexpath=survexblock.survexpath+"."+name,
|
|
||||||
cave=survexfile.cave, survexfile=survexfile,
|
|
||||||
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
|
|
||||||
survexblockdown.save()
|
|
||||||
survexblock.save()
|
|
||||||
survexblock = survexblockdown
|
|
||||||
print(insp+" - ENTERING nested *begin/*end block: {}".format(name))
|
|
||||||
insp += "> "
|
|
||||||
RecursiveLoad(survexblockdown, survexfile, fin)
|
|
||||||
#--------------------------------------------------------
|
|
||||||
# do not close the file as there may be more blocks in this one
|
|
||||||
# and it is re-read afresh with every nested begin-end block.
|
|
||||||
insp = insp[2:]
|
|
||||||
else:
|
|
||||||
iblankbegins += 1
|
|
||||||
|
|
||||||
elif re.match("end$(?i)", cmd):
|
|
||||||
if iblankbegins:
|
|
||||||
print(insp+" - RETURNING from nested *begin/*end block: {}".format(line))
|
|
||||||
iblankbegins -= 1
|
|
||||||
else:
|
|
||||||
legsinblock = survexlegsnumber - previousnlegs
|
|
||||||
print(insp+" - LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber))
|
|
||||||
survexblock.legsall = legsinblock
|
|
||||||
survexblock.save()
|
|
||||||
endstamp = datetime.now()
|
|
||||||
timetaken = endstamp - stamp
|
|
||||||
return
|
|
||||||
|
|
||||||
elif re.match("date$(?i)", cmd):
|
|
||||||
if len(line) == 10:
|
|
||||||
survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone())
|
|
||||||
expeditions = models.Expedition.objects.filter(year=line[:4])
|
|
||||||
if expeditions:
|
|
||||||
assert len(expeditions) == 1
|
|
||||||
survexblock.expedition = expeditions[0]
|
|
||||||
survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date)
|
|
||||||
survexblock.save()
|
survexblock.save()
|
||||||
|
fininclude = includesurvexfile.OpenFile()
|
||||||
|
self.survexlegsnumber = blocklegs
|
||||||
|
self.insp += "> "
|
||||||
|
self.RecursiveLoad(survexblock, includesurvexfile, fininclude)
|
||||||
|
#--------------------------------------------------------
|
||||||
|
fininclude.close()
|
||||||
|
self.insp = self.insp[2:]
|
||||||
|
insp = self.insp
|
||||||
|
blocklegs = self.survexlegsnumber
|
||||||
|
else:
|
||||||
|
print((insp+' ! ERROR *include file not found for %s' % includesurvexfile))
|
||||||
|
|
||||||
elif re.match("team$(?i)", cmd):
|
elif re.match("begin$(?i)", cmd):
|
||||||
pass
|
# On a *begin statement we start a new survexblock.
|
||||||
# print(insp+' - Team found: ')
|
# There should not be any *include inside a begin-end block, so this is a simple
|
||||||
mteammember = rx_team.match(line)
|
# load not a recursive fileload. But there may be many blocks nested to any depth in one file.
|
||||||
if mteammember:
|
if line:
|
||||||
for tm in rx_person.split(mteammember.group(2)):
|
newsvxpath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line))
|
||||||
if tm:
|
# Try to find the cave in the DB if not use the string as before
|
||||||
personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
|
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", newsvxpath)
|
||||||
if (personexpedition, tm) not in teammembers:
|
if path_match:
|
||||||
teammembers.append((personexpedition, tm))
|
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
|
||||||
personrole = models_survex.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
|
# print(insp+pos_cave)
|
||||||
personrole.expeditionday = survexblock.expeditionday
|
cave = models_caves.getCaveByReference(pos_cave)
|
||||||
if personexpedition:
|
if cave:
|
||||||
personrole.person=personexpedition.person
|
survexfile.cave = cave
|
||||||
personrole.save()
|
else:
|
||||||
|
print((insp+' - No match (b) for %s' % newsvxpath))
|
||||||
|
|
||||||
elif cmd == "title":
|
previousnlegs = blocklegs
|
||||||
# unused in troggle.
|
name = line.lower()
|
||||||
#survextitle = models_survex.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexfile.cave)
|
print(insp+' - Begin found for:{}, creating new SurvexBlock '.format(name))
|
||||||
#survextitle.save()
|
survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock,
|
||||||
pass
|
survexpath=survexblock.survexpath+"."+name,
|
||||||
|
cave=survexfile.cave, survexfile=survexfile,
|
||||||
|
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
|
||||||
|
survexblockdown.save()
|
||||||
|
survexblock.save()
|
||||||
|
survexblock = survexblockdown
|
||||||
|
print(insp+" - ENTERING nested *begin/*end block: {}".format(name))
|
||||||
|
self.survexlegsnumber = blocklegs
|
||||||
|
self.insp += "> "
|
||||||
|
self.RecursiveLoad(survexblockdown, survexfile, fin)
|
||||||
|
#--------------------------------------------------------
|
||||||
|
# do not close the file as there may be more blocks in this one
|
||||||
|
# and it is re-read afresh with every nested begin-end block.
|
||||||
|
self.insp = self.insp[2:]
|
||||||
|
insp = self.insp
|
||||||
|
blocklegs = self.survexlegsnumber
|
||||||
|
else:
|
||||||
|
iblankbegins += 1
|
||||||
|
|
||||||
elif cmd == "require":
|
elif re.match("end$(?i)", cmd):
|
||||||
# should we check survex version available for processing?
|
if iblankbegins:
|
||||||
pass
|
print(insp+" - RETURNING from nested *begin/*end block: {}".format(line))
|
||||||
|
iblankbegins -= 1
|
||||||
|
else:
|
||||||
|
legsinblock = self.survexlegsnumber - previousnlegs
|
||||||
|
print(insp+" - LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,self.survexlegsnumber))
|
||||||
|
survexblock.legsall = legsinblock
|
||||||
|
survexblock.save()
|
||||||
|
endstamp = datetime.now()
|
||||||
|
timetaken = endstamp - stamp
|
||||||
|
return
|
||||||
|
|
||||||
elif cmd == "data":
|
elif re.match("date$(?i)", cmd):
|
||||||
ls = line.lower().split()
|
if len(line) == 10:
|
||||||
stardata = { "type":ls[0] }
|
survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone())
|
||||||
for i in range(0, len(ls)):
|
expeditions = models.Expedition.objects.filter(year=line[:4])
|
||||||
stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1
|
if expeditions:
|
||||||
if ls[0] in ["normal", "cartesian", "nosurvey"]:
|
assert len(expeditions) == 1
|
||||||
assert (("from" in stardata and "to" in stardata) or "station" in stardata), line
|
survexblock.expedition = expeditions[0]
|
||||||
elif ls[0] == "default":
|
survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date)
|
||||||
stardata = stardatadefault
|
survexblock.save()
|
||||||
|
|
||||||
|
elif re.match("team$(?i)", cmd):
|
||||||
|
pass
|
||||||
|
# print(insp+' - Team found: ')
|
||||||
|
mteammember = self.rx_team.match(line)
|
||||||
|
if mteammember:
|
||||||
|
for tm in self.rx_person.split(mteammember.group(2)):
|
||||||
|
if tm:
|
||||||
|
personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
|
||||||
|
if (personexpedition, tm) not in teammembers:
|
||||||
|
teammembers.append((personexpedition, tm))
|
||||||
|
personrole = models_survex.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
|
||||||
|
personrole.expeditionday = survexblock.expeditionday
|
||||||
|
if personexpedition:
|
||||||
|
personrole.person=personexpedition.person
|
||||||
|
personrole.save()
|
||||||
|
|
||||||
|
elif cmd == "title":
|
||||||
|
# unused in troggle today - but will become text list on SurvexBlock
|
||||||
|
pass
|
||||||
|
|
||||||
|
elif cmd == "require":
|
||||||
|
# should we check survex version available for processing?
|
||||||
|
pass
|
||||||
|
|
||||||
|
elif cmd == "data":
|
||||||
|
ls = line.lower().split()
|
||||||
|
stardata = { "type":ls[0] }
|
||||||
|
for i in range(0, len(ls)):
|
||||||
|
stardata[self.stardataparamconvert.get(ls[i], ls[i])] = i - 1
|
||||||
|
if ls[0] in ["normal", "cartesian", "nosurvey"]:
|
||||||
|
assert (("from" in stardata and "to" in stardata) or "station" in stardata), line
|
||||||
|
elif ls[0] == "default":
|
||||||
|
stardata = self.stardatadefault
|
||||||
|
else:
|
||||||
|
assert ls[0] == "passage", line
|
||||||
|
|
||||||
|
elif cmd == "equate":
|
||||||
|
#LoadSurvexEquate(survexblock, line)
|
||||||
|
pass
|
||||||
|
|
||||||
|
elif cmd == "set" and re.match("names(?i)", line):
|
||||||
|
pass
|
||||||
|
elif cmd == "flags":
|
||||||
|
# Here we could set on/off 'splay', 'not splay', 'surface', 'not surface', or 'duplicate'
|
||||||
|
# but this data is only used for sense-checking not to actually calculate anything important
|
||||||
|
pass
|
||||||
|
elif cmd == "fix":
|
||||||
|
# troggle does not use survex stations except for entrances which are loaded elsewhere
|
||||||
|
pass
|
||||||
|
elif cmd in ["alias", "calibrate", "cs","entrance", "export", "case",
|
||||||
|
"declination", "infer","instrument", "sd", "units"]:
|
||||||
|
# we ignore all these, which is fine.
|
||||||
|
pass
|
||||||
else:
|
else:
|
||||||
assert ls[0] == "passage", line
|
if cmd not in ["include", "data", "flags", "title", "set", "ref"]:
|
||||||
|
message = "! Bad svx command: [*{}] {} ({}) {}".format(cmd, line, survexblock, survexblock.survexfile.path)
|
||||||
elif cmd == "equate":
|
print((insp+message))
|
||||||
#LoadSurvexEquate(survexblock, line)
|
models.DataIssue.objects.create(parser='survex', message=message)
|
||||||
pass
|
else:
|
||||||
|
message = "! Unparsed [*{}]: '{}' {}".format(cmd, line, survexblock.survexfile.path)
|
||||||
elif cmd == "set" and re.match("names(?i)", line):
|
print((insp+message))
|
||||||
pass
|
models.DataIssue.objects.create(parser='survex', message=message)
|
||||||
elif cmd == "flags":
|
|
||||||
# Here we could set on/off 'splay', 'not splay', 'surface', 'not surface', or 'duplicate'
|
|
||||||
# but this data is only used for sense-checking not to actually calculate anything important
|
|
||||||
pass
|
|
||||||
elif cmd == "fix":
|
|
||||||
# troggle does not use survex stations
|
|
||||||
#survexblock.MakeSurvexStation(line.split()[0])
|
|
||||||
pass
|
|
||||||
elif cmd in ["alias", "calibrate", "cs","entrance", "export", "case",
|
|
||||||
"declination", "infer","instrument", "sd", "units"]:
|
|
||||||
# we ignore all these, which is fine.
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
if cmd not in ["include", "data", "flags", "title", "set", "ref"]:
|
|
||||||
message = "! Bad svx command: [*{}] {} ({}) {}".format(cmd, line, survexblock, survexblock.survexfile.path)
|
|
||||||
print((insp+message))
|
|
||||||
models.DataIssue.objects.create(parser='survex', message=message)
|
|
||||||
else:
|
|
||||||
message = "! Unparsed [*{}]: '{}' {}".format(cmd, line, survexblock.survexfile.path)
|
|
||||||
print((insp+message))
|
|
||||||
models.DataIssue.objects.create(parser='survex', message=message)
|
|
||||||
|
|
||||||
endstamp = datetime.now()
|
|
||||||
timetaken = endstamp - stamp
|
|
||||||
# print(insp+' - Time to process: ' + str(timetaken))
|
|
||||||
|
|
||||||
def get_process_memory():
|
|
||||||
usage=resource.getrusage(resource.RUSAGE_SELF)
|
|
||||||
return usage[2]/1024.0
|
|
||||||
|
|
||||||
|
|
||||||
def FindAndLoadAllSurvex(survexblockroot, survexfileroot):
|
def FindAndLoadAllSurvex(survexblockroot, survexfileroot):
|
||||||
"""Follows the *include links recursively to find files
|
"""Follows the *include links recursively to find files
|
||||||
"""
|
"""
|
||||||
print(' - redirecting stdout to loadsurvexblks.log...')
|
print(' - redirecting stdout to svxblks.log...')
|
||||||
stdout_orig = sys.stdout
|
stdout_orig = sys.stdout
|
||||||
# Redirect sys.stdout to the file
|
# Redirect sys.stdout to the file
|
||||||
sys.stdout = open('loadsurvexblks.log', 'w')
|
sys.stdout = open('svxblks.log', 'w')
|
||||||
|
|
||||||
|
svxl = LoadSurvex()
|
||||||
|
|
||||||
finroot = survexfileroot.OpenFile()
|
finroot = survexfileroot.OpenFile()
|
||||||
RecursiveLoad(survexblockroot, survexfileroot, finroot)
|
svxl.RecursiveLoad(survexblockroot, survexfileroot, finroot)
|
||||||
finroot.close()
|
finroot.close()
|
||||||
|
|
||||||
# Close the logging file
|
survexlegsnumber = svxl.survexlegsnumber
|
||||||
|
survexlegsalllength = svxl.survexlegsalllength
|
||||||
|
|
||||||
|
# Close the logging file, Restore sys.stdout to our old saved file handle
|
||||||
sys.stdout.close()
|
sys.stdout.close()
|
||||||
print("+", file=sys.stderr)
|
print("+", file=sys.stderr)
|
||||||
sys.stderr.flush();
|
sys.stderr.flush();
|
||||||
# Restore sys.stdout to our old saved file handler
|
|
||||||
sys.stdout = stdout_orig
|
sys.stdout = stdout_orig
|
||||||
|
return (survexlegsnumber, survexlegsalllength)
|
||||||
|
|
||||||
|
|
||||||
def LoadAllSurvexBlocks():
|
def LoadAllSurvexBlocks():
|
||||||
global survexlegsalllength
|
|
||||||
global survexlegsnumber
|
|
||||||
|
|
||||||
print(' - Flushing All Survex Blocks...')
|
print(' - Flushing All Survex Blocks...')
|
||||||
|
|
||||||
models_survex.SurvexBlock.objects.all().delete()
|
models_survex.SurvexBlock.objects.all().delete()
|
||||||
models_survex.SurvexFile.objects.all().delete()
|
models_survex.SurvexFile.objects.all().delete()
|
||||||
models_survex.SurvexDirectory.objects.all().delete()
|
models_survex.SurvexDirectory.objects.all().delete()
|
||||||
# models_survex.SurvexEquate.objects.all().delete()
|
|
||||||
# models_survex.SurvexTitle.objects.all().delete()
|
|
||||||
models_survex.SurvexPersonRole.objects.all().delete()
|
models_survex.SurvexPersonRole.objects.all().delete()
|
||||||
models_survex.SurvexStation.objects.all().delete()
|
models_survex.SurvexStation.objects.all().delete()
|
||||||
|
|
||||||
@ -497,7 +492,10 @@ def LoadAllSurvexBlocks():
|
|||||||
survexblockroot.save()
|
survexblockroot.save()
|
||||||
|
|
||||||
print(' - Loading All Survex Blocks...')
|
print(' - Loading All Survex Blocks...')
|
||||||
FindAndLoadAllSurvex(survexblockroot, survexfileroot)
|
memstart = models.get_process_memory()
|
||||||
|
survexlegsnumber, survexlegsalllength = FindAndLoadAllSurvex(survexblockroot, survexfileroot)
|
||||||
|
memend = models.get_process_memory()
|
||||||
|
print(" - MEMORY start:{:.3f} MB end:{:.3f} MB increase={:.3f} MB",format(memstart,memend,memend-memstart))
|
||||||
|
|
||||||
survexblockroot.totalleglength = survexlegsalllength
|
survexblockroot.totalleglength = survexlegsalllength
|
||||||
survexblockroot.legsall = survexlegsnumber
|
survexblockroot.legsall = survexlegsnumber
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user