xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Part one of getting troggle to work with django 1.10

Browse Source 
Major rework of how survex is processed
This commit is contained in:
Sam Wenham
2019-07-16 00:07:37 +01:00
parent a26109cb30
commit 08a41941f9
38 changed files with 450 additions and 918 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
parsers/survex.py
View File

@@ -5,9 +5,14 @@ import troggle.settings as settings
from subprocess import call, Popen, PIPE
from troggle.parsers.people import GetPersonExpeditionNameLookup
from django.utils.timezone import get_current_timezone
from django.utils.timezone import make_aware
import re
import os
from datetime import datetime
line_leg_regex = re.compile(r"[\d\-+.]+$")
def LoadSurvexLineLeg(survexblock, stardata, sline, comment):
ls = sline.lower().split()
@@ -53,8 +58,8 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment):
survexleg.compass = 1000
survexleg.clino = -90.0
else:
assert re.match(r"[\d\-+.]+$", lcompass), ls
assert re.match(r"[\d\-+.]+$", lclino) and lclino != "-", ls
assert line_leg_regex.match(lcompass), ls
assert line_leg_regex.match(lclino) and lclino != "-", ls
survexleg.compass = float(lcompass)
survexleg.clino = float(lclino)
@@ -80,11 +85,16 @@ def LoadSurvexEquate(survexblock, sline):
def LoadSurvexLinePassage(survexblock, stardata, sline, comment):
pass
stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"}
regex_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$")
regex_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(\d+)')
regex_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$')
regex_team = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)")
regex_team_member = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)")
def RecursiveLoad(survexblock, survexfile, fin, textlines):
iblankbegins = 0
text = [ ]
@@ -92,37 +102,47 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
teammembers = [ ]
# uncomment to print out all files during parsing
print("Reading file: " + survexblock.survexfile.path)
print(" - Reading file: " + survexblock.survexfile.path)
stamp = datetime.now()
lineno = 0
while True:
svxline = fin.readline().decode("latin1")
if not svxline:
print(' - Not survex')
return
textlines.append(svxline)
lineno += 1
# print(' - Line: %d' % lineno)
# break the line at the comment
sline, comment = re.match(r"([^;]*?)\s*(?:;\s*(.*))?\n?$", svxline.strip()).groups()
sline, comment = regex_comment.match(svxline.strip()).groups()
# detect ref line pointing to the scans directory
mref = comment and re.match(r'.*?ref.*?(\d+)\s*#\s*(\d+)', comment)
mref = comment and regex_ref.match(comment)
if mref:
refscan = "%s#%s" % (mref.group(1), mref.group(2))
survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan)
if survexscansfolders:
survexblock.survexscansfolder = survexscansfolders[0]
#survexblock.refscandir = "%s/%s%%23%s" % (mref.group(1), mref.group(1), mref.group(2))
survexblock.save()
survexblock.save()
continue
if not sline:
continue
# detect the star command
mstar = re.match(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$', sline)
mstar = regex_star.match(sline)
if not mstar:
if "from" in stardata:
LoadSurvexLineLeg(survexblock, stardata, sline, comment)
# print(' - From: ')
#print(stardata)
elif stardata["type"] == "passage":
LoadSurvexLinePassage(survexblock, stardata, sline, comment)
# print(' - Passage: ')
#Missing "station" in stardata.
continue
@@ -131,7 +151,19 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
cmd = cmd.lower()
if re.match("include$(?i)", cmd):
includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line))
includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave)
print(' - Include file found including - ' + includepath)
# Try to find the cave in the DB if not use the string as before
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
if path_match:
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
print(pos_cave)
cave = models.getCaveByReference(pos_cave)
if not cave:
cave = survexfile.cave
else:
print('No match for %s' % includepath)
cave = survexfile.cave
includesurvexfile = models.SurvexFile(path=includepath, cave=cave)
includesurvexfile.save()
includesurvexfile.SetDirectory()
if includesurvexfile.exists():
@@ -141,6 +173,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
elif re.match("begin$(?i)", cmd):
if line:
name = line.lower()
#print(' - Begin found for: ' + name)
survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile, totalleglength=0.0)
survexblockdown.save()
textlinesdown = [ ]
@@ -154,11 +187,16 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
else:
survexblock.text = "".join(textlines)
survexblock.save()
# print(' - End found: ')
endstamp = datetime.now()
timetaken = endstamp - stamp
# print(' - Time to process: ' + str(timetaken))
return
elif re.match("date$(?i)", cmd):
if len(line) == 10:
survexblock.date = re.sub(r"\.", "-", line)
#print(' - Date found: ' + line)
survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone())
expeditions = models.Expedition.objects.filter(year=line[:4])
if expeditions:
assert len(expeditions) == 1
@@ -167,9 +205,11 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
survexblock.save()
elif re.match("team$(?i)", cmd):
mteammember = re.match(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line)
pass
# print(' - Team found: ')
mteammember = regex_team.match(line)
if mteammember:
for tm in re.split(r" and | / |, | & | \+ |^both$|^none$(?i)", mteammember.group(2)):
for tm in regex_team_member.split(mteammember.group(2)):
if tm:
personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
if (personexpedition, tm) not in teammembers:
@@ -181,6 +221,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
personrole.save()
elif cmd == "title":
#print(' - Title found: ')
survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave)
survextitle.save()
@@ -189,8 +230,11 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
pass
elif cmd == "data":
#print(' - Data found: ')
ls = line.lower().split()
stardata = { "type":ls[0] }
#print(' - Star data: ', stardata)
#print(ls)
for i in range(0, len(ls)):
stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1
if ls[0] in ["normal", "cartesian", "nosurvey"]:
@@ -201,33 +245,47 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
assert ls[0] == "passage", line
elif cmd == "equate":
#print(' - Equate found: ')
LoadSurvexEquate(survexblock, line)
elif cmd == "fix":
#print(' - Fix found: ')
survexblock.MakeSurvexStation(line.split()[0])
else:
#print(' - Stuff')
if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument",
"calibrate", "set", "infer", "alias", "ref", "cs", "declination", "case"]:
print("Unrecognised command in line:", cmd, line, survexblock, survexblock.survexfile.path)
endstamp = datetime.now()
timetaken = endstamp - stamp
# print(' - Time to process: ' + str(timetaken))
def ReloadSurvexCave(survex_cave, area):
print(survex_cave, area)
cave = models.Cave.objects.get(kataster_number=survex_cave, area__short_name=area)
print(cave)
#cave = models.Cave.objects.get(kataster_number=survex_cave)
print(' - Area: ' + str(area) + ' Cave: ' + str(survex_cave))
filt_cave = models.Cave.objects.filter(kataster_number=survex_cave, area__short_name=area)[:1]
if len(filt_cave) < 1:
filt_cave = models.Cave.objects.filter(unofficial_number=survex_cave, area__short_name=area)[:1]
cave = filt_cave[0]
print(' - ' + str(cave))
cave.survexblock_set.all().delete()
cave.survexfile_set.all().delete()
cave.survexdirectory_set.all().delete()
file_stamp_start = datetime.now()
survexfile = models.SurvexFile(path="caves-" + cave.kat_area() + "/" + survex_cave + "/" + survex_cave, cave=cave)
survexfile.save()
survexfile.SetDirectory()
survexblockroot = models.SurvexBlock(name="root", survexpath="caves-" + cave.kat_area(), begin_char=0, cave=cave, survexfile=survexfile, totalleglength=0.0)
survexblockroot.save()
fin = survexfile.OpenFile()
file_stamp_end = datetime.now()
file_time = file_stamp_end - file_stamp_start
print(' - Files time to process: ' + str(file_time))
textlines = [ ]
RecursiveLoad(survexblockroot, survexfile, fin, textlines)
survexblockroot.text = "".join(textlines)
@@ -268,10 +326,18 @@ def LoadAllSurvexBlocks():
print(" - Reloading all caves")
caves = models.Cave.objects.all()
for cave in caves:
rec_stamp_start = datetime.now()
if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves-" + cave.kat_area(), cave.kataster_number)):
if cave.kataster_number not in ['40']:
print("loading", cave, cave.kat_area())
ReloadSurvexCave(cave.kataster_number, cave.kat_area())
print(" - Loading " + str(cave) + " " + cave.kat_area())
#ReloadSurvexCave(cave.kataster_number, cave.kat_area())
rec_stamp_end = datetime.now()
if cave.unofficial_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves-" + cave.kat_area(), cave.unofficial_number)):
print(" - Loading " + str(cave) + " " + cave.kat_area())
#ReloadSurvexCave(cave.unofficial_number, cave.kat_area())
timetaken = rec_stamp_end - rec_stamp_start
# print(' - Time to process: ' + str(timetaken))
# print('--------')
poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
@@ -286,7 +352,7 @@ def LoadPos():
posfile = open("%s%s.pos" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME))
posfile.readline() #Drop header
for line in posfile.readlines():
r = poslineregex.match(line)
r = poslineregex.match(line)
if r:
x, y, z, name = r.groups()
try: