xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

fixing typos and changes in importing

Browse Source
This commit is contained in:
Philip Sargent 2021-04-13 01:13:08 +01:00
parent 7bc73d1ca8
commit 267741fa8b
5 changed files with 70 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/unused.py
View File

@ -1,5 +1,4 @@
import sys
import random
import re
import logging

1
core/utils.py
View File

@ -4,6 +4,7 @@ import datetime
import logging
import re
import resource
import random
from subprocess import call
from urllib.parse import urljoin

2
databaseReset.py
View File

@ -33,7 +33,7 @@ except:
raise
print(" - Memory footprint after loading Django: {:.3f} MB".format(resource.getrusage(resource.RUSAGE_SELF)[2]/1024.0))
from troggle.core.models import DataIssue
from troggle.core.models.troggle import DataIssue
import troggle.core.models.survex
from django.core import management

12
parsers/people.py
View File

@ -4,7 +4,7 @@ from unidecode import unidecode
from django.conf import settings
import troggle.core.models as models
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.utils import save_carefully
'''These functions do not match how the stand-alone folk script works. So the script produces an HTML file which has
@ -50,7 +50,7 @@ def LoadPersonsExpos():
lookupAttribs = {'year':year}
nonLookupAttribs = {'name':"CUCC expo %s" % year}
save_carefully(models.Expedition, lookupAttribs, nonLookupAttribs)
save_carefully(Expedition, lookupAttribs, nonLookupAttribs)
# make persons
print(" - Loading personexpeditions")
@ -84,17 +84,17 @@ def LoadPersonsExpos():
lookupAttribs={'first_name':firstname, 'last_name':(lastname or "")}
nonLookupAttribs={'is_vfho':vfho, 'fullname':fullname}
person, created = save_carefully(models.Person, lookupAttribs, nonLookupAttribs)
person, created = save_carefully(Person, lookupAttribs, nonLookupAttribs)
parseMugShotAndBlurb(personline=personline, header=header, person=person)
# make person expedition from table
for year, attended in list(zip(headers, personline))[5:]:
expedition = models.Expedition.objects.get(year=year)
expedition = Expedition.objects.get(year=year)
if attended == "1" or attended == "-1":
lookupAttribs = {'person':person, 'expedition':expedition}
nonLookupAttribs = {'nickname':nickname, 'is_guest':(personline[header["Guest"]] == "1")}
save_carefully(models.PersonExpedition, lookupAttribs, nonLookupAttribs)
save_carefully(PersonExpedition, lookupAttribs, nonLookupAttribs)
# used in other referencing parser functions
@ -110,7 +110,7 @@ def GetPersonExpeditionNameLookup(expedition):
duplicates = set()
#print("Calculating GetPersonExpeditionNameLookup for " + expedition.year)
personexpeditions = models.PersonExpedition.objects.filter(expedition=expedition)
personexpeditions = PersonExpedition.objects.filter(expedition=expedition)
htmlparser = HTMLParser()
for personexpedition in personexpeditions:
possnames = [ ]

124
parsers/survex.py
View File

@ -12,13 +12,13 @@ from django.utils.timezone import get_current_timezone
from django.utils.timezone import make_aware
import troggle.settings as settings
import troggle.core.models as models
import troggle.core.models.caves as models_caves
import troggle.core.models.survex as models_survex
from troggle.core.utils import get_process_memory, chaosmonkey
from troggle.parsers.people import GetPersonExpeditionNameLookup
from troggle.parsers.logbooks import GetCaveLookup
from troggle.core.views.caves import MapLocations
from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.models.survex import SurvexPersonRole, ScansFolder, SurvexDirectory, SurvexFile, SurvexBlock, SurvexStation
survexblockroot = None
ROOTBLOCK = "rootblock"
@ -129,11 +129,11 @@ class LoadingSurvex():
if cmd in ["include", "data", "flags", "title", "entrance","set", "units", "alias", "ref"]:
message = "! Unparsed [*{}]: '{}' {}".format(cmd, line, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
else:
message = "! Bad svx command: [*{}] {} ({}) {}".format(cmd, line, survexblock, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexTeam(self, survexblock, line):
"""Interpeting the *team fields means interpreting older style survex as well as current survex standard,
@ -154,7 +154,7 @@ class LoadingSurvex():
personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
if (personexpedition, tm) not in teammembers:
teammembers.append((personexpedition, tm))
personrole = models_survex.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
personrole = SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
personrole.save()
personrole.expeditionday = survexblock.expeditionday
if personexpedition:
@ -174,7 +174,7 @@ class LoadingSurvex():
else:
message = "! Bad *ALIAS: '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexUnits(self, survexblock, line):
# all for 4 survex files with measurements in feet. bugger.
@ -188,7 +188,7 @@ class LoadingSurvex():
if debugprint:
message = "! *UNITS NUMERICAL conversion [{}x] '{}' ({}) {}".format(factor, line, survexblock, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survexunits', message=message)
DataIssue.objects.create(parser='survexunits', message=message)
feet = re.match("(?i).*feet$",line)
metres = re.match("(?i).*(METRIC|METRES|METERS)$",line)
@ -199,7 +199,7 @@ class LoadingSurvex():
else:
message = "! *UNITS in YARDS!? - not converted '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survexunits', message=message)
DataIssue.objects.create(parser='survexunits', message=message)
def LoadSurvexDate(self, survexblock, line):
# we should make this a date RANGE for everything
@ -208,7 +208,7 @@ class LoadingSurvex():
if year in self.expos:
expo = self.expos[year]
else:
expeditions = models.Expedition.objects.filter(year=year)
expeditions = Expedition.objects.filter(year=year)
assert len(expeditions) == 1
expo= expeditions[0]
self.expos[year]= expo
@ -237,7 +237,7 @@ class LoadingSurvex():
else:
message = "! DATE unrecognised '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexLeg(self, survexblock, sline, comment):
"""This reads compass, clino and tape data but only keeps the tape lengths,
@ -300,7 +300,7 @@ class LoadingSurvex():
except:
message = ' ! datastar parsing incorrect in line %s in %s' % (ls, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survexleg', message=message)
DataIssue.objects.create(parser='survexleg', message=message)
survexleg.tape = invalid_tape
return
# e.g. '29/09' or '(06.05)' in the tape measurement
@ -313,19 +313,19 @@ class LoadingSurvex():
if debugprint:
message = " ! Units: Length scaled {}m '{}' in ({}) units:{} factor:{}x".format(tape, ls, survexblock.survexfile.path, self.units, self.unitsfactor)
print((self.insp+message))
models.DataIssue.objects.create(parser='survexleg', message=message)
DataIssue.objects.create(parser='survexleg', message=message)
if self.units =="feet":
tape = float(tape) / METRESINFEET
if debugprint:
message = " ! Units: converted to {:.3f}m from {} '{}' in ({})".format(tape, self.units, ls, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survexleg', message=message)
DataIssue.objects.create(parser='survexleg', message=message)
survexleg.tape = float(tape)
self.legsnumber += 1
except ValueError:
message = " ! Value Error: Tape misread in line'{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units)
print((self.insp+message))
models.DataIssue.objects.create(parser='survexleg', message=message)
DataIssue.objects.create(parser='survexleg', message=message)
survexleg.tape = invalid_tape
try:
survexblock.legslength += survexleg.tape
@ -333,14 +333,14 @@ class LoadingSurvex():
except ValueError:
message = " ! Value Error: Tape length not added '{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units)
print((self.insp+message))
models.DataIssue.objects.create(parser='survexleg', message=message)
DataIssue.objects.create(parser='survexleg', message=message)
try:
lcompass = ls[datastar["compass"]]
except:
message = ' ! Value Error: Compass not found in line %s in %s' % (ls, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survexleg', message=message)
DataIssue.objects.create(parser='survexleg', message=message)
lcompass = invalid_compass
try:
@ -350,7 +350,7 @@ class LoadingSurvex():
print((" datastar:", datastar))
print((" Line:", ls))
message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path)
models.DataIssue.objects.create(parser='survexleg', message=message)
DataIssue.objects.create(parser='survexleg', message=message)
lclino = invalid_clino
if lclino == "up":
@ -370,7 +370,7 @@ class LoadingSurvex():
print((" Line:", ls))
message = " ! Value Error: lcompass:'{}' line {} in '{}'".format(lcompass,
ls, survexblock.survexfile.path)
models.DataIssue.objects.create(parser='survexleg', message=message)
DataIssue.objects.create(parser='survexleg', message=message)
survexleg.compass = invalid_compass
# delete the object to save memory
@ -389,7 +389,7 @@ class LoadingSurvex():
if len(args)< 4:
message = " ! Empty or BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
return
argsgps = self.rx_argsref.match(args)
@ -398,7 +398,7 @@ class LoadingSurvex():
else:
message = " ! BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path)
print(self.insp+message)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
return
if not letterx:
@ -413,12 +413,12 @@ class LoadingSurvex():
if int(wallet)>100:
message = " ! Wallet *REF {} - too big in '{}'".format(refscan, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
except:
message = " ! Wallet *REF {} - not numeric in '{}'".format(refscan, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
manyscansfolders = models_survex.ScansFolder.objects.filter(walletname=refscan)
DataIssue.objects.create(parser='survex', message=message)
manyscansfolders = ScansFolder.objects.filter(walletname=refscan)
if manyscansfolders:
survexblock.scansfolder = manyscansfolders[0] # this is a ForeignKey field
print(manyscansfolders[0])
@ -426,11 +426,11 @@ class LoadingSurvex():
if len(manyscansfolders) > 1:
message = " ! Wallet *REF {} - {} scan folders from DB search in {}".format(refscan, len(manyscansfolders), survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
else:
message = " ! Wallet *REF '{}' - NOT found in DB search '{}'".format(refscan, survexblock.survexfile.path)
print((self.insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexQM(self, survexblock, qmline):
insp = self.insp
@ -465,11 +465,11 @@ class LoadingSurvex():
qm.save
# message = " ! QM{} '{}' CREATED in DB in '{}'".format(qm_no, qm_nearest,survexblock.survexfile.path)
# print(insp+message)
# models.DataIssue.objects.create(parser='survex', message=message)
# DataIssue.objects.create(parser='survex', message=message)
except:
message = " ! QM{} FAIL to create {} in'{}'".format(qm_no, qm_nearest,survexblock.survexfile.path)
print(insp+message)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexDataNormal(self,survexblock,args):
"""Sets the order for data elements in this and following blocks, e.g.
@ -498,7 +498,7 @@ class LoadingSurvex():
message = " ! - Unrecognised *data normal statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
return
else:
datastar = self.datastardefault
@ -516,13 +516,13 @@ class LoadingSurvex():
# message = " ! - *data {} blocks ignored. {}|{}" '{}' .format(ls[0].upper(), survexblock.name, survexblock.survexpath, args)
# print(message)
# print(message,file=sys.stderr)
# models.DataIssue.objects.create(parser='survex', message=message)
# DataIssue.objects.create(parser='survex', message=message)
self.datastar["type"] = ls[0]
else:
message = " ! - Unrecognised *data statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexFlags(self, args):
# Valid flags are DUPLICATE, SPLAY, and SURFACE, and a flag may be preceded with NOT to turn it off.
@ -587,7 +587,7 @@ class LoadingSurvex():
if not headpath:
return self.svxdirs[""]
if headpath.lower() not in self.svxdirs:
self.svxdirs[headpath.lower()] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile)
self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile)
self.svxdirs[headpath.lower()].save()
self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory
return self.svxdirs[headpath.lower()]
@ -603,7 +603,7 @@ class LoadingSurvex():
message = " ! {} is not a cave. (while creating '{}' sfile & sdirectory)".format(headpath, includelabel)
print("\n"+message)
print("\n"+message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexFile(self, svxid):
"""Creates SurvexFile in the database, and SurvexDirectory if needed
@ -622,7 +622,7 @@ class LoadingSurvex():
# print("{:2}{} - NEW survexfile:'{}'".format(self.depthbegin, depth, svxid))
headpath = os.path.dirname(svxid)
newfile = models_survex.SurvexFile(path=svxid)
newfile = SurvexFile(path=svxid)
newfile.save() # until we do this there is no internal id so no foreign key works
self.currentsurvexfile = newfile
newdirectory = self.GetSurvexDirectory(headpath)
@ -635,7 +635,7 @@ class LoadingSurvex():
message = " ! 'None' SurvexDirectory returned from GetSurvexDirectory({})".format(headpath)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
if cave:
newdirectory.cave = cave
@ -648,7 +648,7 @@ class LoadingSurvex():
message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(svxid)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
self.currentsurvexfile.save() # django insists on this although it is already saved !?
try:
newdirectory.save()
@ -864,7 +864,7 @@ class LoadingSurvex():
self.units = "metres"
self.currentpersonexped = []
printbegin()
newsurvexblock = models_survex.SurvexBlock(name=blkid, parent=survexblock,
newsurvexblock = SurvexBlock(name=blkid, parent=survexblock,
survexpath=pathlist,
cave=self.currentcave, survexfile=self.currentsurvexfile,
legsall=0, legslength=0.0)
@ -939,7 +939,7 @@ class LoadingSurvex():
message = " ! -ERROR *include command not expected here {}. Re-run a full Survex import.".format(path)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
else:
self.LoadSurvexFallThrough(survexblock, args, cmd)
@ -983,13 +983,13 @@ class LoadingSurvex():
print(message)
print(message,file=flinear)
print("\n"+message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
if self.svxfileslist.count(path) > 20:
message = " ! ERROR. Survex file already seen 20x. Probably an infinite loop so fix your *include statements that include this. Aborting. {}".format(path)
print(message)
print(message,file=flinear)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
return
self.svxfileslist.append(path)
@ -1027,7 +1027,7 @@ class LoadingSurvex():
print(message)
print(message,file=flinear)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop))
fcollate.write(";*edulcni {}\n".format(pop))
fininclude.close()
@ -1037,7 +1037,7 @@ class LoadingSurvex():
message = " ! ERROR *include file not found for:'{}'".format(includepath)
print(message)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
elif re.match("(?i)begin$", cmd):
self.depthbegin += 1
depth = " " * self.depthbegin
@ -1059,7 +1059,7 @@ class LoadingSurvex():
print(message)
print(message,file=flinear)
print(message,file=sys.stderr)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
self.depthbegin -= 1
pass
@ -1199,9 +1199,9 @@ def FindAndLoadSurvex(survexblockroot):
def MakeSurvexFileRoot():
"""Returns a file_object.path = SURVEX_TOPNAME associated with directory_object.path = SURVEX_DATA
"""
fileroot = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
fileroot = SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
fileroot.save()
directoryroot = models_survex.SurvexDirectory(path=settings.SURVEX_DATA, cave=None, primarysurvexfile=fileroot)
directoryroot = SurvexDirectory(path=settings.SURVEX_DATA, cave=None, primarysurvexfile=fileroot)
directoryroot.save()
fileroot.survexdirectory = directoryroot # i.e. SURVEX_DATA/SURVEX_TOPNAME
fileroot.save() # mutually dependent objects need a double-save like this
@ -1210,19 +1210,19 @@ def MakeSurvexFileRoot():
def LoadSurvexBlocks():
print(' - Flushing All Survex Blocks...')
models_survex.SurvexBlock.objects.all().delete()
models_survex.SurvexFile.objects.all().delete()
models_survex.SurvexDirectory.objects.all().delete()
models_survex.SurvexPersonRole.objects.all().delete()
models_survex.SurvexStation.objects.all().delete()
SurvexBlock.objects.all().delete()
SurvexFile.objects.all().delete()
SurvexDirectory.objects.all().delete()
SurvexPersonRole.objects.all().delete()
SurvexStation.objects.all().delete()
print(" - survex Data Issues flushed")
models.DataIssue.objects.filter(parser='survex').delete()
models.DataIssue.objects.filter(parser='survexleg').delete()
models.DataIssue.objects.filter(parser='survexunits').delete()
DataIssue.objects.filter(parser='survex').delete()
DataIssue.objects.filter(parser='survexleg').delete()
DataIssue.objects.filter(parser='survexunits').delete()
survexfileroot = MakeSurvexFileRoot()
# this next makes a block_object assciated with a file_object.path = SURVEX_TOPNAME
survexblockroot = models_survex.SurvexBlock(name=ROOTBLOCK, survexpath="", cave=None, survexfile=survexfileroot,
survexblockroot = SurvexBlock(name=ROOTBLOCK, survexpath="", cave=None, survexfile=survexfileroot,
legsall=0, legslength=0.0)
survexblockroot.save()
@ -1300,14 +1300,14 @@ def LoadPositions():
posfile = open("%s.pos" % (topdata))
posfile.readline() #Drop header
try:
survexblockroot = models_survex.SurvexBlock.objects.get(name=ROOTBLOCK)
survexblockroot = SurvexBlock.objects.get(name=ROOTBLOCK)
except:
try:
survexblockroot = models_survex.SurvexBlock.objects.get(id=1)
survexblockroot = SurvexBlock.objects.get(id=1)
except:
message = ' ! FAILED to find root SurvexBlock'
print(message)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
raise
for line in posfile.readlines():
r = poslineregex.match(line)
@ -1321,25 +1321,25 @@ def LoadPositions():
# via the cave data, not by this half-arsed syntactic match which almost never works. PMS.
if False:
try:
sbqs = models_survex.SurvexBlock.objects.filter(survexpath=blockpath)
sbqs = SurvexBlock.objects.filter(survexpath=blockpath)
if len(sbqs)==1:
sb = sbqs[0]
if len(sbqs)>1:
message = " ! MULTIPLE SurvexBlocks {:3} matching Entrance point {} {} '{}'".format(len(sbqs), blockpath, sid, id)
print(message)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
sb = sbqs[0]
elif len(sbqs)<=0:
message = " ! ZERO SurvexBlocks matching Entrance point {} {} '{}'".format(blockpath, sid, id)
print(message)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
sb = survexblockroot
except:
message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid)
print(message)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
try:
ss = models_survex.SurvexStation(name=id, block=survexblockroot)
ss = SurvexStation(name=id, block=survexblockroot)
ss.x = float(x)
ss.y = float(y)
ss.z = float(z)
@ -1348,7 +1348,7 @@ def LoadPositions():
except:
message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid)
print(message)
models.DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='survex', message=message)
raise
print(" - {} SurvexStation entrances found.".format(found))