From 267741fa8b304112d96842118bc6c3ae335500f7 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Tue, 13 Apr 2021 01:13:08 +0100 Subject: [PATCH] fixing typos and changes in importing --- core/unused.py | 1 - core/utils.py | 1 + databaseReset.py | 2 +- parsers/people.py | 12 ++--- parsers/survex.py | 124 +++++++++++++++++++++++----------------------- 5 files changed, 70 insertions(+), 70 deletions(-) diff --git a/core/unused.py b/core/unused.py index e9943e7..c4e9ba3 100644 --- a/core/unused.py +++ b/core/unused.py @@ -1,5 +1,4 @@ import sys -import random import re import logging diff --git a/core/utils.py b/core/utils.py index b2a9249..4309fee 100644 --- a/core/utils.py +++ b/core/utils.py @@ -4,6 +4,7 @@ import datetime import logging import re import resource +import random from subprocess import call from urllib.parse import urljoin diff --git a/databaseReset.py b/databaseReset.py index af65e15..5165440 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -33,7 +33,7 @@ except: raise print(" - Memory footprint after loading Django: {:.3f} MB".format(resource.getrusage(resource.RUSAGE_SELF)[2]/1024.0)) -from troggle.core.models import DataIssue +from troggle.core.models.troggle import DataIssue import troggle.core.models.survex from django.core import management diff --git a/parsers/people.py b/parsers/people.py index ecc9272..345210c 100644 --- a/parsers/people.py +++ b/parsers/people.py @@ -4,7 +4,7 @@ from unidecode import unidecode from django.conf import settings -import troggle.core.models as models +from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.utils import save_carefully '''These functions do not match how the stand-alone folk script works. So the script produces an HTML file which has @@ -50,7 +50,7 @@ def LoadPersonsExpos(): lookupAttribs = {'year':year} nonLookupAttribs = {'name':"CUCC expo %s" % year} - save_carefully(models.Expedition, lookupAttribs, nonLookupAttribs) + save_carefully(Expedition, lookupAttribs, nonLookupAttribs) # make persons print(" - Loading personexpeditions") @@ -84,17 +84,17 @@ def LoadPersonsExpos(): lookupAttribs={'first_name':firstname, 'last_name':(lastname or "")} nonLookupAttribs={'is_vfho':vfho, 'fullname':fullname} - person, created = save_carefully(models.Person, lookupAttribs, nonLookupAttribs) + person, created = save_carefully(Person, lookupAttribs, nonLookupAttribs) parseMugShotAndBlurb(personline=personline, header=header, person=person) # make person expedition from table for year, attended in list(zip(headers, personline))[5:]: - expedition = models.Expedition.objects.get(year=year) + expedition = Expedition.objects.get(year=year) if attended == "1" or attended == "-1": lookupAttribs = {'person':person, 'expedition':expedition} nonLookupAttribs = {'nickname':nickname, 'is_guest':(personline[header["Guest"]] == "1")} - save_carefully(models.PersonExpedition, lookupAttribs, nonLookupAttribs) + save_carefully(PersonExpedition, lookupAttribs, nonLookupAttribs) # used in other referencing parser functions @@ -110,7 +110,7 @@ def GetPersonExpeditionNameLookup(expedition): duplicates = set() #print("Calculating GetPersonExpeditionNameLookup for " + expedition.year) - personexpeditions = models.PersonExpedition.objects.filter(expedition=expedition) + personexpeditions = PersonExpedition.objects.filter(expedition=expedition) htmlparser = HTMLParser() for personexpedition in personexpeditions: possnames = [ ] diff --git a/parsers/survex.py b/parsers/survex.py index 95953b6..9d23cc8 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -12,13 +12,13 @@ from django.utils.timezone import get_current_timezone from django.utils.timezone import make_aware import troggle.settings as settings -import troggle.core.models as models import troggle.core.models.caves as models_caves -import troggle.core.models.survex as models_survex from troggle.core.utils import get_process_memory, chaosmonkey from troggle.parsers.people import GetPersonExpeditionNameLookup from troggle.parsers.logbooks import GetCaveLookup from troggle.core.views.caves import MapLocations +from troggle.core.models.troggle import DataIssue, Expedition +from troggle.core.models.survex import SurvexPersonRole, ScansFolder, SurvexDirectory, SurvexFile, SurvexBlock, SurvexStation survexblockroot = None ROOTBLOCK = "rootblock" @@ -129,11 +129,11 @@ class LoadingSurvex(): if cmd in ["include", "data", "flags", "title", "entrance","set", "units", "alias", "ref"]: message = "! Unparsed [*{}]: '{}' {}".format(cmd, line, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) else: message = "! Bad svx command: [*{}] {} ({}) {}".format(cmd, line, survexblock, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) def LoadSurvexTeam(self, survexblock, line): """Interpeting the *team fields means interpreting older style survex as well as current survex standard, @@ -154,7 +154,7 @@ class LoadingSurvex(): personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower()) if (personexpedition, tm) not in teammembers: teammembers.append((personexpedition, tm)) - personrole = models_survex.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm) + personrole = SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm) personrole.save() personrole.expeditionday = survexblock.expeditionday if personexpedition: @@ -174,7 +174,7 @@ class LoadingSurvex(): else: message = "! Bad *ALIAS: '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) def LoadSurvexUnits(self, survexblock, line): # all for 4 survex files with measurements in feet. bugger. @@ -188,7 +188,7 @@ class LoadingSurvex(): if debugprint: message = "! *UNITS NUMERICAL conversion [{}x] '{}' ({}) {}".format(factor, line, survexblock, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survexunits', message=message) + DataIssue.objects.create(parser='survexunits', message=message) feet = re.match("(?i).*feet$",line) metres = re.match("(?i).*(METRIC|METRES|METERS)$",line) @@ -199,7 +199,7 @@ class LoadingSurvex(): else: message = "! *UNITS in YARDS!? - not converted '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survexunits', message=message) + DataIssue.objects.create(parser='survexunits', message=message) def LoadSurvexDate(self, survexblock, line): # we should make this a date RANGE for everything @@ -208,7 +208,7 @@ class LoadingSurvex(): if year in self.expos: expo = self.expos[year] else: - expeditions = models.Expedition.objects.filter(year=year) + expeditions = Expedition.objects.filter(year=year) assert len(expeditions) == 1 expo= expeditions[0] self.expos[year]= expo @@ -237,7 +237,7 @@ class LoadingSurvex(): else: message = "! DATE unrecognised '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) def LoadSurvexLeg(self, survexblock, sline, comment): """This reads compass, clino and tape data but only keeps the tape lengths, @@ -300,7 +300,7 @@ class LoadingSurvex(): except: message = ' ! datastar parsing incorrect in line %s in %s' % (ls, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survexleg', message=message) + DataIssue.objects.create(parser='survexleg', message=message) survexleg.tape = invalid_tape return # e.g. '29/09' or '(06.05)' in the tape measurement @@ -313,19 +313,19 @@ class LoadingSurvex(): if debugprint: message = " ! Units: Length scaled {}m '{}' in ({}) units:{} factor:{}x".format(tape, ls, survexblock.survexfile.path, self.units, self.unitsfactor) print((self.insp+message)) - models.DataIssue.objects.create(parser='survexleg', message=message) + DataIssue.objects.create(parser='survexleg', message=message) if self.units =="feet": tape = float(tape) / METRESINFEET if debugprint: message = " ! Units: converted to {:.3f}m from {} '{}' in ({})".format(tape, self.units, ls, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survexleg', message=message) + DataIssue.objects.create(parser='survexleg', message=message) survexleg.tape = float(tape) self.legsnumber += 1 except ValueError: message = " ! Value Error: Tape misread in line'{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units) print((self.insp+message)) - models.DataIssue.objects.create(parser='survexleg', message=message) + DataIssue.objects.create(parser='survexleg', message=message) survexleg.tape = invalid_tape try: survexblock.legslength += survexleg.tape @@ -333,14 +333,14 @@ class LoadingSurvex(): except ValueError: message = " ! Value Error: Tape length not added '{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units) print((self.insp+message)) - models.DataIssue.objects.create(parser='survexleg', message=message) + DataIssue.objects.create(parser='survexleg', message=message) try: lcompass = ls[datastar["compass"]] except: message = ' ! Value Error: Compass not found in line %s in %s' % (ls, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survexleg', message=message) + DataIssue.objects.create(parser='survexleg', message=message) lcompass = invalid_compass try: @@ -350,7 +350,7 @@ class LoadingSurvex(): print((" datastar:", datastar)) print((" Line:", ls)) message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path) - models.DataIssue.objects.create(parser='survexleg', message=message) + DataIssue.objects.create(parser='survexleg', message=message) lclino = invalid_clino if lclino == "up": @@ -370,7 +370,7 @@ class LoadingSurvex(): print((" Line:", ls)) message = " ! Value Error: lcompass:'{}' line {} in '{}'".format(lcompass, ls, survexblock.survexfile.path) - models.DataIssue.objects.create(parser='survexleg', message=message) + DataIssue.objects.create(parser='survexleg', message=message) survexleg.compass = invalid_compass # delete the object to save memory @@ -389,7 +389,7 @@ class LoadingSurvex(): if len(args)< 4: message = " ! Empty or BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) return argsgps = self.rx_argsref.match(args) @@ -398,7 +398,7 @@ class LoadingSurvex(): else: message = " ! BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path) print(self.insp+message) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) return if not letterx: @@ -413,12 +413,12 @@ class LoadingSurvex(): if int(wallet)>100: message = " ! Wallet *REF {} - too big in '{}'".format(refscan, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) except: message = " ! Wallet *REF {} - not numeric in '{}'".format(refscan, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) - manyscansfolders = models_survex.ScansFolder.objects.filter(walletname=refscan) + DataIssue.objects.create(parser='survex', message=message) + manyscansfolders = ScansFolder.objects.filter(walletname=refscan) if manyscansfolders: survexblock.scansfolder = manyscansfolders[0] # this is a ForeignKey field print(manyscansfolders[0]) @@ -426,11 +426,11 @@ class LoadingSurvex(): if len(manyscansfolders) > 1: message = " ! Wallet *REF {} - {} scan folders from DB search in {}".format(refscan, len(manyscansfolders), survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) else: message = " ! Wallet *REF '{}' - NOT found in DB search '{}'".format(refscan, survexblock.survexfile.path) print((self.insp+message)) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) def LoadSurvexQM(self, survexblock, qmline): insp = self.insp @@ -465,11 +465,11 @@ class LoadingSurvex(): qm.save # message = " ! QM{} '{}' CREATED in DB in '{}'".format(qm_no, qm_nearest,survexblock.survexfile.path) # print(insp+message) - # models.DataIssue.objects.create(parser='survex', message=message) + # DataIssue.objects.create(parser='survex', message=message) except: message = " ! QM{} FAIL to create {} in'{}'".format(qm_no, qm_nearest,survexblock.survexfile.path) print(insp+message) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) def LoadSurvexDataNormal(self,survexblock,args): """Sets the order for data elements in this and following blocks, e.g. @@ -498,7 +498,7 @@ class LoadingSurvex(): message = " ! - Unrecognised *data normal statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath) print(message) print(message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) return else: datastar = self.datastardefault @@ -516,13 +516,13 @@ class LoadingSurvex(): # message = " ! - *data {} blocks ignored. {}|{}" '{}' .format(ls[0].upper(), survexblock.name, survexblock.survexpath, args) # print(message) # print(message,file=sys.stderr) - # models.DataIssue.objects.create(parser='survex', message=message) + # DataIssue.objects.create(parser='survex', message=message) self.datastar["type"] = ls[0] else: message = " ! - Unrecognised *data statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath) print(message) print(message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) def LoadSurvexFlags(self, args): # Valid flags are DUPLICATE, SPLAY, and SURFACE, and a flag may be preceded with NOT to turn it off. @@ -587,7 +587,7 @@ class LoadingSurvex(): if not headpath: return self.svxdirs[""] if headpath.lower() not in self.svxdirs: - self.svxdirs[headpath.lower()] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile) + self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile) self.svxdirs[headpath.lower()].save() self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory return self.svxdirs[headpath.lower()] @@ -603,7 +603,7 @@ class LoadingSurvex(): message = " ! {} is not a cave. (while creating '{}' sfile & sdirectory)".format(headpath, includelabel) print("\n"+message) print("\n"+message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) def LoadSurvexFile(self, svxid): """Creates SurvexFile in the database, and SurvexDirectory if needed @@ -622,7 +622,7 @@ class LoadingSurvex(): # print("{:2}{} - NEW survexfile:'{}'".format(self.depthbegin, depth, svxid)) headpath = os.path.dirname(svxid) - newfile = models_survex.SurvexFile(path=svxid) + newfile = SurvexFile(path=svxid) newfile.save() # until we do this there is no internal id so no foreign key works self.currentsurvexfile = newfile newdirectory = self.GetSurvexDirectory(headpath) @@ -635,7 +635,7 @@ class LoadingSurvex(): message = " ! 'None' SurvexDirectory returned from GetSurvexDirectory({})".format(headpath) print(message) print(message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) if cave: newdirectory.cave = cave @@ -648,7 +648,7 @@ class LoadingSurvex(): message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(svxid) print(message) print(message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) self.currentsurvexfile.save() # django insists on this although it is already saved !? try: newdirectory.save() @@ -864,7 +864,7 @@ class LoadingSurvex(): self.units = "metres" self.currentpersonexped = [] printbegin() - newsurvexblock = models_survex.SurvexBlock(name=blkid, parent=survexblock, + newsurvexblock = SurvexBlock(name=blkid, parent=survexblock, survexpath=pathlist, cave=self.currentcave, survexfile=self.currentsurvexfile, legsall=0, legslength=0.0) @@ -939,7 +939,7 @@ class LoadingSurvex(): message = " ! -ERROR *include command not expected here {}. Re-run a full Survex import.".format(path) print(message) print(message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) else: self.LoadSurvexFallThrough(survexblock, args, cmd) @@ -983,13 +983,13 @@ class LoadingSurvex(): print(message) print(message,file=flinear) print("\n"+message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) if self.svxfileslist.count(path) > 20: message = " ! ERROR. Survex file already seen 20x. Probably an infinite loop so fix your *include statements that include this. Aborting. {}".format(path) print(message) print(message,file=flinear) print(message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) return self.svxfileslist.append(path) @@ -1027,7 +1027,7 @@ class LoadingSurvex(): print(message) print(message,file=flinear) print(message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop)) fcollate.write(";*edulcni {}\n".format(pop)) fininclude.close() @@ -1037,7 +1037,7 @@ class LoadingSurvex(): message = " ! ERROR *include file not found for:'{}'".format(includepath) print(message) print(message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) elif re.match("(?i)begin$", cmd): self.depthbegin += 1 depth = " " * self.depthbegin @@ -1059,7 +1059,7 @@ class LoadingSurvex(): print(message) print(message,file=flinear) print(message,file=sys.stderr) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) self.depthbegin -= 1 pass @@ -1199,9 +1199,9 @@ def FindAndLoadSurvex(survexblockroot): def MakeSurvexFileRoot(): """Returns a file_object.path = SURVEX_TOPNAME associated with directory_object.path = SURVEX_DATA """ - fileroot = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None) + fileroot = SurvexFile(path=settings.SURVEX_TOPNAME, cave=None) fileroot.save() - directoryroot = models_survex.SurvexDirectory(path=settings.SURVEX_DATA, cave=None, primarysurvexfile=fileroot) + directoryroot = SurvexDirectory(path=settings.SURVEX_DATA, cave=None, primarysurvexfile=fileroot) directoryroot.save() fileroot.survexdirectory = directoryroot # i.e. SURVEX_DATA/SURVEX_TOPNAME fileroot.save() # mutually dependent objects need a double-save like this @@ -1210,19 +1210,19 @@ def MakeSurvexFileRoot(): def LoadSurvexBlocks(): print(' - Flushing All Survex Blocks...') - models_survex.SurvexBlock.objects.all().delete() - models_survex.SurvexFile.objects.all().delete() - models_survex.SurvexDirectory.objects.all().delete() - models_survex.SurvexPersonRole.objects.all().delete() - models_survex.SurvexStation.objects.all().delete() + SurvexBlock.objects.all().delete() + SurvexFile.objects.all().delete() + SurvexDirectory.objects.all().delete() + SurvexPersonRole.objects.all().delete() + SurvexStation.objects.all().delete() print(" - survex Data Issues flushed") - models.DataIssue.objects.filter(parser='survex').delete() - models.DataIssue.objects.filter(parser='survexleg').delete() - models.DataIssue.objects.filter(parser='survexunits').delete() + DataIssue.objects.filter(parser='survex').delete() + DataIssue.objects.filter(parser='survexleg').delete() + DataIssue.objects.filter(parser='survexunits').delete() survexfileroot = MakeSurvexFileRoot() # this next makes a block_object assciated with a file_object.path = SURVEX_TOPNAME - survexblockroot = models_survex.SurvexBlock(name=ROOTBLOCK, survexpath="", cave=None, survexfile=survexfileroot, + survexblockroot = SurvexBlock(name=ROOTBLOCK, survexpath="", cave=None, survexfile=survexfileroot, legsall=0, legslength=0.0) survexblockroot.save() @@ -1300,14 +1300,14 @@ def LoadPositions(): posfile = open("%s.pos" % (topdata)) posfile.readline() #Drop header try: - survexblockroot = models_survex.SurvexBlock.objects.get(name=ROOTBLOCK) + survexblockroot = SurvexBlock.objects.get(name=ROOTBLOCK) except: try: - survexblockroot = models_survex.SurvexBlock.objects.get(id=1) + survexblockroot = SurvexBlock.objects.get(id=1) except: message = ' ! FAILED to find root SurvexBlock' print(message) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) raise for line in posfile.readlines(): r = poslineregex.match(line) @@ -1321,25 +1321,25 @@ def LoadPositions(): # via the cave data, not by this half-arsed syntactic match which almost never works. PMS. if False: try: - sbqs = models_survex.SurvexBlock.objects.filter(survexpath=blockpath) + sbqs = SurvexBlock.objects.filter(survexpath=blockpath) if len(sbqs)==1: sb = sbqs[0] if len(sbqs)>1: message = " ! MULTIPLE SurvexBlocks {:3} matching Entrance point {} {} '{}'".format(len(sbqs), blockpath, sid, id) print(message) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) sb = sbqs[0] elif len(sbqs)<=0: message = " ! ZERO SurvexBlocks matching Entrance point {} {} '{}'".format(blockpath, sid, id) print(message) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) sb = survexblockroot except: message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid) print(message) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) try: - ss = models_survex.SurvexStation(name=id, block=survexblockroot) + ss = SurvexStation(name=id, block=survexblockroot) ss.x = float(x) ss.y = float(y) ss.z = float(z) @@ -1348,7 +1348,7 @@ def LoadPositions(): except: message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid) print(message) - models.DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message) raise print(" - {} SurvexStation entrances found.".format(found))