expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-22 15:21:52 +00:00
Code Issues 19 Projects 3 Activity

refactor to reduce db hits and fix *team

Browse Source
This commit is contained in:
Philip Sargent 2023-01-28 21:00:38 +00:00
parent e6fd1f0ec5
commit 0d9d307490
3 changed files with 206 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/admin.py
View File

@ -19,6 +19,8 @@ In particular, it enables JSON export of any data with 'export_as_json'
and configures the search fields to be used within the control panel. and configures the search fields to be used within the control panel.
What is the search path for the css and js inclusions in the Media subclasses though ?! What is the search path for the css and js inclusions in the Media subclasses though ?!
The page looks for /static/jquery/jquery.min.js
''' '''
class TroggleModelAdmin(admin.ModelAdmin): class TroggleModelAdmin(admin.ModelAdmin):

2
core/models/survex.py
View File

@ -152,7 +152,7 @@ class SurvexPersonRole(models.Model):
personname = models.CharField(max_length=100) personname = models.CharField(max_length=100)
person = models.ForeignKey('Person', blank=True, null=True,on_delete=models.SET_NULL) person = models.ForeignKey('Person', blank=True, null=True,on_delete=models.SET_NULL)
personexpedition = models.ForeignKey('PersonExpedition', blank=True, null=True,on_delete=models.SET_NULL) personexpedition = models.ForeignKey('PersonExpedition', blank=True, null=True,on_delete=models.SET_NULL)
persontrip = models.ForeignKey('PersonTrip', blank=True, null=True,on_delete=models.SET_NULL) # logbook # persontrip = models.ForeignKey('PersonTrip', blank=True, null=True,on_delete=models.SET_NULL) # logbook thing not a survex thing
expeditionday = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.SET_NULL) expeditionday = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.SET_NULL)
def __str__(self): def __str__(self):

289
parsers/survex.py
View File

@ -22,9 +22,6 @@ It also scans the Loser repo for all the svx files, which it loads individually
todo = """ todo = """
-#BUG, if *date comes after *team, the person's date is not set at all.
It needs re-setting at the end of the block. 'Fixed', but fix not working.
- LoadSurvexFile() Creates a new current survexfile and valid .survexdirectory - LoadSurvexFile() Creates a new current survexfile and valid .survexdirectory
The survexblock passed-in is not necessarily the parent. FIX THIS. The survexblock passed-in is not necessarily the parent. FIX THIS.
@ -47,18 +44,23 @@ debugprinttrigger = "!"
dataissues = [] dataissues = []
def stash_data_issue(parser=None, message=None, url=None): def stash_data_issue(parser=None, message=None, url=None, sb=None):
"""Avoid hitting the database for error messages until the end of the import"""
global dataissues global dataissues
dataissues.append((parser, message, url)) dataissues.append((parser, message, url, sb))
def store_data_issues(): def store_data_issues():
"""Take the stash and store it permanently in the database instead"""
global dataissues global dataissues
print(f" - Storing {len(dataissues)} Data Issues into database") print(f" - Storing {len(dataissues)} Data Issues into database")
for i in dataissues: for issue in dataissues:
parser, message, url = i parser, message, url, sb = issue
if url is None:
if sb is not None:
url = get_offending_filename(sb)
DataIssue.objects.create(parser=parser, message=message, url=url) DataIssue.objects.create(parser=parser, message=message, url=url)
dataissues = [] dataissues = [] # in database now, so empty cache
class MapLocations(object): class MapLocations(object):
"""Class used only for identifying the entrance locations""" """Class used only for identifying the entrance locations"""
@ -104,14 +106,6 @@ class MapLocations(object):
def __str__(self): def __str__(self):
return f"{len(self.p)} map locations" return f"{len(self.p)} map locations"
def get_offending_filename(path):
"""Used to provide the URL for a line in the DataErrors page
whcih reports problems on importing data into troggle
"""
return "/survexfile/" + path + ".svx"
class SurvexLeg: class SurvexLeg:
"""No longer a models.Model subclass, so no longer a database table""" """No longer a models.Model subclass, so no longer a database table"""
@ -119,14 +113,83 @@ class SurvexLeg:
compass = 0.0 compass = 0.0
clino = 0.0 clino = 0.0
def get_offending_filename(path):
"""Used to provide the URL for a line in the DataErrors page
which reports problems on importing data into troggle
"""
return "/survexfile/" + path + ".svx"
trip_people_cache = {}
def get_team_on_trip(survexblock):
"""Uses a cache to avoid a database query if it doesn't need to.
Only used for complete team."""
global trip_people_cache
if survexblock in trip_people_cache:
return trip_people_cache[survexblock]
qpeople = SurvexPersonRole.objects.filter(survexblock=survexblock)
trip_people_cache[survexblock] = qpeople # this is a query list
return qpeople
def get_people_on_trip(survexblock): def get_people_on_trip(survexblock):
qpeople = SurvexPersonRole.objects.filter(survexblock=survexblock) """Gets the displayable names of the people on a survexbock trip.
Only used for complete team."""
qpeople = get_team_on_trip(survexblock)
people = [] people = []
for p in qpeople: for p in qpeople:
people.append(f"{p.personname}") people.append(f"{p.personname}")
return list(set(people)) return list(set(people))
trip_person_cache = {}
def put_person_on_trip(survexblock, personexpedition, tm):
"""Uses a cache to avoid a database query if it doesn't need to.
Only used for a single person"""
global trip_person_cache
if (survexblock, personexpedition) in trip_person_cache:
return True
try:
personrole = SurvexPersonRole.objects.create(
survexblock=survexblock, person = personexpedition.person,
expeditionday = survexblock.expeditionday, personexpedition=personexpedition,
personname=tm
)
except:
message = f"! *team '{tm}' FAIL, already created {survexblock.survexfile.path} ({survexblock}) "
print(self.insp + message)
stash_data_issue(
parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
)
trip_person_cache[(survexblock, personexpedition)] = 1
return False
person_pending_cache = {}
def add_to_pending(survexblock, tm):
global person_pending_cache
if survexblock not in person_pending_cache:
person_pending_cache[survexblock] = set()
person_pending_cache[survexblock].add(tm)
# personexpedition = None
# personrole, created = SurvexPersonRole.objects.update_or_create(
# survexblock=survexblock, personexpedition=personexpedition, personname=tm)
# personrole.save()
def get_team_pending(survexblock):
"""A set of *team names before we get to the *date line in a survexblock"""
global person_pending_cache
if survexblock in person_pending_cache:
teamnames = person_pending_cache[survexblock] # a set of names
person_pending_cache[survexblock] = ()
return teamnames
return
class LoadingSurvex: class LoadingSurvex:
"""A 'survex block' is a *begin...*end set of cave data. """A 'survex block' is a *begin...*end set of cave data.
@ -172,6 +235,7 @@ class LoadingSurvex:
rx_comminc = re.compile(r"(?i)^\|\*include[\s]*([-\w/]*).*$") # inserted by linear collate ;*include rx_comminc = re.compile(r"(?i)^\|\*include[\s]*([-\w/]*).*$") # inserted by linear collate ;*include
rx_commcni = re.compile(r"(?i)^\|\*edulcni[\s]*([-\w/]*).*$") # inserted by linear collate ;*edulcni rx_commcni = re.compile(r"(?i)^\|\*edulcni[\s]*([-\w/]*).*$") # inserted by linear collate ;*edulcni
rx_include = re.compile(r"(?i)^\s*(\*include[\s].*)$") rx_include = re.compile(r"(?i)^\s*(\*include[\s].*)$")
rx_include2 = re.compile("(?i)include$")
rx_commref = re.compile(r"(?i)^\s*ref(?:erence)?[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)") rx_commref = re.compile(r"(?i)^\s*ref(?:erence)?[\s.:]*(\d+)\s*#\s*(X)?\s*(\d+)")
rx_ref_text = re.compile(r'(?i)^\s*\"[^"]*\"\s*$') rx_ref_text = re.compile(r'(?i)^\s*\"[^"]*\"\s*$')
rx_star = re.compile(r"(?i)\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$") rx_star = re.compile(r"(?i)\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$")
@ -180,6 +244,8 @@ class LoadingSurvex:
rx_badmerge = re.compile(r"(?i).*(\>\>\>\>\>)|(\=\=\=\=\=)|(\<\<\<\<\<).*$") rx_badmerge = re.compile(r"(?i).*(\>\>\>\>\>)|(\=\=\=\=\=)|(\<\<\<\<\<).*$")
rx_ref2 = re.compile(r"(?i)\s*ref[.;]?") rx_ref2 = re.compile(r"(?i)\s*ref[.;]?")
rx_commteam = re.compile(r"(?i)\s*(Messteam|Zeichner)\s*[:]?(.*)") rx_commteam = re.compile(r"(?i)\s*(Messteam|Zeichner)\s*[:]?(.*)")
rx_quotedtitle = re.compile(r'(?i)^"(.*)"$')
# This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g. # This interprets the survex "*data normal" command which sets out the order of the fields in the data, e.g.
# *DATA normal from to length gradient bearing ignore ignore ignore ignore # *DATA normal from to length gradient bearing ignore ignore ignore ignore
@ -246,7 +312,7 @@ class LoadingSurvex:
) )
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
else: else:
message = ( message = (
@ -254,7 +320,7 @@ class LoadingSurvex:
) )
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
def LoadSurvexTeam(self, survexblock, line): def LoadSurvexTeam(self, survexblock, line):
@ -265,7 +331,7 @@ class LoadingSurvex:
*team gb, bl *team gb, bl
personrole is used to record that a person was on a survex trip, NOT the role they played. personrole is used to record that a person was on a survex trip, NOT the role they played.
(NB PersonTrip is a logbook thing, not a survex thing. Yes they could be merged, maybe.) (NB PersonTrip is a logbook thing, not a survex thing. )
""" """
def record_team_member(tm, survexblock): def record_team_member(tm, survexblock):
@ -278,45 +344,48 @@ class LoadingSurvex:
# we will have to attach them to the survexblock anyway, and then do a # we will have to attach them to the survexblock anyway, and then do a
# later check on whether they are valid when we get the date. # later check on whether they are valid when we get the date.
# We have hundreds of updated Django database updates when the same person is
# on the same trip in multiple roles. We should de-duplicate these ourselves in Python
# instead of using SurvexPersonRole.objects.update_or_create() which is expensive.
expo = survexblock.expedition # may be None if no *date yet expo = survexblock.expedition # may be None if no *date yet
# this syntax was bizarre.. made more obvious
if expo: if expo:
if not survexblock.expeditionday: # *date has been set if not survexblock.expeditionday: # *date has been set
# should not happen # should not happen
message = f"! *team {expo.year} expo ok, expedition day not in *team {survexblock.survexfile.path} ({survexblock}) " message = f"! *team {expo.year} expo ok, expedition day NOT in *team {survexblock.survexfile.path} ({survexblock}) "
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
personexpedition = GetPersonExpeditionNameLookup(expo).get(tm.lower()) personexpedition = GetPersonExpeditionNameLookup(expo).get(tm.lower())
if personexpedition: if personexpedition:
personrole, created = SurvexPersonRole.objects.update_or_create( put_person_on_trip(survexblock, personexpedition, tm)
survexblock=survexblock, personexpedition=personexpedition, personname=tm
)
personrole.person = personexpedition.person
personrole.expeditionday = survexblock.expeditionday
self.currentpersonexped.append(personexpedition) # used in push/pop block code self.currentpersonexped.append(personexpedition) # used in push/pop block code
personrole.save()
elif known_foreigner(tm): # note, not using .lower() elif known_foreigner(tm): # note, not using .lower()
message = f"- *team {expo.year} '{tm}' known foreigner on *team {survexblock.survexfile.path} ({survexblock}) in '{line}'" message = f"- *team {expo.year} '{tm}' known foreigner on *team {survexblock.survexfile.path} ({survexblock}) in '{line}'"
print(self.insp + message) print(self.insp + message)
# stash_data_issue(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path)) # stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path))
else: else:
# we know the date and expo, but can't find the person # we know the date and expo, but can't find the person
message = f"! *team {expo.year} '{tm}' FAIL personexpedition lookup on *team {survexblock.survexfile.path} ({survexblock}) in '{line}'" message = f"! *team {expo.year} '{tm}' FAIL personexpedition lookup on *team {survexblock.survexfile.path} ({survexblock}) in '{line}'"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
else: else:
personexpedition = None add_to_pending(survexblock, tm)
personrole, created = SurvexPersonRole.objects.update_or_create(
survexblock=survexblock, personexpedition=personexpedition, personname=tm
)
# don't know the date yet, so cannot query the table about validity. # don't know the date yet, so cannot query the table about validity.
# assume the person is valid. It will get picked up with the *date appears # assume the person is valid. It will get picked up with the *date appears
personrole.save() # There are hundreds of these..
message = (
f"- Team/Date mis-ordered: {line} ({survexblock}) {survexblock.survexfile.path}"
)
print(self.insp + message)
# stash_data_issue(
# parser="survex team", message=message, url=None, sb=(survexblock.survexfile.path)
# )
mteammember = self.rx_teammem.match(line) # matches the role at the beginning mteammember = self.rx_teammem.match(line) # matches the role at the beginning
if not mteammember: if not mteammember:
@ -332,7 +401,7 @@ class LoadingSurvex:
message = f"! *team {survexblock.survexfile.path} ({survexblock}) Weird '{mteammember.group(1)}' oldstyle line: '{line}'" message = f"! *team {survexblock.survexfile.path} ({survexblock}) Weird '{mteammember.group(1)}' oldstyle line: '{line}'"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
else: else:
nullmember = self.rx_teamabs.match(line) # matches empty role line. Ignore these. nullmember = self.rx_teamabs.match(line) # matches empty role line. Ignore these.
@ -340,7 +409,7 @@ class LoadingSurvex:
message = f"! *team {survexblock.survexfile.path} ({survexblock}) Bad line: '{line}'" message = f"! *team {survexblock.survexfile.path} ({survexblock}) Bad line: '{line}'"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
else: else:
for tm in self.rx_person.split(mteammember.group(2)): for tm in self.rx_person.split(mteammember.group(2)):
@ -351,7 +420,7 @@ class LoadingSurvex:
message = f"! Weird *team '{mteammember.group(2)}' newstyle line: '{line}' ({survexblock}) {survexblock.survexfile.path}" message = f"! Weird *team '{mteammember.group(2)}' newstyle line: '{line}' ({survexblock}) {survexblock.survexfile.path}"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
def LoadSurvexEntrance(self, survexblock, line): def LoadSurvexEntrance(self, survexblock, line):
@ -408,7 +477,7 @@ class LoadingSurvex:
) )
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
expo = expeditions[0] expo = expeditions[0]
@ -416,25 +485,61 @@ class LoadingSurvex:
return expo return expo
def LoadSurvexDate(self, survexblock, line): def LoadSurvexDate(self, survexblock, line):
# we should make this a date RANGE for everything? """We now have a valid date for this survexblock, so we now know the expo
it relates to and can use GetPersonExpeditionNameLookup(expo) to check whether
the people are correct.
Note that a *team line can come before AND after a *date line"""
def setdate_on_survexblock(year): def setdate_on_survexblock(year):
# We are assuming that deferred *team people are in the same block. Otherwise, ouch. """Either *date comes before any *team, in which case there are no prior
PersonRoles attached, or
*team came before this *date, in which case the names are only in 'pending'"""
global trip_person_cache
expo = self.get_expo_from_year(year) expo = self.get_expo_from_year(year)
survexblock.expedition = expo survexblock.expedition = expo
survexblock.expeditionday = expo.get_expedition_day(survexblock.date) survexblock.expeditionday = expo.get_expedition_day(survexblock.date)
survexblock.save() survexblock.save()
team = SurvexPersonRole.objects.filter(survexblock=survexblock) team = get_team_on_trip(survexblock) # should be empty, should only be in 'pending'
for pr in team: # team = SurvexPersonRole.objects.filter(survexblock=survexblock)
if not pr.expeditionday: # *date and *team in 'wrong' order. All working now. if len(team) > 0:
message = f"! *team {expo.year} Multiple *date in one block? Already someone on team when *date seen. {survexblock.survexfile.path} ({survexblock}) in '{line}'"
print(self.insp + message)
stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path))
if teamnames := get_team_pending(survexblock): # WALRUS https://docs.python.org/3/whatsnew/3.8.html#assignment-expressions
for tm in teamnames:
if known_foreigner(tm):
message = f"- *team {expo.year} '{tm}' known foreigner *date (misordered) {survexblock.survexfile.path} ({survexblock}) in '{line}'"
print(self.insp + message)
# stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path))
else:
pe = GetPersonExpeditionNameLookup(expo).get(tm.lower())
if pe:
put_person_on_trip(survexblock, pe, tm)
self.currentpersonexped.append(pe)
else:
message = f"! *team {year} '{tm}' FAIL personexpedition lookup on *date {survexblock.survexfile.path} ({survexblock}) "
print(self.insp + message)
stash_data_issue(
parser="survex",
message=message,
url=None, sb=(survexblock.survexfile.path),
)
# All this next section should not happen unless there are >1 *date lines in a block
for pr in team: # pr is a PersonRole object
if not pr.expeditionday: # *date and *team in 'wrong' order.
pr.expeditionday = survexblock.expeditionday pr.expeditionday = survexblock.expeditionday
pr.save() pr.save()
if not pr.personexpedition: # again, we didn't know the date until now if not pr.personexpedition:
pe = GetPersonExpeditionNameLookup(expo).get(pr.personname.lower()) pe = GetPersonExpeditionNameLookup(expo).get(pr.personname.lower())
if pe: if pe: # pe is a PersonExpedition
# message = "! {} ({}) Fixing undated personexpedition '{}'".format(survexblock.survexfile.path, survexblock, p.personname) # message = "! {} ({}) Fixing undated personexpedition '{}'".format(survexblock.survexfile.path, survexblock, p.personname)
# print(self.insp+message) # print(self.insp+message)
# stash_data_issue(parser='survex', message=message) # stash_data_issue(parser='survex', message=message)
@ -445,21 +550,21 @@ class LoadingSurvex:
elif known_foreigner(pr.personname): # note, not using .lower() elif known_foreigner(pr.personname): # note, not using .lower()
message = f"- *team {expo.year} '{pr.personname}' known foreigner on *date {survexblock.survexfile.path} ({survexblock}) in '{line}'" message = f"- *team {expo.year} '{pr.personname}' known foreigner on *date {survexblock.survexfile.path} ({survexblock}) in '{line}'"
print(self.insp + message) print(self.insp + message)
# stash_data_issue(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path)) # stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path))
else: else:
message = f"! *team {year} '{pr.personname}' FAIL personexpedition lookup on *date {survexblock.survexfile.path} ({survexblock}) '{pr.personname}'" message = f"! *team {year} '{pr.personname}' FAIL personexpedition lookup on *date {survexblock.survexfile.path} ({survexblock}) '{pr.personname}'"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survex", parser="survex",
message=message, message=message,
url=get_offending_filename(survexblock.survexfile.path), url=None, sb=(survexblock.survexfile.path),
) )
oline = line oline = line
if len(line) > 10: if len(line) > 10:
# message = "! DATE Warning LONG DATE '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path) # message = "! DATE Warning LONG DATE '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
# print(self.insp+message) # print(self.insp+message)
# stash_data_issue(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path)) # stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path))
if line[10] == "-": # ie a range, just look at first date if line[10] == "-": # ie a range, just look at first date
line = line[0:10] line = line[0:10]
if len(line) == 10: if len(line) == 10:
@ -467,41 +572,41 @@ class LoadingSurvex:
# TO DO set to correct Austrian timezone Europe/Vienna ? # TO DO set to correct Austrian timezone Europe/Vienna ?
# %m and %d need leading zeros. Source svx files require them. # %m and %d need leading zeros. Source svx files require them.
survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d") survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d")
setdate_on_survexblock(year)
elif len(line) == 7: elif len(line) == 7:
year = line[:4] year = line[:4]
perps = get_people_on_trip(survexblock) # What, you don't know Judge Dredd slang ? perps = get_people_on_trip(survexblock) # What, you don't know Judge Dredd slang ?
message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}" message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="svxdate", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
) )
survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m") # sets to first of month survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m") # sets to first of month
setdate_on_survexblock(year)
elif len(line) == 4: elif len(line) == 4:
year = line[:4] year = line[:4]
perps = get_people_on_trip(survexblock) perps = get_people_on_trip(survexblock)
message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}" message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="svxdate", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
) )
survexblock.date = datetime.strptime(line, "%Y") # sets to January 1st survexblock.date = datetime.strptime(line, "%Y") # sets to January 1st
setdate_on_survexblock(year)
else: else:
# these errors are reporting the wrong survexblock, which is actually a SurvexFile (!) # these errors are reporting the wrong survexblock, which is actually a SurvexFile (!)
# see To Do notes on how to trigger this. Still needs investigating..
message = ( message = (
f"! DATE Error unrecognised '{oline}-{survexblock}' ({type(survexblock)}) {survexblock.survexfile.path}" f"! DATE Error unrecognised '{oline}-{survexblock}' ({type(survexblock)}) {survexblock.survexfile.path}"
) )
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
print(f" {type(survexblock)=}") # survexblock.parent fails as a SurvexFile has no .parent ...ugh. print(f" {type(survexblock)=}") # survexblock.parent fails as a SurvexFile has no .parent ...ugh.
print(f" {survexblock.survexpath=}") print(f" {survexblock.survexpath=}")
print(f" {survexblock.survexfile=}") print(f" {survexblock.survexfile=}")
# raise # raise
setdate_on_survexblock(year)
def LoadSurvexLeg(self, survexblock, sline, comment, svxline): def LoadSurvexLeg(self, survexblock, sline, comment, svxline):
"""This reads compass, clino and tape data but only keeps the tape lengths, """This reads compass, clino and tape data but only keeps the tape lengths,
the rest is discarded after error-checking. the rest is discarded after error-checking.
@ -549,7 +654,7 @@ class LoadingSurvex:
print(f" Line: {sline}\nsvxline: {svxline}") print(f" Line: {sline}\nsvxline: {svxline}")
message = f" ! Not 5 fields in line '{sline.lower()}' {self.datastar=} {ls=} in\n{survexblock}\n{survexblock.survexfile}\n{survexblock.survexfile.path}" message = f" ! Not 5 fields in line '{sline.lower()}' {self.datastar=} {ls=} in\n{survexblock}\n{survexblock.survexfile}\n{survexblock.survexfile.path}"
stash_data_issue( stash_data_issue(
parser="survexleg", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path)
) )
datastar = self.datastar # shallow copy: alias but the things inside are the same things datastar = self.datastar # shallow copy: alias but the things inside are the same things
@ -578,7 +683,7 @@ class LoadingSurvex:
message = f" ! datastar parsing from/to incorrect in line {ls} in {survexblock.survexfile.path}" message = f" ! datastar parsing from/to incorrect in line {ls} in {survexblock.survexfile.path}"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survexleg", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path)
) )
return return
@ -588,7 +693,7 @@ class LoadingSurvex:
message = f" ! datastar parsing incorrect in line {ls} in {survexblock.survexfile.path}" message = f" ! datastar parsing incorrect in line {ls} in {survexblock.survexfile.path}"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survexleg", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path)
) )
survexleg.tape = invalid_tape survexleg.tape = invalid_tape
return return
@ -603,7 +708,7 @@ class LoadingSurvex:
message = f" ! Units: Length scaled {tape}m '{ls}' in ({survexblock.survexfile.path}) units:{self.units} factor:{self.unitsfactor}x" message = f" ! Units: Length scaled {tape}m '{ls}' in ({survexblock.survexfile.path}) units:{self.units} factor:{self.unitsfactor}x"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survexleg", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path)
) )
if self.units == "feet": if self.units == "feet":
tape = float(tape) / METRESINFEET tape = float(tape) / METRESINFEET
@ -611,7 +716,7 @@ class LoadingSurvex:
message = f" ! Units: converted to {tape:.3f}m from {self.units} '{ls}' in ({survexblock.survexfile.path})" message = f" ! Units: converted to {tape:.3f}m from {self.units} '{ls}' in ({survexblock.survexfile.path})"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survexleg", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path)
) )
survexleg.tape = float(tape) survexleg.tape = float(tape)
self.legsnumber += 1 self.legsnumber += 1
@ -619,7 +724,7 @@ class LoadingSurvex:
message = f" ! Value Error: Tape misread in line'{ls}' in {survexblock.survexfile.path} units:{self.units}" message = f" ! Value Error: Tape misread in line'{ls}' in {survexblock.survexfile.path} units:{self.units}"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survexleg", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path)
) )
survexleg.tape = invalid_tape survexleg.tape = invalid_tape
try: try:
@ -631,7 +736,7 @@ class LoadingSurvex:
) )
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survexleg", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path)
) )
try: try:
@ -640,7 +745,7 @@ class LoadingSurvex:
message = f" ! Value Error: Compass not found in line {ls} in {survexblock.survexfile.path}" message = f" ! Value Error: Compass not found in line {ls} in {survexblock.survexfile.path}"
print(self.insp + message) print(self.insp + message)
stash_data_issue( stash_data_issue(
parser="survexleg", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path)
) )
lcompass = invalid_compass lcompass = invalid_compass
@ -652,7 +757,7 @@ class LoadingSurvex:
print((" Line:", ls)) print((" Line:", ls))
message = f" ! Value Error: Clino misread in line '{sline.lower()}' {datastar=} {self.datastar=} {ls=} in\n{survexblock}\n{survexblock.survexfile}\n{survexblock.survexfile.path}" message = f" ! Value Error: Clino misread in line '{sline.lower()}' {datastar=} {self.datastar=} {ls=} in\n{survexblock}\n{survexblock.survexfile}\n{survexblock.survexfile.path}"
stash_data_issue( stash_data_issue(
parser="survexleg", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path)
) )
lclino = invalid_clino lclino = invalid_clino
@ -673,7 +778,7 @@ class LoadingSurvex:
print((" Line:", ls)) print((" Line:", ls))
message = " ! Value Error: lcompass:'{}' line {} in '{}'".format(lcompass, ls, survexblock.survexfile.path) message = " ! Value Error: lcompass:'{}' line {} in '{}'".format(lcompass, ls, survexblock.survexfile.path)
stash_data_issue( stash_data_issue(
parser="survexleg", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path)
) )
survexleg.compass = invalid_compass survexleg.compass = invalid_compass
@ -765,13 +870,13 @@ class LoadingSurvex:
message = f' ! QM TICK find FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" qmlist:"{qm}" in "{survexblock.survexfile.path}" + comment:"{qmtick.group(3)}" ' message = f' ! QM TICK find FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" qmlist:"{qm}" in "{survexblock.survexfile.path}" + comment:"{qmtick.group(3)}" '
print(message) print(message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
if len(qm) > 1: if len(qm) > 1:
message = f' ! QM TICK MULTIPLE found FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" in "{survexblock.survexfile.path}" + comment:"{qmtick.group(3)}" ' message = f' ! QM TICK MULTIPLE found FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" in "{survexblock.survexfile.path}" + comment:"{qmtick.group(3)}" '
print(message) print(message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
qm[0].ticked = True qm[0].ticked = True
qm[0].save() qm[0].save()
@ -831,7 +936,7 @@ class LoadingSurvex:
message = f" ! QM{qm_no} FAIL to create {qm_nearest} in'{survexblock.survexfile.path}'" message = f" ! QM{qm_no} FAIL to create {qm_nearest} in'{survexblock.survexfile.path}'"
print(insp + message) print(insp + message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
def LoadSurvexDataNormal(self, survexblock, args): def LoadSurvexDataNormal(self, survexblock, args):
@ -864,7 +969,7 @@ class LoadingSurvex:
print(message) print(message)
print(message, file=sys.stderr) print(message, file=sys.stderr)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
return return
else: else:
@ -876,7 +981,7 @@ class LoadingSurvex:
print(message) print(message)
print(message, file=sys.stderr) print(message, file=sys.stderr)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
return False return False
@ -905,7 +1010,7 @@ class LoadingSurvex:
print(message) print(message)
print(message, file=sys.stderr) print(message, file=sys.stderr)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
self.datastar["type"] = ls[0] self.datastar["type"] = ls[0]
@ -1021,7 +1126,7 @@ class LoadingSurvex:
print("\n" + message) print("\n" + message)
print("\n" + message, file=sys.stderr) print("\n" + message, file=sys.stderr)
print(f"{self.pending}", end="", file=sys.stderr) print(f"{self.pending}", end="", file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=get_offending_filename(includelabel)) stash_data_issue(parser="survex", message=message, url=None, sb=(includelabel))
# print(f' # datastack in LoadSurvexFile:{includelabel}', file=sys.stderr) # print(f' # datastack in LoadSurvexFile:{includelabel}', file=sys.stderr)
# for dict in self.datastack: # for dict in self.datastack:
# print(f' type: <{dict["type"].upper()} >', file=sys.stderr) # print(f' type: <{dict["type"].upper()} >', file=sys.stderr)
@ -1140,7 +1245,7 @@ class LoadingSurvex:
message = f' ! QM Unrecognised as valid in "{survexblock.survexfile.path}" QM{qml.group(1)} "{qml.group(2)}" : regex failure, typo?' message = f' ! QM Unrecognised as valid in "{survexblock.survexfile.path}" QM{qml.group(1)} "{qml.group(2)}" : regex failure, typo?'
print(message) print(message)
stash_data_issue( stash_data_issue(
parser="survex", message=message, url=get_offending_filename(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
included = self.rx_comminc.match(comment) included = self.rx_comminc.match(comment)
@ -1306,7 +1411,7 @@ class LoadingSurvex:
self.unitsstack.append((self.units, self.unitsfactor)) self.unitsstack.append((self.units, self.unitsfactor))
self.legsnumberstack.append(self.legsnumber) self.legsnumberstack.append(self.legsnumber)
self.slengthstack.append(self.slength) self.slengthstack.append(self.slength)
self.personexpedstack.append(self.currentpersonexped) self.personexpedstack.append(self.currentpersonexped) # just one person?!
pushblock() pushblock()
# PUSH state ++++++++++++++ # PUSH state ++++++++++++++
self.legsnumber = 0 self.legsnumber = 0
@ -1351,7 +1456,7 @@ class LoadingSurvex:
raise raise
# POP state ++++++++++++++ # POP state ++++++++++++++
popblock() popblock()
self.currentpersonexped = self.personexpedstack.pop() self.currentpersonexped = self.personexpedstack.pop() # just one person?!
self.legsnumber = self.legsnumberstack.pop() self.legsnumber = self.legsnumberstack.pop()
self.units, self.unitsfactor = self.unitsstack.pop() self.units, self.unitsfactor = self.unitsstack.pop()
self.slength = self.slengthstack.pop() self.slength = self.slengthstack.pop()
@ -1364,7 +1469,7 @@ class LoadingSurvex:
# ----------------------------- # -----------------------------
elif self.rx_title.match(cmd): elif self.rx_title.match(cmd):
quotedtitle = re.match('(?i)^"(.*)"$', args) quotedtitle = self.rx_quotedtitle.match(args)
if quotedtitle: if quotedtitle:
survexblock.title = quotedtitle.groups()[0] survexblock.title = quotedtitle.groups()[0]
else: else:
@ -1463,7 +1568,7 @@ class LoadingSurvex:
) )
print(message) print(message)
print(message, file=sys.stderr) print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=get_offending_filename(path)) stash_data_issue(parser="survex", message=message, url=None, sb=(path))
return # skip this survex file and all things *included in it return # skip this survex file and all things *included in it
includestmt = self.rx_include.match(svxline) includestmt = self.rx_include.match(svxline)
@ -1475,7 +1580,9 @@ class LoadingSurvex:
if star: # yes we are reading a *cmd if star: # yes we are reading a *cmd
cmd, args = star.groups() cmd, args = star.groups()
cmd = cmd.lower() cmd = cmd.lower()
if re.match("(?i)include$", cmd): if self.rx_include2.match(cmd):
# rx_include2 = re.compile("(?i)include$")
# if re.match("(?i)include$", cmd):
includepath = os.path.normpath(os.path.join(os.path.split(path)[0], re.sub(r"\.svx$", "", args))) includepath = os.path.normpath(os.path.join(os.path.split(path)[0], re.sub(r"\.svx$", "", args)))
fullpath = os.path.join(settings.SURVEX_DATA, includepath + ".svx") fullpath = os.path.join(settings.SURVEX_DATA, includepath + ".svx")
@ -1499,7 +1606,7 @@ class LoadingSurvex:
print(message) print(message)
print(message, file=flinear) print(message, file=flinear)
print(message, file=sys.stderr) print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=get_offending_filename(path)) stash_data_issue(parser="survex", message=message, url=None, sb=(path))
flinear.write(f"{self.depthinclude:2} {indent} *edulcni {pop}\n") flinear.write(f"{self.depthinclude:2} {indent} *edulcni {pop}\n")
fcollate.write(f";|*edulcni {pop}\n") fcollate.write(f";|*edulcni {pop}\n")
# fininclude.close() # fininclude.close()
@ -1509,7 +1616,7 @@ class LoadingSurvex:
message = f" ! ERROR *include file '{includepath}' not found, listed in '{fin.name}'" message = f" ! ERROR *include file '{includepath}' not found, listed in '{fin.name}'"
print(message) print(message)
print(message, file=sys.stderr) print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=get_offending_filename(path)) stash_data_issue(parser="survex", message=message, url=None, sb=(path))
elif re.match("(?i)begin$", cmd): elif re.match("(?i)begin$", cmd):
self.depthbegin += 1 self.depthbegin += 1
depth = " " * self.depthbegin depth = " " * self.depthbegin
@ -1533,7 +1640,7 @@ class LoadingSurvex:
print(message) print(message)
print(message, file=flinear) print(message, file=flinear)
print(message, file=sys.stderr) print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=get_offending_filename(path)) stash_data_issue(parser="survex", message=message, url=None, sb=(path))
self.depthbegin -= 1 self.depthbegin -= 1
pass pass
@ -1561,13 +1668,13 @@ class LoadingSurvex:
print(message) print(message)
print(message, file=flinear) print(message, file=flinear)
# print(message,file=sys.stderr) # print(message,file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=get_offending_filename(path)) stash_data_issue(parser="survex", message=message, url=None, sb=(path))
if self.svxfileslist.count(path) > 2: if self.svxfileslist.count(path) > 2:
message = f" ! ERROR. Should have been caught before this. Survex file already *included 2x. Probably an infinite loop so fix your *include statements that include this. Aborting. {path}" message = f" ! ERROR. Should have been caught before this. Survex file already *included 2x. Probably an infinite loop so fix your *include statements that include this. Aborting. {path}"
print(message) print(message)
print(message, file=flinear) print(message, file=flinear)
# print(message,file=sys.stderr) # print(message,file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=get_offending_filename(path)) stash_data_issue(parser="survex", message=message, url=None, sb=(path))
return return
return return
try: try:
@ -1583,13 +1690,13 @@ class LoadingSurvex:
message = f" ! ERROR *include file '{path}' in '{survexblock}' has UnicodeDecodeError. Omitted." message = f" ! ERROR *include file '{path}' in '{survexblock}' has UnicodeDecodeError. Omitted."
print(message) print(message)
print(message, file=sys.stderr) print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=get_offending_filename(path)) stash_data_issue(parser="survex", message=message, url=None, sb=(path))
return # skip this survex file and all things *included in it return # skip this survex file and all things *included in it
except: except:
message = f" ! ERROR *include file '{path}' in '{survexblock}' has unexpected error. Omitted." message = f" ! ERROR *include file '{path}' in '{survexblock}' has unexpected error. Omitted."
print(message) print(message)
print(message, file=sys.stderr) print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=get_offending_filename(path)) stash_data_issue(parser="survex", message=message, url=None, sb=(path))
return # skip this survex file and all things *included in it return # skip this survex file and all things *included in it
def checkUniqueness(self, fullpath): def checkUniqueness(self, fullpath):
@ -1960,6 +2067,7 @@ def LoadSurvexBlocks():
DataIssue.objects.filter(parser="svxdate").delete() DataIssue.objects.filter(parser="svxdate").delete()
DataIssue.objects.filter(parser="survexleg").delete() DataIssue.objects.filter(parser="survexleg").delete()
DataIssue.objects.filter(parser="survexunits").delete() DataIssue.objects.filter(parser="survexunits").delete()
DataIssue.objects.filter(parser="survex team").delete()
DataIssue.objects.filter(parser="entrances").delete() DataIssue.objects.filter(parser="entrances").delete()
DataIssue.objects.filter(parser="xEntrances").delete() DataIssue.objects.filter(parser="xEntrances").delete()
print(" - survex Data Issues flushed") print(" - survex Data Issues flushed")
@ -1993,12 +2101,19 @@ def LoadSurvexBlocks():
print(f" - MEMORY start:{memstart:.3f} MB end:{memend:.3f} MB increase={memend - memstart:.3f} MB") print(f" - MEMORY start:{memstart:.3f} MB end:{memend:.3f} MB increase={memend - memstart:.3f} MB")
survexblockroot.save() survexblockroot.save()
global person_pending_cache
for sb in person_pending_cache:
if len(person_pending_cache[sb]) > 0:
print(f" ")
message = f" ! PENDING team list not emptied {sb.survexfile.path} {len(person_pending_cache[sb])} people: {person_pending_cache[sb]}"
stash_data_issue(parser="survex", message=message, url=None, sb=(sb.survexfile.path))
print(message)
# duration = time.time() - start # duration = time.time() - start
# print(f" - TIME: {duration:7.2f} s", file=sys.stderr) # print(f" - TIME: {duration:7.2f} s", file=sys.stderr)
store_data_issues() store_data_issues()
# duration = time.time() - start # duration = time.time() - start
# print(f" - TIME: {duration:7.2f} s", file=sys.stderr) # print(f" - TIME: {duration:7.2f} s", file=sys.stderr)
print(" - Loaded All Survex Blocks.") print(" - Loaded All Survex Blocks.")