expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-22 07:11:52 +00:00
Code Issues 19 Projects 3 Activity

Inherit *team to anonymous survex blocks

Browse Source
This commit is contained in:
Philip Sargent 2023-01-29 01:30:10 +00:00
parent 03fa5f5548
commit 071f68080c
2 changed files with 104 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
core/views/logbooks.py
View File

@ -148,9 +148,7 @@ def person(request, first_name='', last_name='', ):
def get_person_chronology(personexpedition): def get_person_chronology(personexpedition):
'''Horrible bug here when there is more than one survex block per day, it duplicates the entry but gets it wrong '''
Fortunately this is just the display on this page which is wroing, no bad calculations get into the database.
This is just a nasty convoluted way of trying the make the template do more work than it is sensible to ask it to do. This is just a nasty convoluted way of trying the make the template do more work than it is sensible to ask it to do.
Rewrite more simply with the login in the python, not in Django template language (you bastard Curtis). Rewrite more simply with the login in the python, not in Django template language (you bastard Curtis).
''' '''
@ -160,8 +158,9 @@ def get_person_chronology(personexpedition):
a.setdefault("persontrips", [ ]).append(persontrip) a.setdefault("persontrips", [ ]).append(persontrip)
for personrole in personexpedition.survexpersonrole_set.all(): for personrole in personexpedition.survexpersonrole_set.all():
a = res.setdefault(personrole.survexblock.date, { }) if personrole.survexblock.date: # avoid bad data from another bug
a.setdefault("personroles", [ ]).append(personrole.survexblock) a = res.setdefault(personrole.survexblock.date, { })
a.setdefault("personroles", [ ]).append(personrole.survexblock)
# build up the tables # build up the tables
rdates = sorted(list(res.keys())) rdates = sorted(list(res.keys()))

152
parsers/survex.py
View File

@ -21,6 +21,8 @@ It also scans the Loser repo for all the svx files, which it loads individually
""" """
todo = """ todo = """
- Lots to do to cut down on unnecessary .save() calls to avoid hitting the db so much. SHould
speed it up noticably.
- LoadSurvexFile() Creates a new current survexfile and valid .survexdirectory - LoadSurvexFile() Creates a new current survexfile and valid .survexdirectory
The survexblock passed-in is not necessarily the parent. FIX THIS. The survexblock passed-in is not necessarily the parent. FIX THIS.
@ -119,7 +121,7 @@ def get_offending_filename(path):
""" """
return "/survexfile/" + path + ".svx" return "/survexfile/" + path + ".svx"
trip_people_cache = {} trip_people_cache = {} # DANGEROUS, should clean it on PUSH/POP begin/end
def get_team_on_trip(survexblock): def get_team_on_trip(survexblock):
"""Uses a cache to avoid a database query if it doesn't need to. """Uses a cache to avoid a database query if it doesn't need to.
Only used for complete team.""" Only used for complete team."""
@ -143,7 +145,7 @@ def get_people_on_trip(survexblock):
return list(set(people)) return list(set(people))
trip_person_cache = {} trip_person_cache = {} # pre survexblock, so robust wrt PUSH/POP begin/end
def put_person_on_trip(survexblock, personexpedition, tm): def put_person_on_trip(survexblock, personexpedition, tm):
"""Uses a cache to avoid a database query if it doesn't need to. """Uses a cache to avoid a database query if it doesn't need to.
Only used for a single person""" Only used for a single person"""
@ -168,21 +170,20 @@ def put_person_on_trip(survexblock, personexpedition, tm):
trip_person_cache[(survexblock, personexpedition)] = 1 trip_person_cache[(survexblock, personexpedition)] = 1
return False return False
person_pending_cache = {} person_pending_cache = {} # pre survexblock, so robust wrt PUSH/POP begin/end
def add_to_pending(survexblock, tm): def add_to_pending(survexblock, tm):
"""Collects team names before we have a date so cannot validate against
expo attendance yet"""
global person_pending_cache global person_pending_cache
if survexblock not in person_pending_cache: if survexblock not in person_pending_cache:
person_pending_cache[survexblock] = set() person_pending_cache[survexblock] = set()
person_pending_cache[survexblock].add(tm) person_pending_cache[survexblock].add(tm)
# personexpedition = None
# personrole, created = SurvexPersonRole.objects.update_or_create(
# survexblock=survexblock, personexpedition=personexpedition, personname=tm)
# personrole.save()
def get_team_pending(survexblock): def get_team_pending(survexblock):
"""A set of *team names before we get to the *date line in a survexblock""" """A set of *team names before we get to the *date line in a survexblock
"""
global person_pending_cache global person_pending_cache
if survexblock in person_pending_cache: if survexblock in person_pending_cache:
@ -267,7 +268,8 @@ class LoadingSurvex:
unitsstack = [] unitsstack = []
legsnumberstack = [] legsnumberstack = []
slengthstack = [] slengthstack = []
teamexpedstack = [] teaminheritstack = []
teamcurrentstack = []
stackbegin = [] stackbegin = []
flagsstack = [] flagsstack = []
datastack = [] datastack = []
@ -297,7 +299,8 @@ class LoadingSurvex:
currentsurvexfile = None currentsurvexfile = None
currentcave = None currentcave = None
caverndate = None caverndate = None
currentteamexped = [] currentteam = set()
inheritteam = set()
pending = [] pending = []
def __init__(self): def __init__(self):
@ -327,6 +330,47 @@ class LoadingSurvex:
parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
) )
def get_team_inherited(self, survexblock): # survexblock only used for debug mesgs
"""See get_team_pending(survexblock) which gets called at the same time,
when we see a *date line"""
global person_pending_cache
if self.inheritteam:
message = (
f"- INHERITING ({survexblock.parent})>({survexblock}) {survexblock.survexfile.path} '{self.inheritteam}'"
)
print(self.insp + message)
# stash_data_issue(
# parser="survex", message=message, url=None, sb=(survexblock.survexfile.path)
# )
return self.inheritteam
def fix_anonymous(self, survexblock):
"""Called when we reach *end of a block
Checks to see if the block has no team attached, in which case it uses the
inherited team.
If the block has no date, then it is assumed to be an abstract container,
with no relevant team, and anyway we can't attach a PersonExpedition without
knowing the year. Unless its parent has an identified expo"""
if survexblock.parent.name == "troggle_unseens":
# Bolluxed up if we try to inherit from this random junk
return
expo = survexblock.expedition # may be None if no *date yet
if not expo:
expo = survexblock.parent.expedition # immediate parent works mostly
if not expo:
return
if not self.currentteam: # i.e. if it is a dated block and has no team
if teamnames := self.get_team_inherited(survexblock):# WALRUS
for tm in teamnames:
personexpedition = GetPersonExpeditionNameLookup(expo).get(tm.lower())
if personexpedition:
put_person_on_trip(survexblock, personexpedition, tm)
return
def LoadSurvexTeam(self, survexblock, line): def LoadSurvexTeam(self, survexblock, line):
"""Interpeting the *team fields means interpreting older style survex as well as current survex standard, """Interpeting the *team fields means interpreting older style survex as well as current survex standard,
*team Insts Anthony Day - this is how most of our files specify the team member *team Insts Anthony Day - this is how most of our files specify the team member
@ -341,17 +385,14 @@ class LoadingSurvex:
def record_team_member(tm, survexblock): def record_team_member(tm, survexblock):
tm = tm.strip("\"'").strip() tm = tm.strip("\"'").strip()
# Refactor. The dict GetPersonExpeditionNameLookup(expo) indexes by name and has values of personexpedition # Refactor. The dict GetPersonExpeditionNameLookup(expo) indexes by name and has values of personexpedition
# This is convoluted, the whole personexpedition concept is unnecessary. # This is convoluted
# we need the current expedition, but if there has been no date yet in the survex file, we don't know which one it is. # we need the current expedition, but if there has been no date yet in the survex file, we don't know which one it is.
# so we can't validate whether the person was on expo or not. # so we can't validate whether the person was on expo or not.
# we will have to attach them to the survexblock anyway, and then do a # we will have to attach them to the survexblock anyway, and then do a
# later check on whether they are valid when we get the date. # later check on whether they are valid when we get the date.
# We have hundreds of updated Django database updates when the same person is self.currentteam.add(tm) # used in push/pop block code
# on the same trip in multiple roles. We should de-duplicate these ourselves in Python
# instead of using SurvexPersonRole.objects.update_or_create() which is expensive.
expo = survexblock.expedition # may be None if no *date yet expo = survexblock.expedition # may be None if no *date yet
if expo: if expo:
@ -366,7 +407,7 @@ class LoadingSurvex:
personexpedition = GetPersonExpeditionNameLookup(expo).get(tm.lower()) personexpedition = GetPersonExpeditionNameLookup(expo).get(tm.lower())
if personexpedition: if personexpedition:
put_person_on_trip(survexblock, personexpedition, tm) put_person_on_trip(survexblock, personexpedition, tm)
self.currentteamexped.append(personexpedition) # used in push/pop block code
elif known_foreigner(tm): # note, not using .lower() elif known_foreigner(tm): # note, not using .lower()
message = f"- *team {expo.year} '{tm}' known foreigner on *team {survexblock.survexfile.path} ({survexblock}) in '{line}'" message = f"- *team {expo.year} '{tm}' known foreigner on *team {survexblock.survexfile.path} ({survexblock}) in '{line}'"
print(self.insp + message) print(self.insp + message)
@ -523,7 +564,7 @@ class LoadingSurvex:
pe = GetPersonExpeditionNameLookup(expo).get(tm.lower()) pe = GetPersonExpeditionNameLookup(expo).get(tm.lower())
if pe: if pe:
put_person_on_trip(survexblock, pe, tm) put_person_on_trip(survexblock, pe, tm)
self.currentteamexped.append(pe)
else: else:
message = f"! *team {year} '{tm}' FAIL personexpedition lookup on *date {survexblock.survexfile.path} ({survexblock}) " message = f"! *team {year} '{tm}' FAIL personexpedition lookup on *date {survexblock.survexfile.path} ({survexblock}) "
print(self.insp + message) print(self.insp + message)
@ -531,38 +572,39 @@ class LoadingSurvex:
parser="survex", parser="survex",
message=message, message=message,
url=None, sb=(survexblock.survexfile.path), url=None, sb=(survexblock.survexfile.path),
) )
# All this next section should not happen unless there are >1 *date lines in a block # All this next section should not happen unless there are >1 *date lines in a block
for pr in team: # pr is a PersonRole object # for pr in team: # pr is a PersonRole object
if not pr.expeditionday: # *date and *team in 'wrong' order. # if not pr.expeditionday: # *date and *team in 'wrong' order.
pr.expeditionday = survexblock.expeditionday # pr.expeditionday = survexblock.expeditionday
pr.save() # pr.save()
if not pr.personexpedition: # if not pr.personexpedition:
pe = GetPersonExpeditionNameLookup(expo).get(pr.personname.lower()) # pe = GetPersonExpeditionNameLookup(expo).get(pr.personname.lower())
if pe: # pe is a PersonExpedition # if pe: # pe is a PersonExpedition
# message = "! {} ({}) Fixing undated personexpedition '{}'".format(survexblock.survexfile.path, survexblock, p.personname) # # message = "! {} ({}) Fixing undated personexpedition '{}'".format(survexblock.survexfile.path, survexblock, p.personname)
# print(self.insp+message) # # print(self.insp+message)
# stash_data_issue(parser='survex', message=message) # # stash_data_issue(parser='survex', message=message)
pr.personexpedition = pe # pr.personexpedition = pe
pr.person = pr.personexpedition.person # pr.person = pr.personexpedition.person
pr.save() # pr.save()
self.currentteamexped.append(pe) # used in push/pop block code
elif known_foreigner(pr.personname): # note, not using .lower() # elif known_foreigner(pr.personname): # note, not using .lower()
message = f"- *team {expo.year} '{pr.personname}' known foreigner on *date {survexblock.survexfile.path} ({survexblock}) in '{line}'" # message = f"- *team {expo.year} '{pr.personname}' known foreigner on *date {survexblock.survexfile.path} ({survexblock}) in '{line}'"
print(self.insp + message) # print(self.insp + message)
# stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path)) # # stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path))
else: # else:
message = f"! *team {year} '{pr.personname}' FAIL personexpedition lookup on *date {survexblock.survexfile.path} ({survexblock}) '{pr.personname}'" # message = f"! *team {year} '{pr.personname}' FAIL personexpedition lookup on *date {survexblock.survexfile.path} ({survexblock}) '{pr.personname}'"
print(self.insp + message) # print(self.insp + message)
stash_data_issue( # stash_data_issue(
parser="survex", # parser="survex",
message=message, # message=message,
url=None, sb=(survexblock.survexfile.path), # url=None, sb=(survexblock.survexfile.path),
) # )
oline = line oline = line
if len(line) > 10: if len(line) > 10:
@ -1218,12 +1260,12 @@ class LoadingSurvex:
# rx_ref2 = re.compile(r'(?i)\s*ref[.;]?') # rx_ref2 = re.compile(r'(?i)\s*ref[.;]?')
# This should also check that the QM survey point rxists in the block # This should also check that the QM survey point rxists in the block
depth = " " * self.depthbegin
refline = self.rx_commref.match(comment) refline = self.rx_commref.match(comment)
if refline: if refline:
# comment = re.sub('(?i)\s*ref[.;]?',"",comment.strip())
comment = self.rx_ref2.sub("", comment.strip()) comment = self.rx_ref2.sub("", comment.strip())
print(f"rx_ref2 -- {comment=} in {survexblock.survexfile.path} :: {survexblock}") print(f"{self.depthbegin:2}{depth} - rx_ref2 -- {comment=} in {survexblock.survexfile.path} :: {survexblock}")
self.LoadSurvexRef(survexblock, comment) self.LoadSurvexRef(survexblock, comment)
# handle # handle
@ -1411,17 +1453,20 @@ class LoadingSurvex:
if self.rx_begin.match(cmd): if self.rx_begin.match(cmd):
blkid = args.lower() blkid = args.lower()
# PUSH state ++++++++++++++ # PUSH state ++++++++++++++
self.depthbegin += 1
self.stackbegin.append(blkid) self.stackbegin.append(blkid)
self.unitsstack.append((self.units, self.unitsfactor)) self.unitsstack.append((self.units, self.unitsfactor))
self.legsnumberstack.append(self.legsnumber) self.legsnumberstack.append(self.legsnumber)
self.slengthstack.append(self.slength) self.slengthstack.append(self.slength)
self.teamexpedstack.append(self.currentteamexped) # just one person?! self.teaminheritstack.append(self.inheritteam)
self.teamcurrentstack.append(self.currentteam)
pushblock() pushblock()
# PUSH state ++++++++++++++ # PUSH state ++++++++++++++
self.legsnumber = 0 self.legsnumber = 0
self.slength = 0.0 self.slength = 0.0
self.units = "metres" self.units = "metres"
self.currentteamexped = [] self.inheritteam = self.currentteam
self.currentteam = set() # zero the current team when we start a new block
printbegin() printbegin()
newsurvexblock = SurvexBlock( newsurvexblock = SurvexBlock(
name=blkid, name=blkid,
@ -1447,6 +1492,8 @@ class LoadingSurvex:
printend() printend()
slengthtotal += self.slength slengthtotal += self.slength
nlegstotal += self.legsnumber nlegstotal += self.legsnumber
self.fix_anonymous(survexblock)
try: try:
survexblock.parent.save() # django insists on this although it is already saved !? survexblock.parent.save() # django insists on this although it is already saved !?
@ -1460,7 +1507,8 @@ class LoadingSurvex:
raise raise
# POP state ++++++++++++++ # POP state ++++++++++++++
popblock() popblock()
self.currentteamexped = self.teamexpedstack.pop() # just one person?! self.inheritteam = self.teaminheritstack.pop()
self.currentteam = self.teamcurrentstack.pop()
self.legsnumber = self.legsnumberstack.pop() self.legsnumber = self.legsnumberstack.pop()
self.units, self.unitsfactor = self.unitsstack.pop() self.units, self.unitsfactor = self.unitsstack.pop()
self.slength = self.slengthstack.pop() self.slength = self.slengthstack.pop()
@ -1917,10 +1965,10 @@ def FindAndLoadSurvex(survexblockroot):
) )
u.write(f"; autogenerated by parser/survex.py from databasereset.py on '{datetime.now(timezone.utc)}'\n") u.write(f"; autogenerated by parser/survex.py from databasereset.py on '{datetime.now(timezone.utc)}'\n")
u.write(f"; omitting any file beginning with {excpts}\n\n") u.write(f"; omitting any file beginning with {excpts}\n\n")
u.write("*begin unseens\n") u.write("*begin troggle_unseens\n")
for x in sorted(unseens): for x in sorted(unseens):
u.write(f" *include {x}\n") u.write(f" *include {x}\n")
u.write("*end unseens\n") u.write("*end troggle_unseens\n")
survexfileroot = survexblockroot.survexfile # i.e. SURVEX_TOPNAME only survexfileroot = survexblockroot.survexfile # i.e. SURVEX_TOPNAME only