diff --git a/core/views/logbooks.py b/core/views/logbooks.py index 9748aed..ece3f98 100644 --- a/core/views/logbooks.py +++ b/core/views/logbooks.py @@ -148,9 +148,7 @@ def person(request, first_name='', last_name='', ): def get_person_chronology(personexpedition): - '''Horrible bug here when there is more than one survex block per day, it duplicates the entry but gets it wrong - Fortunately this is just the display on this page which is wroing, no bad calculations get into the database. - + ''' This is just a nasty convoluted way of trying the make the template do more work than it is sensible to ask it to do. Rewrite more simply with the login in the python, not in Django template language (you bastard Curtis). ''' @@ -160,8 +158,9 @@ def get_person_chronology(personexpedition): a.setdefault("persontrips", [ ]).append(persontrip) for personrole in personexpedition.survexpersonrole_set.all(): - a = res.setdefault(personrole.survexblock.date, { }) - a.setdefault("personroles", [ ]).append(personrole.survexblock) + if personrole.survexblock.date: # avoid bad data from another bug + a = res.setdefault(personrole.survexblock.date, { }) + a.setdefault("personroles", [ ]).append(personrole.survexblock) # build up the tables rdates = sorted(list(res.keys())) diff --git a/parsers/survex.py b/parsers/survex.py index 985517b..b741e30 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -21,6 +21,8 @@ It also scans the Loser repo for all the svx files, which it loads individually """ todo = """ +- Lots to do to cut down on unnecessary .save() calls to avoid hitting the db so much. SHould +speed it up noticably. - LoadSurvexFile() Creates a new current survexfile and valid .survexdirectory The survexblock passed-in is not necessarily the parent. FIX THIS. @@ -119,7 +121,7 @@ def get_offending_filename(path): """ return "/survexfile/" + path + ".svx" -trip_people_cache = {} +trip_people_cache = {} # DANGEROUS, should clean it on PUSH/POP begin/end def get_team_on_trip(survexblock): """Uses a cache to avoid a database query if it doesn't need to. Only used for complete team.""" @@ -143,7 +145,7 @@ def get_people_on_trip(survexblock): return list(set(people)) -trip_person_cache = {} +trip_person_cache = {} # pre survexblock, so robust wrt PUSH/POP begin/end def put_person_on_trip(survexblock, personexpedition, tm): """Uses a cache to avoid a database query if it doesn't need to. Only used for a single person""" @@ -168,21 +170,20 @@ def put_person_on_trip(survexblock, personexpedition, tm): trip_person_cache[(survexblock, personexpedition)] = 1 return False -person_pending_cache = {} +person_pending_cache = {} # pre survexblock, so robust wrt PUSH/POP begin/end def add_to_pending(survexblock, tm): + """Collects team names before we have a date so cannot validate against + expo attendance yet""" global person_pending_cache if survexblock not in person_pending_cache: person_pending_cache[survexblock] = set() person_pending_cache[survexblock].add(tm) - # personexpedition = None - # personrole, created = SurvexPersonRole.objects.update_or_create( - # survexblock=survexblock, personexpedition=personexpedition, personname=tm) - # personrole.save() def get_team_pending(survexblock): - """A set of *team names before we get to the *date line in a survexblock""" + """A set of *team names before we get to the *date line in a survexblock + """ global person_pending_cache if survexblock in person_pending_cache: @@ -267,7 +268,8 @@ class LoadingSurvex: unitsstack = [] legsnumberstack = [] slengthstack = [] - teamexpedstack = [] + teaminheritstack = [] + teamcurrentstack = [] stackbegin = [] flagsstack = [] datastack = [] @@ -297,7 +299,8 @@ class LoadingSurvex: currentsurvexfile = None currentcave = None caverndate = None - currentteamexped = [] + currentteam = set() + inheritteam = set() pending = [] def __init__(self): @@ -327,6 +330,47 @@ class LoadingSurvex: parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) ) + def get_team_inherited(self, survexblock): # survexblock only used for debug mesgs + """See get_team_pending(survexblock) which gets called at the same time, + when we see a *date line""" + global person_pending_cache + + if self.inheritteam: + message = ( + f"- INHERITING ({survexblock.parent})>({survexblock}) {survexblock.survexfile.path} '{self.inheritteam}'" + ) + print(self.insp + message) + # stash_data_issue( + # parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + # ) + return self.inheritteam + + def fix_anonymous(self, survexblock): + """Called when we reach *end of a block + Checks to see if the block has no team attached, in which case it uses the + inherited team. + If the block has no date, then it is assumed to be an abstract container, + with no relevant team, and anyway we can't attach a PersonExpedition without + knowing the year. Unless its parent has an identified expo""" + + if survexblock.parent.name == "troggle_unseens": + # Bolluxed up if we try to inherit from this random junk + return + + expo = survexblock.expedition # may be None if no *date yet + if not expo: + expo = survexblock.parent.expedition # immediate parent works mostly + if not expo: + return + + if not self.currentteam: # i.e. if it is a dated block and has no team + if teamnames := self.get_team_inherited(survexblock):# WALRUS + for tm in teamnames: + personexpedition = GetPersonExpeditionNameLookup(expo).get(tm.lower()) + if personexpedition: + put_person_on_trip(survexblock, personexpedition, tm) + return + def LoadSurvexTeam(self, survexblock, line): """Interpeting the *team fields means interpreting older style survex as well as current survex standard, *team Insts Anthony Day - this is how most of our files specify the team member @@ -341,17 +385,14 @@ class LoadingSurvex: def record_team_member(tm, survexblock): tm = tm.strip("\"'").strip() # Refactor. The dict GetPersonExpeditionNameLookup(expo) indexes by name and has values of personexpedition - # This is convoluted, the whole personexpedition concept is unnecessary. + # This is convoluted # we need the current expedition, but if there has been no date yet in the survex file, we don't know which one it is. # so we can't validate whether the person was on expo or not. # we will have to attach them to the survexblock anyway, and then do a # later check on whether they are valid when we get the date. - # We have hundreds of updated Django database updates when the same person is - # on the same trip in multiple roles. We should de-duplicate these ourselves in Python - # instead of using SurvexPersonRole.objects.update_or_create() which is expensive. - + self.currentteam.add(tm) # used in push/pop block code expo = survexblock.expedition # may be None if no *date yet if expo: @@ -366,7 +407,7 @@ class LoadingSurvex: personexpedition = GetPersonExpeditionNameLookup(expo).get(tm.lower()) if personexpedition: put_person_on_trip(survexblock, personexpedition, tm) - self.currentteamexped.append(personexpedition) # used in push/pop block code + elif known_foreigner(tm): # note, not using .lower() message = f"- *team {expo.year} '{tm}' known foreigner on *team {survexblock.survexfile.path} ({survexblock}) in '{line}'" print(self.insp + message) @@ -523,7 +564,7 @@ class LoadingSurvex: pe = GetPersonExpeditionNameLookup(expo).get(tm.lower()) if pe: put_person_on_trip(survexblock, pe, tm) - self.currentteamexped.append(pe) + else: message = f"! *team {year} '{tm}' FAIL personexpedition lookup on *date {survexblock.survexfile.path} ({survexblock}) " print(self.insp + message) @@ -531,38 +572,39 @@ class LoadingSurvex: parser="survex", message=message, url=None, sb=(survexblock.survexfile.path), - ) - + ) + + # All this next section should not happen unless there are >1 *date lines in a block - for pr in team: # pr is a PersonRole object - if not pr.expeditionday: # *date and *team in 'wrong' order. + # for pr in team: # pr is a PersonRole object + # if not pr.expeditionday: # *date and *team in 'wrong' order. - pr.expeditionday = survexblock.expeditionday - pr.save() + # pr.expeditionday = survexblock.expeditionday + # pr.save() - if not pr.personexpedition: - pe = GetPersonExpeditionNameLookup(expo).get(pr.personname.lower()) - if pe: # pe is a PersonExpedition - # message = "! {} ({}) Fixing undated personexpedition '{}'".format(survexblock.survexfile.path, survexblock, p.personname) - # print(self.insp+message) - # stash_data_issue(parser='survex', message=message) - pr.personexpedition = pe - pr.person = pr.personexpedition.person - pr.save() - self.currentteamexped.append(pe) # used in push/pop block code - elif known_foreigner(pr.personname): # note, not using .lower() - message = f"- *team {expo.year} '{pr.personname}' known foreigner on *date {survexblock.survexfile.path} ({survexblock}) in '{line}'" - print(self.insp + message) - # stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path)) - else: - message = f"! *team {year} '{pr.personname}' FAIL personexpedition lookup on *date {survexblock.survexfile.path} ({survexblock}) '{pr.personname}'" - print(self.insp + message) - stash_data_issue( - parser="survex", - message=message, - url=None, sb=(survexblock.survexfile.path), - ) + # if not pr.personexpedition: + # pe = GetPersonExpeditionNameLookup(expo).get(pr.personname.lower()) + # if pe: # pe is a PersonExpedition + # # message = "! {} ({}) Fixing undated personexpedition '{}'".format(survexblock.survexfile.path, survexblock, p.personname) + # # print(self.insp+message) + # # stash_data_issue(parser='survex', message=message) + # pr.personexpedition = pe + # pr.person = pr.personexpedition.person + # pr.save() + + # elif known_foreigner(pr.personname): # note, not using .lower() + # message = f"- *team {expo.year} '{pr.personname}' known foreigner on *date {survexblock.survexfile.path} ({survexblock}) in '{line}'" + # print(self.insp + message) + # # stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path)) + # else: + # message = f"! *team {year} '{pr.personname}' FAIL personexpedition lookup on *date {survexblock.survexfile.path} ({survexblock}) '{pr.personname}'" + # print(self.insp + message) + # stash_data_issue( + # parser="survex", + # message=message, + # url=None, sb=(survexblock.survexfile.path), + # ) oline = line if len(line) > 10: @@ -1218,12 +1260,12 @@ class LoadingSurvex: # rx_ref2 = re.compile(r'(?i)\s*ref[.;]?') # This should also check that the QM survey point rxists in the block + depth = " " * self.depthbegin refline = self.rx_commref.match(comment) if refline: - # comment = re.sub('(?i)\s*ref[.;]?',"",comment.strip()) comment = self.rx_ref2.sub("", comment.strip()) - print(f"rx_ref2 -- {comment=} in {survexblock.survexfile.path} :: {survexblock}") + print(f"{self.depthbegin:2}{depth} - rx_ref2 -- {comment=} in {survexblock.survexfile.path} :: {survexblock}") self.LoadSurvexRef(survexblock, comment) # handle @@ -1411,17 +1453,20 @@ class LoadingSurvex: if self.rx_begin.match(cmd): blkid = args.lower() # PUSH state ++++++++++++++ + self.depthbegin += 1 self.stackbegin.append(blkid) self.unitsstack.append((self.units, self.unitsfactor)) self.legsnumberstack.append(self.legsnumber) self.slengthstack.append(self.slength) - self.teamexpedstack.append(self.currentteamexped) # just one person?! + self.teaminheritstack.append(self.inheritteam) + self.teamcurrentstack.append(self.currentteam) pushblock() # PUSH state ++++++++++++++ self.legsnumber = 0 self.slength = 0.0 self.units = "metres" - self.currentteamexped = [] + self.inheritteam = self.currentteam + self.currentteam = set() # zero the current team when we start a new block printbegin() newsurvexblock = SurvexBlock( name=blkid, @@ -1447,6 +1492,8 @@ class LoadingSurvex: printend() slengthtotal += self.slength nlegstotal += self.legsnumber + + self.fix_anonymous(survexblock) try: survexblock.parent.save() # django insists on this although it is already saved !? @@ -1460,7 +1507,8 @@ class LoadingSurvex: raise # POP state ++++++++++++++ popblock() - self.currentteamexped = self.teamexpedstack.pop() # just one person?! + self.inheritteam = self.teaminheritstack.pop() + self.currentteam = self.teamcurrentstack.pop() self.legsnumber = self.legsnumberstack.pop() self.units, self.unitsfactor = self.unitsstack.pop() self.slength = self.slengthstack.pop() @@ -1917,10 +1965,10 @@ def FindAndLoadSurvex(survexblockroot): ) u.write(f"; autogenerated by parser/survex.py from databasereset.py on '{datetime.now(timezone.utc)}'\n") u.write(f"; omitting any file beginning with {excpts}\n\n") - u.write("*begin unseens\n") + u.write("*begin troggle_unseens\n") for x in sorted(unseens): u.write(f" *include {x}\n") - u.write("*end unseens\n") + u.write("*end troggle_unseens\n") survexfileroot = survexblockroot.survexfile # i.e. SURVEX_TOPNAME only