diff --git a/core/models/survex.py b/core/models/survex.py index 3a1d2a4..4ea8c98 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -1,3 +1,4 @@ +import uuid import math import os import re @@ -218,6 +219,14 @@ class SurvexBlock(models.Model): Blocks can span several *included survexfile though. """ + # This ID is generated as soon as you call SurvexBlock((). So we can use it while assembling the data + # into the survexblock without having to keep doing a database transaction + _blockid = models.UUIDField( + primary_key=False, + default=uuid.uuid4, + editable=False + ) + objects = SurvexBlockLookUpManager() # overwrites SurvexBlock.objects and enables lookup() name = models.CharField(blank=True, max_length=100) title = models.CharField(blank=True, max_length=200) diff --git a/parsers/survex.py b/parsers/survex.py index b5817d3..b1672b2 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -80,9 +80,9 @@ dataissues = set() # Caches for ORM minimization survexblock_cache = None # {scanswallet_id: [SurvexBlock, ...]} -personrole_cache = None # {survexblock_id: [SurvexPersonRole, ...]} +personrole_cache = None # {survexblock._blockid: [SurvexPersonRole, ...]} wallet_cache = None # {walletname: [Wallet, ...]} -trip_people_cache = {} # indexed by survexblock, so never needs cleaning out +trip_people_cache = {} # indexed by survexblock._blockid, so never needs cleaning out class SurvexLeg: """No longer a models.Model subclass, so no longer a database table""" @@ -152,7 +152,11 @@ def store_data_issues(): parser, message, url, sb = issue if url is None: if sb is not None: - url = get_offending_filename(sb) + try: + url = get_offending_filename(sb.survexfile) + except Exception as e: + print(f" ! store_data_issues() {e} '{sb=}' -- '{url=}'", file=sys.stderr) + url = get_offending_filename(sb) di_list.append(DataIssue(parser=parser, message=message, url=url)) # Now commit to db DataIssue.objects.bulk_create(di_list) @@ -166,20 +170,28 @@ def get_offending_filename(path): def get_team_on_trip(survexblock): - """Uses a cache to avoid a database query if it doesn't need to. Only used for complete team.""" + """Uses a cache to avoid a database query if it doesn't need to. Only used for complete team. + + Should personrole_cache be a set() ? + + This all seems a bit baroque, can we refactor it please? + """ global trip_people_cache, personrole_cache if personrole_cache is None: - # Build cache: {survexblock_id: [SurvexPersonRole, ...]} + # Build cache: {survexblock._blockid: [SurvexPersonRole, ...]} personrole_cache = {} - for pr in SurvexPersonRole.objects.all().select_related("person", "personexpedition"): - if pr.survexblock_id not in personrole_cache: - personrole_cache[pr.survexblock_id] = [] - personrole_cache[pr.survexblock_id].append(pr) - if survexblock in trip_people_cache: - if len(trip_people_cache[survexblock]) > 0: - return trip_people_cache[survexblock] - qpeople = personrole_cache.get(survexblock.id, []) - trip_people_cache[survexblock] = qpeople + for pr in SurvexPersonRole.objects.all().select_related("person", "personexpedition"): # WASTEFUL ! Optimise this + if pr.survexblock._blockid not in personrole_cache: + personrole_cache[pr.survexblock._blockid] = [] + personrole_cache[pr.survexblock._blockid].append(pr) + # print(f" PR {pr} {survexblock._blockid }", file=sys.stderr) + + if survexblock._blockid in trip_people_cache: + if len(trip_people_cache[survexblock._blockid]) > 0: + return trip_people_cache[survexblock._blockid] + + qpeople = personrole_cache.get(survexblock._blockid, []) + trip_people_cache[survexblock._blockid] = qpeople return qpeople def get_people_on_trip(survexblock): @@ -196,14 +208,14 @@ def get_people_on_trip(survexblock): # THIS SHOULD NOT BE GLOBAL ! Should be per instance of file loader, even though they are globally unique trip_person_record = {} # a dict indexed by tuples (survexblock, personexpedition) = 1 -trip_team_cache = {} # a dict of lists indexed by survexblock +trip_team_cache = {} # a dict of lists indexed by survexblock._blockid def put_person_on_trip(survexblock, personexpedition, tm): """Uses a cache to avoid a database query if it doesn't need to. Only used for a single person""" global trip_person_record global trip_team_cache - if (survexblock, personexpedition) in trip_person_record: + if (survexblock._blockid, personexpedition) in trip_person_record: return True try: @@ -220,55 +232,44 @@ def put_person_on_trip(survexblock, personexpedition, tm): parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) ) - if survexblock not in trip_team_cache: - trip_team_cache[survexblock] = [] - trip_team_cache[survexblock].append(personrole) - print(f"-- trip_team_cache {survexblock}, {trip_team_cache[survexblock]}, {personrole}") + if survexblock._blockid not in trip_team_cache: + trip_team_cache[survexblock._blockid] = [] + trip_team_cache[survexblock._blockid].append(personrole) + # print(f"-- trip_team_cache\n -- {survexblock=} - {survexblock._blockid}\n -- {trip_team_cache[survexblock._blockid]}\n -- {personrole}", file=sys.stderr) - trip_person_record[(survexblock, personexpedition)] = 1 + trip_person_record[(survexblock._blockid, personexpedition)] = 1 return False -def confirm_team_on_trip(survexblock): - global trip_team_cache +def hack_save(survexblock): + # #### Horrible hack to be properly written as a cache + sb_list =[] + sb = survexblock + while sb.parent and sb != sb.parent: + sb_list.append((sb._blockid, sb)) + sb = sb.parent + # print(sb_list, file=sys.stderr) - if survexblock not in trip_team_cache: - return - # Now commit to db - SurvexPersonRole.objects.bulk_create(trip_team_cache[survexblock]) - trip_team_cache[survexblock] = [] # in database now, so empty cache - -def check_team_cache(label=None): - global trip_team_cache - message = f"! check_team_cache() called.. " - print(message) - print(message, file=sys.stderr) - for block in trip_team_cache: - message = f"! *team CACHEFAIL, trip_team_cache {block.survexfile.path} ({block}). label:{label}" - print(message) - print(message, file=sys.stderr) - -person_pending_cache = {} # indexed per survexblock, so robust wrt PUSH/POP begin/end -def add_to_pending(survexblock, tm): - """Collects team names before we have a date so cannot validate against - expo attendance yet""" - global person_pending_cache - - if survexblock not in person_pending_cache: - person_pending_cache[survexblock] = set() - person_pending_cache[survexblock].add(tm) - print(f"-- person_pending_cache {survexblock}, {person_pending_cache[survexblock]}, {tm}") + sb_list.reverse() + for sbo in sb_list: + id, sb = sbo + sb.save() + # #### Horrible hack to be properly written as a cache + -def get_team_pending(survexblock): - """A set of *team names added at the end of the survex block +def blockid_raw(survexfile, name): + if name and survexfile: + return f"{survexfile}-{name}" + return False + +def blockid(survexblock): + """When parsing all the survex file we need to maintain a number of caches as we want to + hit the database with updates only when we have collected all the data. + + But since we have not saved to the database, we don't have a unique survexblock.id that we can + use. So we have to roll our own. """ - global person_pending_cache - - if survexblock in person_pending_cache: - teamnames = person_pending_cache[survexblock] # a set of names - person_pending_cache[survexblock] = set() #re zero the cache - return teamnames - return - + return survexblock._blockid # new UUID + class LoadingSurvex: """A 'survex block' is a *begin...*end set of cave data. A survex file can contain many begin-end blocks, which can be nested, and which can *include @@ -489,6 +490,8 @@ class LoadingSurvex: inheritdate = None pending = [] adhocload = False + person_pending_cache = {} # indexed per survexblock UUID, so robust wrt PUSH/POP begin/end + def __init__(self): self.caveslist = GetCaveLookup() @@ -516,9 +519,79 @@ class LoadingSurvex: stash_data_issue( parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) ) + + def confirm_team_on_trip(self, survexblock): + """This is only called when processing a *end statement + """ + global trip_team_cache + + if survexblock._blockid not in trip_team_cache: + return + #### STRIP THIS OUT and cache the SurvexPersonRole for the end of the survex block ! + hack_save(survexblock) + + # Now commit to db + pr_list = trip_team_cache[survexblock._blockid] + # print(f" PR_LIST {pr_list} {survexblock._blockid }", file=sys.stderr) + valid_list = [] + for pr in pr_list: + try: + # print(f"___ {pr.survexblock=} {pr.survexblock.id=} {pr.person=} {pr.personexpedition=}", file=sys.stderr) + pr.full_clean() + valid_list.append(pr) + except ValidationError as e: + print(f" ! PR is invalid: {e} {survexblock} {pr}", file=sys.stderr) + print(f" ! PR is invalid: {e} {survexblock} {pr}") + + + SurvexPersonRole.objects.bulk_create(valid_list) + # for pr in pr_list: + # print(f"+++ {pr.survexblock=} {pr.survexblock.id=} {pr.person=} {pr.personexpedition=}", file=sys.stderr) + # SurvexPersonRole.objects.create(pr).save() + + # Not working, so do not clear cache! + trip_team_cache[survexblock] = [] # in database now, so empty cache + + def check_team_cache(self, label=None): + global trip_team_cache + message = f"! check_team_cache() called.. " + print(message) + print(message, file=sys.stderr) + for block in trip_team_cache: + message = f"! *team CACHEFAIL, trip_team_cache {block.survexfile.path} ({block}). label:{label}" + print(message) + print(message, file=sys.stderr) + + def add_to_pending(self, survexblock, tm): + """Collects team names before we have a date so cannot validate against + expo attendance yet""" + global person_pending_cache + + if survexblock._blockid not in self.person_pending_cache: + self.person_pending_cache[survexblock._blockid] = set() + self.person_pending_cache[survexblock._blockid].add(tm) + print(f"-- person_pending_cache {survexblock}, {self.person_pending_cache[survexblock._blockid]}, {tm}") + def get_team_pending(self, survexblock): + """A set of *team names added at the end of the survex block + """ + + if blockid in self.person_pending_cache: + teamnames = person_pending_cache[blockid] # a set of names + person_pending_cache[blockid] = set() #re zero the cache + return teamnames + return + + def check_cache_clean(self): + for sbid in self.person_pending_cache: + if len(person_pending_cache[sbid]) > 0: + print(f" ") + message = f" ! PENDING team list not emptied {sbid} {len(person_pending_cache[sbid])} people: {person_pending_cache[sbid]}" + stash_data_issue(parser="survex", message=message, url=None) #, sb=(sbid) + print(message) + def get_team_inherited(self, survexblock): # survexblock only used for debug mesgs - """See get_team_pending(survexblock) which gets called at the same time, + """See get_team_pending(survexblock._blockid) which gets called at the same time, when we see a *date line""" global person_pending_cache @@ -641,6 +714,10 @@ class LoadingSurvex: put_person_on_trip(survexblock, personexpedition, tm) return + def cache_survexblock(self, survexblock): + # appends to list, creates an empty list to append to if it doen't exist yet + self._pending_block_saves.setdefault(survexblock._blockid, []).append(survexblock) + def LoadSurvexTeam(self, survexblock, line): """Interpeting the *team fields has been updated to current 2025 survex standard, *team Insts Anthony Day - this is how most of our files used to specify the team member @@ -650,6 +727,9 @@ class LoadingSurvex: personrole is used to record that a person was on a survex trip, NOT the role they played. (NB PersonLogEntry is a logbook thing, not a survex thing. ) + + DONT do this here. Just collect the members, but wait untilt he *end before we use + the list to create anything in the db """ def record_team_member(tm, survexblock): @@ -688,13 +768,14 @@ class LoadingSurvex: parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) ) else: - add_to_pending(survexblock, tm) + self.add_to_pending(survexblock, tm) # don't know the date yet, so cannot query the table about validity. # assume the person is valid. It will get picked up with the *date appears # There are hundreds of these.. message = ( f"- Team before Date: {line} ({survexblock}) {survexblock.survexfile.path}" ) + # teamfix = r"(?i)(.*?)\s+" + roles + r"?(?:es|s)?$" -- (.*?) means a non-greedy capture if fixstyle := self.rx_teamfix.match(line): # matches the optional role at the the end of the string WALRUS tmlist = fixstyle.group(1).strip('\"') # remove quotes, if present @@ -747,7 +828,11 @@ class LoadingSurvex: def LoadSurvexFix(self, survexblock, line): """*fix is a station geolocation, units depend on a previous *cs setting NOTE 'line' is not the full line, it is 'arg' and the comments have been stripped ! - SO we have to recognise the '*fix' too + So we have to recognise the '*fix' too + + Note that the cache self.fixes would simply use survexblock.id as a key, + but at this point int he parsing we have not yet saved survexblock to the db so + survexblock.id is not available. """ # *fix|36|reference|36359.40|82216.08|2000.00\n # *fix|36|36359.40|82216.08|2000.00\n @@ -761,11 +846,15 @@ class LoadingSurvex: # \s+([\d\.]*) # Capture group 3: another number (digits and periods) # \s+([\d\.]*) # Capture group 4: yet another number (digits and periods) # \s*;? # Optional whitespace and optional semicolon - # (.*)$ # Capture group 5: remainder of the line (any characters), a comment + # (.*)$ # Capture group 5: remainder of the line (any characters), a comment + + + + fixid = blockid(survexblock) + rx_fixline = re.compile(r"(?i)^\s*[*]fix\s+([\w\d_\.\-]+)\s+(?:reference)?\s*([\d\.]*)\s+([\d\.]*)\s+([\d\.]*)\s*;?(.*)$") line = line.replace("\n","") - #fixline = self.rx_fixline.match(line) fixline = rx_fixline.match(line) if not fixline: display = line.replace(" ","|") @@ -777,8 +866,8 @@ class LoadingSurvex: #print(fixline.group(1), fixline.group(5)) #print(f"'{line}'") name = fixdata[0] - if (survexblock, name) in self.fixes: - message = f"! Duplicate *FIX: id '{line}' ({survexblock}) {survexblock.survexfile.path}" + if fixid in self.fixes: + message = f"! Duplicate *FIX: id '{line}' '{fixid}' ({survexblock}) {survexblock.survexfile.path} " print(self.insp + message) stash_data_issue(parser="survex", message=message) @@ -788,7 +877,7 @@ class LoadingSurvex: try: #_, _, alt, *rest = (fixdata + [None]*5)[:5] name, _, _, alt, comment = (list(fixdata) + [None]*5)[:5] - fixid = str(survexblock.id)+ ":"+ name + self.fixes[fixid] = (survexblock, name, alt, comment) message = f"{name}, {fixdata=}, last:{fixline.groups()[-1]}" except Exception as e: @@ -928,7 +1017,7 @@ class LoadingSurvex: print(self.insp + message) stash_data_issue(parser='survex', message=message, url=None, sb=(survexblock.survexfile.path)) - if teamnames := get_team_pending(survexblock): + if teamnames := self.get_team_pending(survexblock._blockid): for tm in teamnames: if known_foreigner(tm): message = f"- *team {expo.year} '{tm}' known foreigner *date (misordered) {survexblock.survexfile.path} ({survexblock}) in '{line}'" @@ -1030,7 +1119,7 @@ class LoadingSurvex: if survexblock.date: # do not actually need a distict variable 'currentdate' but it makes the code clearer self.currentdate = survexblock.date - survexblock.save() + # survexblock.save() def LoadSurvexLeg(self, survexblock, sline, comment, svxline): """This reads compass, clino and tape data but only keeps the tape lengths, @@ -1241,7 +1330,7 @@ class LoadingSurvex: Currently this just sets a flag that the survex block is not CUCC """ survexblock.foreigners = True - survexblock.save(update_fields=["foreigners"]) + # survexblock.save(update_fields=["foreigners"]) def LoadSurvexRef(self, survexblock, args): """Interpret the *ref record, and all the many variants @@ -1272,7 +1361,7 @@ class LoadingSurvex: if reftxt: # only store it if not an empty string survexblock.ref_text = reftxt[:399] # truncate or MariaDB crashes on databaseReset ! - survexblock.save() + # survexblock.save() return if len(args) < 4: @@ -1339,7 +1428,7 @@ class LoadingSurvex: if manywallets[0]: survexblock.scanswallet = manywallets[0] # this is a ForeignKey field # Only save if changed - survexblock.save(update_fields=["scanswallet"]) + # survexblock.save(update_fields=["scanswallet"]) # This is where we should check that the wallet JSON contains a link to the survexfile # and that the JSON date and walletdate are set correctly to the survexblock date. set_walletdate(survexblock.scanswallet) @@ -1715,8 +1804,10 @@ class LoadingSurvex: print(f">> why is survexblock not set ?! in LoadSurvexQM()/n {survexblock.survexfile.path}") expoyear = settings.EPOCH.year # 1970 - - + ### HORRIBLE HACK, replace with cache + hack_save(survexblock) + ### HORRIBLE HACK, replace with cache + try: qm = QM.objects.create( number=qm_no, @@ -1827,7 +1918,7 @@ class LoadingSurvex: Loads the begin/end blocks using a stack for labels. Uses the python generator idiom to avoid loading the whole file (21MB) into memory. """ - blkid = None + blk_name = None pathlist = None args = None oldflags = None @@ -1837,7 +1928,7 @@ class LoadingSurvex: nlegstotal = 0 self.relativefilename = path - self._pending_block_saves = set() # Cache for survex blocks to save at the end + self._pending_block_saves = {} # Cache for survex blocks to save at the end #self.IdentifyCave(path, svxid, depth) # this will produce null for survex files which are geographic collections self.currentsurvexfile = survexblock.survexfile @@ -1862,13 +1953,13 @@ class LoadingSurvex: sys.stderr.flush() def printbegin(): - nonlocal blkid + nonlocal blk_name nonlocal pathlist depth = " " * self.depthbegin self.insp = depth if debugprint: - print(f"{self.depthbegin:2}{depth} - Begin for :'{blkid}'") + print(f"{self.depthbegin:2}{depth} - Begin for :'{blk_name}'") pathlist = "" for id in self.stackbegin: if len(id) > 0: @@ -1887,9 +1978,9 @@ class LoadingSurvex: ) def pushblock(): - nonlocal blkid + nonlocal blk_name if debugprint: - print(f" # datastack at 1 *begin {blkid} 'type':", end="") + print(f" # datastack at 1 *begin {blk_name} 'type':", end="") for dict in self.datastack: print(f"'{dict['type'].upper()}' ", end="") print("") @@ -1898,7 +1989,7 @@ class LoadingSurvex: self.datastack.append(copy.deepcopy(self.datastar)) # ------------ * DATA if debugprint: - print(f" # datastack at 2 *begin {blkid} 'type':", end="") + print(f" # datastack at 2 *begin {blk_name} 'type':", end="") for dict in self.datastack: print(f"'{dict['type'].upper()}' ", end="") print("") @@ -1910,10 +2001,10 @@ class LoadingSurvex: pass def popblock(): - nonlocal blkid + nonlocal blk_name nonlocal oldflags if debugprint: - print(f" # datastack at *end '{blkid} 'type':", end="") + print(f" # datastack at *end '{blk_name} 'type':", end="") for dict in self.datastack: print(f"'{dict['type'].upper()}' ", end="") print("") @@ -1922,7 +2013,7 @@ class LoadingSurvex: self.datastar = copy.deepcopy(self.datastack.pop()) # ------------ * DATA if debugprint: - print(f" # datastack after *end '{blkid} 'type':", end="") + print(f" # datastack after *end '{blk_name} 'type':", end="") for dict in self.datastack: print(f"'{dict['type'].upper()}' ", end="") print("") @@ -1940,7 +2031,7 @@ class LoadingSurvex: # ...existing code... """Interprets a survex comamnd where * is the first character on the line, e.g. *begin""" nonlocal survexblock - nonlocal blkid + nonlocal blk_name nonlocal pathlist nonlocal args nonlocal oldflags @@ -1953,10 +2044,10 @@ class LoadingSurvex: # ------------------------BEGIN if self.rx_begin.match(cmd): t_block = time.perf_counter() - blkid = args.lower() + blk_name = args.lower() # PUSH state ++++++++++++++ self.depthbegin += 1 - self.stackbegin.append(blkid) + self.stackbegin.append(blk_name) self.unitsstack.append((self.units, self.unitsfactor)) self.legsnumberstack.append(self.legsnumber) self.slengthstack.append(self.slength) @@ -1974,8 +2065,11 @@ class LoadingSurvex: self.inheritdate = self.currentdate self.currentdate = None # zero the current date when we start a new block printbegin() + + # creating the SurvexBlock automatically creates a UUID in ._blockid + # Note that this does not create it in the database newsurvexblock = SurvexBlock( - name=blkid, + name=blk_name, parent=survexblock, survexfile=self.currentsurvexfile, legsall=0, @@ -1986,7 +2080,7 @@ class LoadingSurvex: "(" + survexblock.title + ")" ) # copy parent inititally, overwrite if it has its own survexblock = newsurvexblock - survexblock.save() # Only save once, after all fields are set + survexblock.save() # Only save once, after all fields are set, or try to delay until *end using caches tickle() # ---------------------------END @@ -1999,20 +2093,10 @@ class LoadingSurvex: nlegstotal += self.legsnumber self.fix_undated(survexblock) - self.fix_anonymous(survexblock) - # This is the most time-consuming step within *end processing: was 47% - # Instead of saving parent here, cache for later - if hasattr(survexblock, 'parent') and survexblock.parent: - self._pending_block_saves.add(survexblock) - try: - # This is the second most time-consuming step within *end processing: was 35% - self._pending_block_saves.add(survexblock) - # update_fields=["legsall", "legslength"] - except Exception: - print(f"{survexblock=}", file=sys.stderr) - raise - confirm_team_on_trip(survexblock) - # POP state ++++++++++++++ + self.fix_anonymous(survexblock) + self.confirm_team_on_trip(survexblock) + self.cache_survexblock(survexblock) + # POP state ++++++++++++++ popblock() self.inheritteam = self.teaminheritstack.pop() self.currentteam = self.teamcurrentstack.pop() @@ -2021,7 +2105,7 @@ class LoadingSurvex: self.legsnumber = self.legsnumberstack.pop() self.units, self.unitsfactor = self.unitsstack.pop() self.slength = self.slengthstack.pop() - blkid = self.stackbegin.pop() + blk_name = self.stackbegin.pop() self.currentsurvexblock = survexblock.parent survexblock = survexblock.parent oldflags = self.flagsstar @@ -2099,7 +2183,7 @@ class LoadingSurvex: if mfail: message = f"\n ! - ERROR version control merge failure\n - '{sline}'\n" message = ( - message + f" - line {self.lineno} in {blkid} in {survexblock}\n - NERD++ needed to fix it" + message + f" - line {self.lineno} in {blk_name} in {survexblock}\n - NERD++ needed to fix it" ) print(message) print(message, file=sys.stderr) @@ -2122,7 +2206,10 @@ class LoadingSurvex: # At the end, save all cached survexblocks using bulk_update - blocks = list(getattr(self, '_pending_block_saves', set())) + blocks = [] + for blockid in self._pending_block_saves: + blocks.append(self._pending_block_saves[blockid]) + # blocks = list(getattr(self, '_pending_block_saves', set())) if blocks: # valid_blocks = [] # for block in blocks: @@ -2522,6 +2609,9 @@ def FindAndLoadSurvex(): flinear.write(f"{svx_scan.depthinclude:2} {indent} *edulcni {survexfileroot.path}\n") io_collate.write(f";*edulcni {survexfileroot.path}\n") + + svx_scan.check_cache_clean() + mem1 = get_process_memory() flinear.write(f"\n - MEM:{mem1:.2f} MB STOP {survexfileroot.path}\n") flinear.write(f" - MEM:{mem1 - mem0:.3f} MB ADDITIONALLY USED\n") @@ -2637,6 +2727,8 @@ def FindAndLoadSurvex(): flinear.write(f"{omit_scan.depthinclude:2} {indent} *edulcni {unseensroot}\n") io_collate.write(f";*edulcni {UNSEENS}\n") + omit_scan.check_cache_clean() + mem1 = get_process_memory() flinear.write(f"\n - MEM:{mem1:.2f} MB STOP {UNSEENS} Unseen Oddments\n") flinear.write(f" - MEM:{mem1 - mem0:.3f} MB ADDITIONALLY USED Unseen Oddments\n") @@ -3139,13 +3231,7 @@ def LoadSurvexBlocks(): memend = get_process_memory() print(f" - MEMORY start:{memstart:.3f} MB end:{memend:.3f} MB increase={memend - memstart:.3f} MB") - global person_pending_cache - for sb in person_pending_cache: - if len(person_pending_cache[sb]) > 0: - print(f" ") - message = f" ! PENDING team list not emptied {sb.survexfile.path} {len(person_pending_cache[sb])} people: {person_pending_cache[sb]}" - stash_data_issue(parser="survex", message=message, url=None, sb=(sb.survexfile.path)) - print(message) + # duration = time.time() - start # print(f" - TIME: {duration:7.2f} s", file=sys.stderr) store_data_issues()