diff --git a/parsers/survex.py b/parsers/survex.py index 9db13db..cd331d1 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1820,6 +1820,9 @@ class LoadingSurvex: slengthtotal = 0.0 nlegstotal = 0 self.relativefilename = path + + # Cache for parent blocks to save at the end + self._pending_parent_saves = set() #self.IdentifyCave(path, svxid, depth) # this will produce null for survex files which are geographic collections self.currentsurvexfile = survexblock.survexfile @@ -1828,6 +1831,8 @@ class LoadingSurvex: self.datastar = copy.deepcopy(self.datastardefault) self.flagsstar = copy.deepcopy(self.flagsdefault) + + def tickle(): nonlocal blockcount @@ -1982,12 +1987,9 @@ class LoadingSurvex: self.fix_undated(survexblock) self.fix_anonymous(survexblock) # This is the most time-consuming step within *end processing: 47% - try: - if hasattr(survexblock, 'parent') and survexblock.parent: - survexblock.parent.save(update_fields=None) # Only if parent fields changed - except Exception: - print(f"{survexblock.parent=}", file=sys.stderr) - raise + # Instead of saving parent here, cache for later + if hasattr(survexblock, 'parent') and survexblock.parent: + self._pending_parent_saves.add(survexblock.parent) try: # This is the second most time-consuming step within *end processing: 35% survexblock.save(update_fields=["legsall", "legslength"]) # Only update changed fields @@ -2065,6 +2067,7 @@ class LoadingSurvex: # ...timing removed... + # this is a python generator idiom. # see https://realpython.com/introduction-to-python-generators/ # this is the first use of generators in troggle (Oct.2022) and saves 21 MB of memory @@ -2103,6 +2106,13 @@ class LoadingSurvex: self.legsnumber = nlegstotal self.slength = slengthtotal + # At the end, save all cached parent blocks + for parent in getattr(self, '_pending_parent_saves', set()): + try: + parent.save(update_fields=None) + except Exception as e: + print(f"Error saving parent block {parent}: {e}", file=sys.stderr) + # ...timing removed... def PushdownStackScan(self, survexblock, path, finname, flinear, io_collate): @@ -2993,10 +3003,10 @@ def survexifywallets(): w = wallets[wid] w.persons.add(*people) total_added += len(people) - if total_added % 1000 == 0 and total_added > 0: - print(f" - Batched {total_added} people to wallets", file=sys.stderr) + # if total_added % 1000 == 0 and total_added > 0: + # print(f" - Batched {total_added} people to wallets", file=sys.stderr) duration = time.time() - start - print(f" - {duration:7.2f} s to batch add people to wallets (total {total_added})", file=sys.stderr) + print(f" - {duration:7.2f} s to batch-add {total_added} people to wallets", file=sys.stderr) start = time.time() @@ -3024,7 +3034,7 @@ def survexifywallets(): # if total_wallets % 100 == 0: # print(f" - Batched {total_wallets} wallets for caves", file=sys.stderr) duration = time.time() - start - print(f" - {duration:7.2f} s to batch add caves to wallets (total {total_wallets})", file=sys.stderr) + print(f" - {duration:7.2f} s to batch-add caves to {total_wallets} wallets", file=sys.stderr) start = time.time()