expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2026-02-08 10:19:53 +00:00
Code Issues 18 Projects 3 Activity

cached parent blocks: saved to db only at the end

Browse Source
This commit is contained in:
Philip Sargent
2026-01-28 18:54:15 +00:00
parent 8f576c0a94
commit 0d6d1fd4b6
1 changed files with 20 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
parsers/survex.py
View File

@@ -1820,6 +1820,9 @@ class LoadingSurvex:
slengthtotal = 0.0 slengthtotal = 0.0
nlegstotal = 0 nlegstotal = 0
self.relativefilename = path self.relativefilename = path
# Cache for parent blocks to save at the end
self._pending_parent_saves = set()
#self.IdentifyCave(path, svxid, depth) # this will produce null for survex files which are geographic collections #self.IdentifyCave(path, svxid, depth) # this will produce null for survex files which are geographic collections
self.currentsurvexfile = survexblock.survexfile self.currentsurvexfile = survexblock.survexfile
@@ -1828,6 +1831,8 @@ class LoadingSurvex:
self.datastar = copy.deepcopy(self.datastardefault) self.datastar = copy.deepcopy(self.datastardefault)
self.flagsstar = copy.deepcopy(self.flagsdefault) self.flagsstar = copy.deepcopy(self.flagsdefault)
def tickle(): def tickle():
nonlocal blockcount nonlocal blockcount
@@ -1982,12 +1987,9 @@ class LoadingSurvex:
self.fix_undated(survexblock) self.fix_undated(survexblock)
self.fix_anonymous(survexblock) self.fix_anonymous(survexblock)
# This is the most time-consuming step within *end processing: 47% # This is the most time-consuming step within *end processing: 47%
try: # Instead of saving parent here, cache for later
if hasattr(survexblock, 'parent') and survexblock.parent: if hasattr(survexblock, 'parent') and survexblock.parent:
survexblock.parent.save(update_fields=None) # Only if parent fields changed self._pending_parent_saves.add(survexblock.parent)
except Exception:
print(f"{survexblock.parent=}", file=sys.stderr)
raise
try: try:
# This is the second most time-consuming step within *end processing: 35% # This is the second most time-consuming step within *end processing: 35%
survexblock.save(update_fields=["legsall", "legslength"]) # Only update changed fields survexblock.save(update_fields=["legsall", "legslength"]) # Only update changed fields
@@ -2065,6 +2067,7 @@ class LoadingSurvex:
# ...timing removed... # ...timing removed...
# this is a python generator idiom. # this is a python generator idiom.
# see https://realpython.com/introduction-to-python-generators/ # see https://realpython.com/introduction-to-python-generators/
# this is the first use of generators in troggle (Oct.2022) and saves 21 MB of memory # this is the first use of generators in troggle (Oct.2022) and saves 21 MB of memory
@@ -2103,6 +2106,13 @@ class LoadingSurvex:
self.legsnumber = nlegstotal self.legsnumber = nlegstotal
self.slength = slengthtotal self.slength = slengthtotal
# At the end, save all cached parent blocks
for parent in getattr(self, '_pending_parent_saves', set()):
try:
parent.save(update_fields=None)
except Exception as e:
print(f"Error saving parent block {parent}: {e}", file=sys.stderr)
# ...timing removed... # ...timing removed...
def PushdownStackScan(self, survexblock, path, finname, flinear, io_collate): def PushdownStackScan(self, survexblock, path, finname, flinear, io_collate):
@@ -2993,10 +3003,10 @@ def survexifywallets():
w = wallets[wid] w = wallets[wid]
w.persons.add(*people) w.persons.add(*people)
total_added += len(people) total_added += len(people)
if total_added % 1000 == 0 and total_added > 0: # if total_added % 1000 == 0 and total_added > 0:
print(f" - Batched {total_added} people to wallets", file=sys.stderr) # print(f" - Batched {total_added} people to wallets", file=sys.stderr)
duration = time.time() - start duration = time.time() - start
print(f" - {duration:7.2f} s to batch add people to wallets (total {total_added})", file=sys.stderr) print(f" - {duration:7.2f} s to batch-add {total_added} people to wallets", file=sys.stderr)
start = time.time() start = time.time()
@@ -3024,7 +3034,7 @@ def survexifywallets():
# if total_wallets % 100 == 0: # if total_wallets % 100 == 0:
# print(f" - Batched {total_wallets} wallets for caves", file=sys.stderr) # print(f" - Batched {total_wallets} wallets for caves", file=sys.stderr)
duration = time.time() - start duration = time.time() - start
print(f" - {duration:7.2f} s to batch add caves to wallets (total {total_wallets})", file=sys.stderr) print(f" - {duration:7.2f} s to batch-add caves to {total_wallets} wallets", file=sys.stderr)
start = time.time() start = time.time()