cached parent blocks: saved to db only at the end

parsers/survex.py

@@ -1820,6 +1820,9 @@ class LoadingSurvex:
         slengthtotal = 0.0
         nlegstotal = 0
         self.relativefilename = path
+
+        # Cache for parent blocks to save at the end
+        self._pending_parent_saves = set()
         #self.IdentifyCave(path, svxid, depth)  # this will produce null for survex files which are geographic collections
 
         self.currentsurvexfile = survexblock.survexfile
@@ -1828,6 +1831,8 @@ class LoadingSurvex:
         self.datastar = copy.deepcopy(self.datastardefault)
         self.flagsstar = copy.deepcopy(self.flagsdefault)
 
+            
+
         def tickle():
             nonlocal blockcount
 
@@ -1982,12 +1987,9 @@ class LoadingSurvex:
                 self.fix_undated(survexblock)
                 self.fix_anonymous(survexblock)
                 # This is the most time-consuming step within *end processing: 47%
-                try:
+                # Instead of saving parent here, cache for later
                 if hasattr(survexblock, 'parent') and survexblock.parent:
-                        survexblock.parent.save(update_fields=None)  # Only if parent fields changed
-                except Exception:
-                    print(f"{survexblock.parent=}", file=sys.stderr)
-                    raise
+                    self._pending_parent_saves.add(survexblock.parent)
                 try:
                     # This is the second most time-consuming step within *end processing: 35%
                     survexblock.save(update_fields=["legsall", "legslength"])  # Only update changed fields
@@ -2065,6 +2067,7 @@ class LoadingSurvex:
 
         # ...timing removed...
 
+
         # this is a python generator idiom.
         # see https://realpython.com/introduction-to-python-generators/
         # this is the first use of generators in troggle (Oct.2022) and saves 21 MB of memory
@@ -2103,6 +2106,13 @@ class LoadingSurvex:
             self.legsnumber = nlegstotal
             self.slength = slengthtotal
 
+        # At the end, save all cached parent blocks
+        for parent in getattr(self, '_pending_parent_saves', set()):
+            try:
+                parent.save(update_fields=None)
+            except Exception as e:
+                print(f"Error saving parent block {parent}: {e}", file=sys.stderr)
+
         # ...timing removed...
 
     def PushdownStackScan(self, survexblock, path, finname, flinear, io_collate):
@@ -2993,10 +3003,10 @@ def survexifywallets():
         w = wallets[wid]
         w.persons.add(*people)
         total_added += len(people)
-        if total_added % 1000 == 0 and total_added > 0:
-            print(f"   - Batched {total_added} people to wallets", file=sys.stderr)
+        # if total_added % 1000 == 0 and total_added > 0:
+        #     print(f"   - Batched {total_added} people to wallets", file=sys.stderr)
     duration = time.time() - start
-    print(f" -  {duration:7.2f} s to batch add people to wallets (total {total_added})", file=sys.stderr)
+    print(f" -  {duration:7.2f} s to batch-add {total_added} people to wallets", file=sys.stderr)
     start = time.time()
                 
  
@@ -3024,7 +3034,7 @@ def survexifywallets():
         # if total_wallets % 100 == 0:
         #     print(f"   - Batched {total_wallets} wallets for caves", file=sys.stderr)
     duration = time.time() - start
-    print(f" -  {duration:7.2f} s to batch add caves to wallets (total {total_wallets})", file=sys.stderr)
+    print(f" -  {duration:7.2f} s to batch-add caves to {total_wallets} wallets", file=sys.stderr)
     start = time.time()