From c7ddd799784ed398d5ba799d4e9c18040afc7fef Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@gmail.com>
Date: Fri, 30 Jan 2026 20:31:07 +0000
Subject: [PATCH] sequential fallback on exceptions in bulk update

---
 parsers/survex.py | 91 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 66 insertions(+), 25 deletions(-)

diff --git a/parsers/survex.py b/parsers/survex.py
index de4e72f..b1f0377 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -679,33 +679,57 @@ class LoadingSurvex:
     def save_personroles_to_db(self):
         """This should be run only after all the survexblocks have 
         been saved to the database and so have _id that can be used as a ForeignKey
+        
+        survexblock = models.ForeignKey("SurvexBlock", on_delete=models.CASCADE, db_index=True)
+        # increasing levels of precision, Surely we only need survexblock and (either person or personexpedition)?
+        personname = models.CharField(max_length=100)
+        person = models.ForeignKey("Person", blank=True, null=True, on_delete=models.CASCADE, db_index=True) # not needed
+        personexpedition = models.ForeignKey("PersonExpedition", blank=True, null=True, on_delete=models.SET_NULL, db_index=True)
+    
         """
         print(f" - Saving {len(self._pending_pr_saves):,} SurvexPersonRoles to db", file=sys.stderr)
+        pr_list = []
         for blk in self._pending_pr_saves:
-                           
-            # Now commit to db
-            pr_list = self._pending_pr_saves[blk]
-            # print(f" PR_LIST {pr_list} {blk}", file=sys.stderr)
-            valid_list = []
-            for pr in pr_list:
-                try:
-                    pr.full_clean()
-                    valid_list.append(pr)
-                except ValidationError as e:
-                    message = f"  ! PR is invalid: {e}  {pr}"
-                    print(message, file=sys.stderr)
-                    stash_data_issue(
-                        parser="survex",
-                        message=message
-                    )  
-            try:            
-                SurvexPersonRole.objects.bulk_create(valid_list)
-            except Exception as e:
-                message = f"\n ! - EXCEPTION '{e}' - in buk update. Falling back on sequential updates"
-                print(message)
-                print(message, file=sys.stderr)
-                stash_data_issue(parser="survex", message=message)
+            pr_list + self._pending_pr_saves[blk]
             
+        # Now commit to db
+        # print(f" PR_LIST {pr_list} {blk}", file=sys.stderr)
+        valid_list = []
+        for pr in pr_list:
+            try:
+                pr.full_clean()
+                valid_list.append(pr)
+            except ValidationError as e:
+                message = f"  ! PR is invalid: {e}  {pr}"
+                print(message, file=sys.stderr)
+                stash_data_issue(
+                    parser="survex",
+                    message=message
+                )  
+        try:            
+            SurvexPersonRole.objects.bulk_create(valid_list)
+        except Exception as e:
+            message = f"\n ! - EXCEPTION '{e}' - in PR bulk update. Falling back onto sequential updates"
+            print(message)
+            print(message, file=sys.stderr)
+            stash_data_issue(parser="survex", message=message)
+            
+            for pr in valid_list:
+                got_obj, created = SurvexPersonRole.objects.get_or_create(
+                    survexblock=pr.survexblock,
+                    personname=pr.personname,
+                    person=pr.person,
+                    personexpedition=pr.personexpedition,
+                    defaults={'survexblockblock': pr.survexblock} # Fields to set only if creating
+                )
+                if created:
+                    print(f"  - {qm} Created", file=sys.stderr)
+                else:
+                    # update the block if it changed
+                    got_obj.block = qm.block
+                    got_obj.save()
+                    # print(f"  - {qm} SAVED", file=sys.stderr)
+        print(f"  - PRs saved to db", file=sys.stderr)
                
         _pending_pr_saves = {} # in database now, so empty cache    
 
@@ -744,7 +768,6 @@ class LoadingSurvex:
                 try:
                     for qm in self._pending_qm_saves[blk]:
                         qms.append(qm)
-                        #qm.save()
                 except Exception as e:
                     message = f"\n ! - EXCEPTION '{e}' - in buk update. Falling back on sequential updates"
                     print(message)
@@ -763,10 +786,28 @@ class LoadingSurvex:
             try:            
                  QM.objects.bulk_create(qms, **bulk_kwargs)                    
             except Exception as e:
-                message = f"\n ! - EXCEPTION '{e}' - in buk update. Falling back on sequential updates"
+                message = f"\n ! - EXCEPTION '{e}' - in QM bulk update. Falling back onto sequential updates"
                 print(message)
                 print(message, file=sys.stderr)
                 stash_data_issue(parser="survex", message=message)
+                
+                for qm in qms:
+                    got_obj, created = QM.objects.get_or_create(
+                        cave_id=qm.cave_id,
+                        blockname=qm.blockname,
+                        grade=qm.grade,
+                        number=qm.number,
+                        expoyear=qm.expoyear,
+                        defaults={'block': qm.block} # Fields to set only if creating
+                    )
+                    if created:
+                        print(f"  - {qm} Created", file=sys.stderr)
+                    else:
+                        # update the block if it changed
+                        got_obj.block = qm.block
+                        got_obj.save()
+                        # print(f"  - {qm} SAVED", file=sys.stderr)
+        print(f"  - QMs saved to db", file=sys.stderr)
     
     def add_to_pending(self, survexblock, tm):
         """Collects team names. We might not have a date so cannot validate