sequential fallback on exceptions in bulk update

parsers/survex.py

@@ -679,12 +679,20 @@ class LoadingSurvex:
     def save_personroles_to_db(self):
         """This should be run only after all the survexblocks have 
         been saved to the database and so have _id that can be used as a ForeignKey
+        
+        survexblock = models.ForeignKey("SurvexBlock", on_delete=models.CASCADE, db_index=True)
+        # increasing levels of precision, Surely we only need survexblock and (either person or personexpedition)?
+        personname = models.CharField(max_length=100)
+        person = models.ForeignKey("Person", blank=True, null=True, on_delete=models.CASCADE, db_index=True) # not needed
+        personexpedition = models.ForeignKey("PersonExpedition", blank=True, null=True, on_delete=models.SET_NULL, db_index=True)
+    
         """
         print(f" - Saving {len(self._pending_pr_saves):,} SurvexPersonRoles to db", file=sys.stderr)
+        pr_list = []
         for blk in self._pending_pr_saves:
+            pr_list + self._pending_pr_saves[blk]
             
         # Now commit to db
-            pr_list = self._pending_pr_saves[blk]
         # print(f" PR_LIST {pr_list} {blk}", file=sys.stderr)
         valid_list = []
         for pr in pr_list:
@@ -701,11 +709,27 @@ class LoadingSurvex:
         try:            
             SurvexPersonRole.objects.bulk_create(valid_list)
         except Exception as e:
-                message = f"\n ! - EXCEPTION '{e}' - in buk update. Falling back on sequential updates"
+            message = f"\n ! - EXCEPTION '{e}' - in PR bulk update. Falling back onto sequential updates"
             print(message)
             print(message, file=sys.stderr)
             stash_data_issue(parser="survex", message=message)
             
+            for pr in valid_list:
+                got_obj, created = SurvexPersonRole.objects.get_or_create(
+                    survexblock=pr.survexblock,
+                    personname=pr.personname,
+                    person=pr.person,
+                    personexpedition=pr.personexpedition,
+                    defaults={'survexblockblock': pr.survexblock} # Fields to set only if creating
+                )
+                if created:
+                    print(f"  - {qm} Created", file=sys.stderr)
+                else:
+                    # update the block if it changed
+                    got_obj.block = qm.block
+                    got_obj.save()
+                    # print(f"  - {qm} SAVED", file=sys.stderr)
+        print(f"  - PRs saved to db", file=sys.stderr)
                
         _pending_pr_saves = {} # in database now, so empty cache    
 
@@ -744,7 +768,6 @@ class LoadingSurvex:
                 try:
                     for qm in self._pending_qm_saves[blk]:
                         qms.append(qm)
-                        #qm.save()
                 except Exception as e:
                     message = f"\n ! - EXCEPTION '{e}' - in buk update. Falling back on sequential updates"
                     print(message)
@@ -763,11 +786,29 @@ class LoadingSurvex:
             try:            
                  QM.objects.bulk_create(qms, **bulk_kwargs)                    
             except Exception as e:
-                message = f"\n ! - EXCEPTION '{e}' - in buk update. Falling back on sequential updates"
+                message = f"\n ! - EXCEPTION '{e}' - in QM bulk update. Falling back onto sequential updates"
                 print(message)
                 print(message, file=sys.stderr)
                 stash_data_issue(parser="survex", message=message)
                 
+                for qm in qms:
+                    got_obj, created = QM.objects.get_or_create(
+                        cave_id=qm.cave_id,
+                        blockname=qm.blockname,
+                        grade=qm.grade,
+                        number=qm.number,
+                        expoyear=qm.expoyear,
+                        defaults={'block': qm.block} # Fields to set only if creating
+                    )
+                    if created:
+                        print(f"  - {qm} Created", file=sys.stderr)
+                    else:
+                        # update the block if it changed
+                        got_obj.block = qm.block
+                        got_obj.save()
+                        # print(f"  - {qm} SAVED", file=sys.stderr)
+        print(f"  - QMs saved to db", file=sys.stderr)
+    
     def add_to_pending(self, survexblock, tm):
         """Collects team names. We might not have a date so cannot validate
         against expo attendance yet