re-systematized

parsers/survex.py

@@ -596,7 +596,6 @@ class LoadingSurvex:
                 
                 sb_save_mysql()
                 
-            
         def sb_save_mysql():
             nc = 0
             ns = 0
@@ -645,7 +644,7 @@ class LoadingSurvex:
 
     def save_personroles_to_db(self):
         """This should be run only after all the survexblocks have 
-        been saved to the database and so have _id that can be used as a ForeignKey
+        been saved to the database and so have _blockid that can be used as a ForeignKey
         Fields:
             survexblock = models.ForeignKey("SurvexBlock", ..
             personname = models.CharField(max_length=100)
@@ -655,8 +654,11 @@ class LoadingSurvex:
         """
         def pr_save_sqlite():
             try:            
-                SurvexPersonRole.objects.bulk_create(valid_list)
+                SurvexPersonRole.objects.bulk_create(valid_list,
-                print(f"  - all SurvexPersonRoles bulk-created to db", file=sys.stderr)
+                    update_fields = ['survexblock', 'personname', 'person', 'personexpedition'],
+                    unique_fields = ['survexblock', 'personname', 'person', 'personexpedition']
+                    )
+                print(f"  - all {len(valid_list)} SurvexPersonRoles bulk-created to db", file=sys.stderr)
             except Exception as e:
                 message = f"\n ! - EXCEPTION '{e}' - in PR bulk update. Falling back onto sequential updates method"
                 print(message)
@@ -686,13 +688,12 @@ class LoadingSurvex:
                     got_obj.save()
             print(f"  - {ns}/{nc} SurvexPersonRoles saved/created to db", file=sys.stderr)
 
-        print(f" - Saving {len(self._pending_pr_saves):,} SurvexPersonRoles to db.. ({connection.vendor})", file=sys.stderr)
         pr_list = []
         for blk in self._pending_pr_saves:
-            pr_list + self._pending_pr_saves[blk]
+            pr_list += self._pending_pr_saves[blk]
+        print(f" - Saving {len(pr_list):,} SurvexPersonRoles to db.. ({connection.vendor})", file=sys.stderr)
             
         # Now commit to db
-        # print(f" PR_LIST {pr_list} {blk}", file=sys.stderr)
         valid_list = []
         for pr in pr_list:
             try:
@@ -705,6 +706,7 @@ class LoadingSurvex:
                     parser="survex",
                     message=message
                 )  
+        print(f"  - {len(valid_list)} SurvexPersonRoles in list", file=sys.stderr)
         if connection.vendor == 'mysql': 
             pr_save_mysql()
         else:
@@ -714,8 +716,48 @@ class LoadingSurvex:
 
     def save_qms_to_db(self):
         """This should be run only after all the survexblocks have 
-        been saved to the database and so have _id that can be used as a ForeignKey
+        been saved to the database and so have _blockid that can be used as a ForeignKey
         """
+        def qm_save_sqlite():
+            # MariaDB/MySQL do not support (and don't need) unique_fields by sqlite neeeds them
+            bulk_kwargs = {
+                "update_conflicts": True,
+                "update_fields": ['block', 'cave', 'blockname', 'grade', 'number', 'expoyear'],
+                "unique_fields": ['cave', 'blockname', 'grade', 'number', 'expoyear'],
+            }
+            
+            bulk_kwargs
+            try:            
+                 QM.objects.bulk_create(qms, **bulk_kwargs)                    
+            except Exception as e:
+                message = f"\n ! - EXCEPTION '{e}' - in QM bulk update. Falling back onto sequential updates"
+                print(message)
+                print(message, file=sys.stderr)
+                stash_data_issue(parser="survex", message=message)
+                
+                qm_save_mysql()
+                           
+        def qm_save_mysql():
+            nc = 0
+            ns = 0
+            for qm in qms:
+                got_obj, created = QM.objects.get_or_create(
+                    cave_id=qm.cave_id,
+                    blockname=qm.blockname,
+                    grade=qm.grade,
+                    number=qm.number,
+                    expoyear=qm.expoyear,
+                    defaults={'block': qm.block} # Fields to set only if creating
+                )
+                if created:
+                    nc += 1
+                    print(f"  - {qm} Created", file=sys.stderr)
+                else:
+                    ns += 1
+                    got_obj.block = qm.block
+                    got_obj.save()
+            print(f"  - {ns}/{nc} QMs saved/created to db", file=sys.stderr)        
+ 
         qms = []
         for blk in self._pending_qm_saves:
             qm_list = self._pending_qm_saves[blk]
@@ -753,39 +795,14 @@ class LoadingSurvex:
                     print(message, file=sys.stderr)
                     stash_data_issue(parser="survex", message=message)
                     
-            bulk_kwargs = {
+ 
-                "update_conflicts": True,
+            if connection.vendor == 'mysql': 
-                "update_fields": ['block', 'cave', 'blockname', 'grade', 'number', 'expoyear'],
+                qm_save_mysql()
-            }
+            else:
+                qm_save_sqlite()
+            
 
-            # MariaDB/MySQL do not support (and don't need) unique_fields here
+  
-            if connection.vendor != 'mysql': 
-                bulk_kwargs["unique_fields"] = ['cave', 'blockname', 'grade', 'number', 'expoyear']
-
-            try:            
-                 QM.objects.bulk_create(qms, **bulk_kwargs)                    
-            except Exception as e:
-                message = f"\n ! - EXCEPTION '{e}' - in QM bulk update. Falling back onto sequential updates"
-                print(message)
-                print(message, file=sys.stderr)
-                stash_data_issue(parser="survex", message=message)
-                
-                for qm in qms:
-                    got_obj, created = QM.objects.get_or_create(
-                        cave_id=qm.cave_id,
-                        blockname=qm.blockname,
-                        grade=qm.grade,
-                        number=qm.number,
-                        expoyear=qm.expoyear,
-                        defaults={'block': qm.block} # Fields to set only if creating
-                    )
-                    if created:
-                        print(f"  - {qm} Created", file=sys.stderr)
-                    else:
-                        # update the block if it changed
-                        got_obj.block = qm.block
-                        got_obj.save()
-                        # print(f"  - {qm} SAVED", file=sys.stderr)
         print(f"  - QMs saved to db", file=sys.stderr)
                 
     def put_personrole_on_trip(self, survexblock, personexpedition, tm):
@@ -2437,12 +2454,16 @@ class LoadingSurvex:
         # Getting round MariaDB foibles: put these in different transactions
         with transaction.atomic():
             self.save_survexblocks_to_db()
+        n = SurvexBlock.objects.all().count()
+        print(f"  + Now {n}  SurvexBlocks in total", file=sys.stderr)
         with transaction.atomic():            
             self.save_personroles_to_db()
+        n = SurvexPersonRole.objects.all().count()
+        print(f"  + Now {n}  SurvexPersonRoles in total", file=sys.stderr)
         with transaction.atomic():            
             self.save_qms_to_db()
-        qms_n = QM.objects.all().count()
+        n = QM.objects.all().count()
-        print(f"  - Now {qms_n}  QMs in total", file=sys.stderr)
+        print(f"  + Now {n}  QMs in total", file=sys.stderr)
 
     def PushdownStackScan(self, survexblock, path, finname, flinear, io_collate):
         """Follows the *include links in all the survex files from the root file  (usually 1623.svx)