From c5c9b91374fc3d9612e499e6f9d3c83940f928e9 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sat, 31 Jan 2026 00:24:55 +0000 Subject: [PATCH] re-systematized --- parsers/survex.py | 105 +++++++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 42 deletions(-) diff --git a/parsers/survex.py b/parsers/survex.py index bf8c5a2..c25f22a 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -596,7 +596,6 @@ class LoadingSurvex: sb_save_mysql() - def sb_save_mysql(): nc = 0 ns = 0 @@ -645,7 +644,7 @@ class LoadingSurvex: def save_personroles_to_db(self): """This should be run only after all the survexblocks have - been saved to the database and so have _id that can be used as a ForeignKey + been saved to the database and so have _blockid that can be used as a ForeignKey Fields: survexblock = models.ForeignKey("SurvexBlock", .. personname = models.CharField(max_length=100) @@ -655,8 +654,11 @@ class LoadingSurvex: """ def pr_save_sqlite(): try: - SurvexPersonRole.objects.bulk_create(valid_list) - print(f" - all SurvexPersonRoles bulk-created to db", file=sys.stderr) + SurvexPersonRole.objects.bulk_create(valid_list, + update_fields = ['survexblock', 'personname', 'person', 'personexpedition'], + unique_fields = ['survexblock', 'personname', 'person', 'personexpedition'] + ) + print(f" - all {len(valid_list)} SurvexPersonRoles bulk-created to db", file=sys.stderr) except Exception as e: message = f"\n ! - EXCEPTION '{e}' - in PR bulk update. Falling back onto sequential updates method" print(message) @@ -686,13 +688,12 @@ class LoadingSurvex: got_obj.save() print(f" - {ns}/{nc} SurvexPersonRoles saved/created to db", file=sys.stderr) - print(f" - Saving {len(self._pending_pr_saves):,} SurvexPersonRoles to db.. ({connection.vendor})", file=sys.stderr) pr_list = [] for blk in self._pending_pr_saves: - pr_list + self._pending_pr_saves[blk] + pr_list += self._pending_pr_saves[blk] + print(f" - Saving {len(pr_list):,} SurvexPersonRoles to db.. ({connection.vendor})", file=sys.stderr) # Now commit to db - # print(f" PR_LIST {pr_list} {blk}", file=sys.stderr) valid_list = [] for pr in pr_list: try: @@ -705,6 +706,7 @@ class LoadingSurvex: parser="survex", message=message ) + print(f" - {len(valid_list)} SurvexPersonRoles in list", file=sys.stderr) if connection.vendor == 'mysql': pr_save_mysql() else: @@ -714,8 +716,48 @@ class LoadingSurvex: def save_qms_to_db(self): """This should be run only after all the survexblocks have - been saved to the database and so have _id that can be used as a ForeignKey + been saved to the database and so have _blockid that can be used as a ForeignKey """ + def qm_save_sqlite(): + # MariaDB/MySQL do not support (and don't need) unique_fields by sqlite neeeds them + bulk_kwargs = { + "update_conflicts": True, + "update_fields": ['block', 'cave', 'blockname', 'grade', 'number', 'expoyear'], + "unique_fields": ['cave', 'blockname', 'grade', 'number', 'expoyear'], + } + + bulk_kwargs + try: + QM.objects.bulk_create(qms, **bulk_kwargs) + except Exception as e: + message = f"\n ! - EXCEPTION '{e}' - in QM bulk update. Falling back onto sequential updates" + print(message) + print(message, file=sys.stderr) + stash_data_issue(parser="survex", message=message) + + qm_save_mysql() + + def qm_save_mysql(): + nc = 0 + ns = 0 + for qm in qms: + got_obj, created = QM.objects.get_or_create( + cave_id=qm.cave_id, + blockname=qm.blockname, + grade=qm.grade, + number=qm.number, + expoyear=qm.expoyear, + defaults={'block': qm.block} # Fields to set only if creating + ) + if created: + nc += 1 + print(f" - {qm} Created", file=sys.stderr) + else: + ns += 1 + got_obj.block = qm.block + got_obj.save() + print(f" - {ns}/{nc} QMs saved/created to db", file=sys.stderr) + qms = [] for blk in self._pending_qm_saves: qm_list = self._pending_qm_saves[blk] @@ -753,39 +795,14 @@ class LoadingSurvex: print(message, file=sys.stderr) stash_data_issue(parser="survex", message=message) - bulk_kwargs = { - "update_conflicts": True, - "update_fields": ['block', 'cave', 'blockname', 'grade', 'number', 'expoyear'], - } + + if connection.vendor == 'mysql': + qm_save_mysql() + else: + qm_save_sqlite() + - # MariaDB/MySQL do not support (and don't need) unique_fields here - if connection.vendor != 'mysql': - bulk_kwargs["unique_fields"] = ['cave', 'blockname', 'grade', 'number', 'expoyear'] - - try: - QM.objects.bulk_create(qms, **bulk_kwargs) - except Exception as e: - message = f"\n ! - EXCEPTION '{e}' - in QM bulk update. Falling back onto sequential updates" - print(message) - print(message, file=sys.stderr) - stash_data_issue(parser="survex", message=message) - - for qm in qms: - got_obj, created = QM.objects.get_or_create( - cave_id=qm.cave_id, - blockname=qm.blockname, - grade=qm.grade, - number=qm.number, - expoyear=qm.expoyear, - defaults={'block': qm.block} # Fields to set only if creating - ) - if created: - print(f" - {qm} Created", file=sys.stderr) - else: - # update the block if it changed - got_obj.block = qm.block - got_obj.save() - # print(f" - {qm} SAVED", file=sys.stderr) + print(f" - QMs saved to db", file=sys.stderr) def put_personrole_on_trip(self, survexblock, personexpedition, tm): @@ -2437,12 +2454,16 @@ class LoadingSurvex: # Getting round MariaDB foibles: put these in different transactions with transaction.atomic(): self.save_survexblocks_to_db() + n = SurvexBlock.objects.all().count() + print(f" + Now {n} SurvexBlocks in total", file=sys.stderr) with transaction.atomic(): self.save_personroles_to_db() + n = SurvexPersonRole.objects.all().count() + print(f" + Now {n} SurvexPersonRoles in total", file=sys.stderr) with transaction.atomic(): self.save_qms_to_db() - qms_n = QM.objects.all().count() - print(f" - Now {qms_n} QMs in total", file=sys.stderr) + n = QM.objects.all().count() + print(f" + Now {n} QMs in total", file=sys.stderr) def PushdownStackScan(self, survexblock, path, finname, flinear, io_collate): """Follows the *include links in all the survex files from the root file (usually 1623.svx)