diff --git a/parsers/survex.py b/parsers/survex.py index c25f22a..49227e5 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -680,7 +680,6 @@ class LoadingSurvex: ) if created: nc += 1 - print(f" - {pr} Created", file=sys.stderr) else: ns += 1 # update the block if it changed @@ -728,7 +727,7 @@ class LoadingSurvex: bulk_kwargs try: - QM.objects.bulk_create(qms, **bulk_kwargs) + QM.objects.bulk_create(qms_valid_list, **bulk_kwargs) except Exception as e: message = f"\n ! - EXCEPTION '{e}' - in QM bulk update. Falling back onto sequential updates" print(message) @@ -740,7 +739,7 @@ class LoadingSurvex: def qm_save_mysql(): nc = 0 ns = 0 - for qm in qms: + for qm in qms_valid_list: got_obj, created = QM.objects.get_or_create( cave_id=qm.cave_id, blockname=qm.blockname, @@ -751,7 +750,6 @@ class LoadingSurvex: ) if created: nc += 1 - print(f" - {qm} Created", file=sys.stderr) else: ns += 1 got_obj.block = qm.block @@ -760,41 +758,20 @@ class LoadingSurvex: qms = [] for blk in self._pending_qm_saves: - qm_list = self._pending_qm_saves[blk] - qms = qms + qm_list + qms = qms + self._pending_qm_saves[blk] print(f" - Saving {len(qms)} QMs to db", file=sys.stderr) - for blk in self._pending_qm_saves: - - # Now commit to db - qm_list = self._pending_qm_saves[blk] - # print(f" PR_LIST {pr_list} {blk}", file=sys.stderr) - valid_list = [] - for qm in qm_list: - # nqms = QM.objects.filter( - # cave=qm.cave, blockname=qm.blockname, grade=qm.grade, - # number=qm.number, expoyear=qm.expoyear) - # print(f"QM found a prior match {nqms}", file=sys.stderr) - try: - qm.full_clean() - valid_list.append(qm) - except ValidationError as e: - message = f" ! QM is invalid: {e} {qm}" - print(message, file=sys.stderr) - stash_data_issue( - parser="survex", - message=message - ) - qms = [] - for blk in self._pending_qm_saves: - try: - for qm in self._pending_qm_saves[blk]: - qms.append(qm) - except Exception as e: - message = f"\n ! - EXCEPTION '{e}' - in buk update. Falling back on sequential updates" - print(message) - print(message, file=sys.stderr) - stash_data_issue(parser="survex", message=message) - + qms_valid_list = [] + for qm in qms: + try: + qm.full_clean() + qms_valid_list.append(qm) + except ValidationError as e: + message = f" ! QM is invalid: '{e}' '{qm}'" + print(message, file=sys.stderr) + stash_data_issue( + parser="survex", + message=message + ) if connection.vendor == 'mysql': qm_save_mysql()