expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2026-02-08 10:38:25 +00:00
Code Issues 18 Projects 3 Activity

re-systematized

Browse Source
This commit is contained in:
Philip Sargent
2026-01-31 00:24:55 +00:00
parent bbaf52b8b7
commit c5c9b91374
1 changed files with 63 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
parsers/survex.py
View File

@@ -596,7 +596,6 @@ class LoadingSurvex:
sb_save_mysql()
def sb_save_mysql():
nc = 0
ns = 0
@@ -645,7 +644,7 @@ class LoadingSurvex:
def save_personroles_to_db(self):
"""This should be run only after all the survexblocks have
been saved to the database and so have _id that can be used as a ForeignKey
been saved to the database and so have _blockid that can be used as a ForeignKey
Fields:
survexblock = models.ForeignKey("SurvexBlock", ..
personname = models.CharField(max_length=100)
@@ -655,8 +654,11 @@ class LoadingSurvex:
"""
def pr_save_sqlite():
try:
SurvexPersonRole.objects.bulk_create(valid_list)
print(f" - all SurvexPersonRoles bulk-created to db", file=sys.stderr)
SurvexPersonRole.objects.bulk_create(valid_list,
update_fields = ['survexblock', 'personname', 'person', 'personexpedition'],
unique_fields = ['survexblock', 'personname', 'person', 'personexpedition']
)
print(f" - all {len(valid_list)} SurvexPersonRoles bulk-created to db", file=sys.stderr)
except Exception as e:
message = f"\n ! - EXCEPTION '{e}' - in PR bulk update. Falling back onto sequential updates method"
print(message)
@@ -686,13 +688,12 @@ class LoadingSurvex:
got_obj.save()
print(f" - {ns}/{nc} SurvexPersonRoles saved/created to db", file=sys.stderr)
print(f" - Saving {len(self._pending_pr_saves):,} SurvexPersonRoles to db.. ({connection.vendor})", file=sys.stderr)
pr_list = []
for blk in self._pending_pr_saves:
pr_list + self._pending_pr_saves[blk]
pr_list += self._pending_pr_saves[blk]
print(f" - Saving {len(pr_list):,} SurvexPersonRoles to db.. ({connection.vendor})", file=sys.stderr)
# Now commit to db
# print(f" PR_LIST {pr_list} {blk}", file=sys.stderr)
valid_list = []
for pr in pr_list:
try:
@@ -705,6 +706,7 @@ class LoadingSurvex:
parser="survex",
message=message
)
print(f" - {len(valid_list)} SurvexPersonRoles in list", file=sys.stderr)
if connection.vendor == 'mysql':
pr_save_mysql()
else:
@@ -714,8 +716,48 @@ class LoadingSurvex:
def save_qms_to_db(self):
"""This should be run only after all the survexblocks have
been saved to the database and so have _id that can be used as a ForeignKey
been saved to the database and so have _blockid that can be used as a ForeignKey
"""
def qm_save_sqlite():
# MariaDB/MySQL do not support (and don't need) unique_fields by sqlite neeeds them
bulk_kwargs = {
"update_conflicts": True,
"update_fields": ['block', 'cave', 'blockname', 'grade', 'number', 'expoyear'],
"unique_fields": ['cave', 'blockname', 'grade', 'number', 'expoyear'],
}
bulk_kwargs
try:
QM.objects.bulk_create(qms, **bulk_kwargs)
except Exception as e:
message = f"\n ! - EXCEPTION '{e}' - in QM bulk update. Falling back onto sequential updates"
print(message)
print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message)
qm_save_mysql()
def qm_save_mysql():
nc = 0
ns = 0
for qm in qms:
got_obj, created = QM.objects.get_or_create(
cave_id=qm.cave_id,
blockname=qm.blockname,
grade=qm.grade,
number=qm.number,
expoyear=qm.expoyear,
defaults={'block': qm.block} # Fields to set only if creating
)
if created:
nc += 1
print(f" - {qm} Created", file=sys.stderr)
else:
ns += 1
got_obj.block = qm.block
got_obj.save()
print(f" - {ns}/{nc} QMs saved/created to db", file=sys.stderr)
qms = []
for blk in self._pending_qm_saves:
qm_list = self._pending_qm_saves[blk]
@@ -753,39 +795,14 @@ class LoadingSurvex:
print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message)
bulk_kwargs = {
"update_conflicts": True,
"update_fields": ['block', 'cave', 'blockname', 'grade', 'number', 'expoyear'],
}
if connection.vendor == 'mysql':
qm_save_mysql()
else:
qm_save_sqlite()
# MariaDB/MySQL do not support (and don't need) unique_fields here
if connection.vendor != 'mysql':
bulk_kwargs["unique_fields"] = ['cave', 'blockname', 'grade', 'number', 'expoyear']
try:
QM.objects.bulk_create(qms, **bulk_kwargs)
except Exception as e:
message = f"\n ! - EXCEPTION '{e}' - in QM bulk update. Falling back onto sequential updates"
print(message)
print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message)
for qm in qms:
got_obj, created = QM.objects.get_or_create(
cave_id=qm.cave_id,
blockname=qm.blockname,
grade=qm.grade,
number=qm.number,
expoyear=qm.expoyear,
defaults={'block': qm.block} # Fields to set only if creating
)
if created:
print(f" - {qm} Created", file=sys.stderr)
else:
# update the block if it changed
got_obj.block = qm.block
got_obj.save()
# print(f" - {qm} SAVED", file=sys.stderr)
print(f" - QMs saved to db", file=sys.stderr)
def put_personrole_on_trip(self, survexblock, personexpedition, tm):
@@ -2437,12 +2454,16 @@ class LoadingSurvex:
# Getting round MariaDB foibles: put these in different transactions
with transaction.atomic():
self.save_survexblocks_to_db()
n = SurvexBlock.objects.all().count()
print(f" + Now {n} SurvexBlocks in total", file=sys.stderr)
with transaction.atomic():
self.save_personroles_to_db()
n = SurvexPersonRole.objects.all().count()
print(f" + Now {n} SurvexPersonRoles in total", file=sys.stderr)
with transaction.atomic():
self.save_qms_to_db()
qms_n = QM.objects.all().count()
print(f" - Now {qms_n} QMs in total", file=sys.stderr)
n = QM.objects.all().count()
print(f" + Now {n} QMs in total", file=sys.stderr)
def PushdownStackScan(self, survexblock, path, finname, flinear, io_collate):
"""Follows the *include links in all the survex files from the root file (usually 1623.svx)