expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2026-02-08 10:19:53 +00:00
Code Issues 18 Projects 3 Activity

SurvexPersonTeam renamed form survexpersonrole

Browse Source
This commit is contained in:
Philip Sargent
2026-02-01 14:56:31 +00:00
parent c899c0749c
commit fd2f920745
11 changed files with 37 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
parsers/locations.py
View File

@@ -10,7 +10,7 @@ from pathlib import Path
import troggle.settings as settings
from troggle.core.models.caves import Cave, Entrance
from troggle.core.models.logbooks import QM
from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole, SurvexStation
from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonTeam, SurvexStation
from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.models.wallets import Wallet
from troggle.core.utils import chaosmonkey, get_process_memory

38
parsers/survex.py
View File

@@ -17,7 +17,7 @@ from django.db import transaction
import troggle.settings as settings
from troggle.core.models.caves import Cave, Entrance, GetCaveLookup
from troggle.core.models.logbooks import QM
from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole, SurvexStation, SurvexFix
from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonTeam, SurvexStation, SurvexFix
from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.models.wallets import Wallet
from troggle.core.utils import chaosmonkey, get_process_memory
@@ -84,7 +84,7 @@ dataissues = set()
# Caches for ORM minimization
survexblock_cache = None # {scanswallet_id: [SurvexBlock, ...]}
personrole_cache = None # {survexblock._blockid: [SurvexPersonRole, ...]}
personrole_cache = None # {survexblock._blockid: [SurvexPersonTeam, ...]}
wallet_cache = None # {walletname: [Wallet, ...]}
trip_people_cache = {} # indexed by survexblock._blockid, so never needs cleaning out
@@ -200,9 +200,9 @@ def get_team_on_trip(survexblock):
"""
global trip_people_cache, personrole_cache
if personrole_cache is None:
# Build cache: {survexblock._blockid: [SurvexPersonRole, ...]}
# Build cache: {survexblock._blockid: [SurvexPersonTeam, ...]}
personrole_cache = {}
for pr in SurvexPersonRole.objects.all().select_related("person", "personexpedition"): # WASTEFUL ! Optimise this
for pr in SurvexPersonTeam.objects.all().select_related("person", "personexpedition"): # WASTEFUL ! Optimise this
if pr.survexblock._blockid not in personrole_cache:
personrole_cache[pr.survexblock._blockid] = []
personrole_cache[pr.survexblock._blockid].append(pr)
@@ -668,12 +668,12 @@ class LoadingSurvex:
# pr_save_mysql()
# return
try:
SurvexPersonRole.objects.bulk_create(valid_list,
SurvexPersonTeam.objects.bulk_create(valid_list,
update_fields = ['survexblock', 'personname', 'person',
'personexpedition'],
unique_fields = ['survexblock', 'personname', 'person', 'personexpedition']
)
print(f" - all {len(valid_list)} SurvexPersonRoles bulk-created to db", file=sys.stderr)
print(f" - all {len(valid_list)} SurvexPersonTeams bulk-created to db", file=sys.stderr)
except Exception as e:
message = f"\n ! - EXCEPTION '{e}' - in PR bulk update. Falling back onto sequential updates method"
print(message)
@@ -690,7 +690,7 @@ class LoadingSurvex:
continue
# Not using on MariaDB yet: tricky to get foreign keys working
# This is not the complete set of fields we need:
got_obj, created = SurvexPersonRole.objects.get_or_create(
got_obj, created = SurvexPersonTeam.objects.get_or_create(
survexblock=pr.survexblock,
personname=pr.personname,
person=pr.person,
@@ -705,12 +705,12 @@ class LoadingSurvex:
got_obj.survexblock = pr.survexblock
got_obj.save()
stash_data_issue(parser="survex", message=f"PR saved {pr}")
print(f" - {ns}/{nc} SurvexPersonRoles saved/created to db", file=sys.stderr)
print(f" - {ns}/{nc} SurvexPersonTeams saved/created to db", file=sys.stderr)
pr_list = []
for blk in self._pending_pr_saves:
pr_list += self._pending_pr_saves[blk]
print(f" - Saving {len(pr_list):,} SurvexPersonRoles to db.. ({connection.vendor})", file=sys.stderr)
print(f" - Saving {len(pr_list):,} SurvexPersonTeams to db.. ({connection.vendor})", file=sys.stderr)
# Now commit to db
valid_list = []
@@ -725,7 +725,7 @@ class LoadingSurvex:
parser="survex",
message=message
)
print(f" - {len(valid_list)} SurvexPersonRoles in list, saving.. ({connection.vendor})", file=sys.stderr)
print(f" - {len(valid_list)} SurvexPersonTeams in list, saving.. ({connection.vendor})", file=sys.stderr)
if connection.vendor == 'mysql':
pr_save_mysql()
else:
@@ -823,12 +823,12 @@ class LoadingSurvex:
def put_personrole_on_trip(self, survexblock, personexpedition, tm):
"""
Only used for a single person.
Creates a SurvexPersonRole object, but this is not committed to the database until
Creates a SurvexPersonTeam object, but this is not committed to the database until
all the survexblocks have been saved.
"""
try:
personrole = SurvexPersonRole( # does not commit to db yet
personrole = SurvexPersonTeam( # does not commit to db yet
survexblock=survexblock, # survexblock has no _id yet
person = personexpedition.person,
personexpedition=personexpedition,
@@ -1296,7 +1296,7 @@ class LoadingSurvex:
survexblock.expedition = expo
team = get_team_on_trip(survexblock) # should be empty, should only be in 'pending'
# team = SurvexPersonRole.objects.filter(survexblock=survexblock)
# team = SurvexPersonTeam.objects.filter(survexblock=survexblock)
if len(team) > 0:
message = f"! *team {expo.year} Multiple *date in one block? Already someone on team when *date seen. {survexblock.survexfile.path} ({survexblock}) in '{line}'"
print(self.insp + message)
@@ -2453,8 +2453,8 @@ class LoadingSurvex:
with transaction.atomic():
self.save_personroles_to_db()
n = SurvexPersonRole.objects.all().count()
print(f" + Now {n} SurvexPersonRoles in total", file=sys.stderr)
n = SurvexPersonTeam.objects.all().count()
print(f" + Now {n} SurvexPersonTeams in total", file=sys.stderr)
n = QM.objects.all().count()
print(f" - {n} QMs already", file=sys.stderr)
@@ -3349,7 +3349,7 @@ def survexifywallets():
# Batch add people to wallets to minimize DB hits
from collections import defaultdict
wallet_to_people = defaultdict(set)
sprsall = SurvexPersonRole.objects.all().select_related("person").select_related("survexblock")
sprsall = SurvexPersonTeam.objects.all().select_related("person").select_related("survexblock")
for spr in sprsall:
w = spr.survexblock.scanswallet
if w and spr.person:
@@ -3398,13 +3398,13 @@ def survexifywallets():
# Find the survex blocks which are 'ours' i.e. ignore all those (ARGE etc) without expo people attached.
cuccblocks = set()
sprs = SurvexPersonRole.objects.all()
sprs = SurvexPersonTeam.objects.all()
cuccblocks_count = 0
for spr in sprs:
cuccblocks.add(spr.survexblock)
cuccblocks_count += 1
# if cuccblocks_count % 1000 == 0:
# print(f" - Processed {cuccblocks_count} SurvexPersonRole for cuccblocks in {time.time() - start:.2f}s", file=sys.stderr)
# print(f" - Processed {cuccblocks_count} SurvexPersonTeam for cuccblocks in {time.time() - start:.2f}s", file=sys.stderr)
# Because we have just run set_survexblocks(w), this should only complain if there is no *ref and no wallet that links to its parent file
sentinelbad = Wallet.objects.get(walletname="1983#00")
@@ -3445,7 +3445,7 @@ def LoadSurvexBlocks():
# https://docs.djangoproject.com/en/dev/ref/models/fields/#django.db.models.ForeignKey.on_delete
SurvexBlock.objects.all().delete()
SurvexFile.objects.all().delete()
SurvexPersonRole.objects.all().delete()
SurvexPersonTeam.objects.all().delete()
SurvexStation.objects.all().delete()
qms_to_go = QM.objects.filter(loaded_from_csv=False) # the survex QMs, not the CSV QMs
qms_to_keep = QM.objects.filter(loaded_from_csv=True) # the survex QMs, not the CSV QMs