expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-12-17 17:37:08 +00:00
Code Issues 18 Projects 3 Activity

Revised (faster, less brain damaged) wallet pages

Browse Source
This commit is contained in:
Philip Sargent
2023-10-25 01:56:30 +03:00
parent 33477f2b40
commit 0843a27966
8 changed files with 151 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
parsers/imports.py
View File

@@ -56,7 +56,7 @@ def import_survex():
troggle.parsers.survex.LoadSurvexBlocks()
print(" - Survex entrances x/y/z Positions")
with transaction.atomic():
troggle.parsers.survex.caveifywallets()
troggle.parsers.survex.survexifywallets()
with transaction.atomic():
troggle.parsers.locations.LoadPositions()

6
parsers/scans.py
View File

@@ -7,6 +7,9 @@ from troggle.core.models.troggle import DataIssue
from troggle.core.models.wallets import Wallet
"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. Loads all the wallets .
todo - update the survexblock scanswallet field to refer to the wallet, if it isn't already.
But we can't do that, we only have the survexfile, not the blcok. But we could set it for ALL the blocks? Hmm.
"""
contentsjson = "contents.json"
@@ -25,7 +28,7 @@ def set_JSONwalletdate(wallet):
def set_caves(wallet):
_ = wallet.allcaves() # don't need return value. Just calling this saves it as w.caves This ONLY gets the list on the wallet JSON
def load_all_scans():
"""This iterates through the scans directories (either here or on the remote server)
and builds up the models we can access later.
@@ -173,6 +176,7 @@ def load_all_scans():
for wallet in Wallet.objects.all():
# this reads JSON
wallet.check_survexlist()

110
parsers/survex.py
View File

@@ -138,7 +138,7 @@ def get_offending_filename(path):
"""
return "/survexfile/" + path + ".svx"
# THIS SHOULD NOT BE GLOBAL ! SHould be per instance of file loader..
trip_people_cache = {} # indexed by survexblock, so never needs cleaning out
def get_team_on_trip(survexblock):
"""Uses a cache to avoid a database query if it doesn't need to.
@@ -146,7 +146,8 @@ def get_team_on_trip(survexblock):
global trip_people_cache
if survexblock in trip_people_cache:
return trip_people_cache[survexblock]
if len(trip_people_cache[survexblock]) > 0:
return trip_people_cache[survexblock]
qpeople = SurvexPersonRole.objects.filter(survexblock=survexblock) # not very good Django style
trip_people_cache[survexblock] = qpeople # this is a query list
@@ -154,7 +155,8 @@ def get_team_on_trip(survexblock):
def get_people_on_trip(survexblock):
"""Gets the displayable names of the people on a survexbock trip.
Only used for complete team."""
Only used for complete team.
Seems to be only used for error messages."""
qpeople = get_team_on_trip(survexblock) # qpeople is a Query List
people = []
@@ -699,7 +701,7 @@ class LoadingSurvex:
else:
message = f"! DATE Warning LONG DATE '{oline}' ({survexblock}) {survexblock.survexfile.path}"
print(self.insp+message)
stash_data_issue(parser='svxdate', message=message, url=None, sb=(survexblock.survexfile.path))
stash_data_issue(parser='xSvxDate', message=message, url=None, sb=(survexblock.survexfile.path))
if len(line) == 10:
@@ -712,7 +714,7 @@ class LoadingSurvex:
message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m") # sets to first of month
elif len(line) == 4:
@@ -720,7 +722,7 @@ class LoadingSurvex:
message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
survexblock.date = datetime.strptime(line, "%Y") # sets to January 1st
elif len(line) == 9 or len(line) == 8:
@@ -728,7 +730,7 @@ class LoadingSurvex:
message = f"! DATE format WARNING, single digit day or month number,'{oline}' [{line[-5]}][{line[-2]}] ({survexblock}) {survexblock.survexfile.path}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
if line[-2] == "-" or line[-2] == ".":
line = line[:-1] + '0' + line[-1]
@@ -745,7 +747,7 @@ class LoadingSurvex:
)
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
else:
# these errors are reporting the wrong survexblock, which is actually a SurvexFile (!)
@@ -985,7 +987,7 @@ class LoadingSurvex:
perps = get_people_on_trip(survexblock)
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' malformed id '{args}' {perps}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
return
if not letterx:
@@ -994,24 +996,24 @@ class LoadingSurvex:
letterx = "X"
message = f" ! Wallet *REF has LETTER in '{survexblock.survexfile.path}' malformed id '{args}' {perps}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
if len(wallet) < 2:
wallet = "0" + wallet
if not (int(yr) > 1960 and int(yr) < 2050):
message = " ! Wallet year out of bounds {yr} '{refscan}' {survexblock.survexfile.path}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
refscan = f"{yr}#{letterx}{wallet}"
try:
if int(wallet) > 99:
message = f" ! Wallet *REF {refscan} - very big (more than 99) so probably wrong in '{survexblock.survexfile.path}'"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
except:
message = f" ! Wallet *REF {refscan} - not numeric in '{survexblock.survexfile.path}'"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
manywallets = Wallet.objects.filter(
walletname=refscan
@@ -1020,13 +1022,13 @@ class LoadingSurvex:
if len(manywallets) > 1:
message = f" ! Wallet *REF {refscan} - more than one found {len(manywallets)} wallets in db with same id {survexblock.survexfile.path}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
if survexblock.scanswallet:
if survexblock.scanswallet.walletname != refscan:
message = f" ! Wallet *REF {refscan} in {survexblock.survexfile.path} - Already a DIFFERENT wallet is set for this block '{survexblock.scanswallet.walletname}'"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
else:
survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
survexblock.save()
@@ -1037,7 +1039,7 @@ class LoadingSurvex:
perps = get_people_on_trip(survexblock)
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
def LoadSurvexDataNormal(self, survexblock, args):
"""Sets the order for data elements in this and following blocks, e.g.
@@ -2481,17 +2483,82 @@ def MakeFileRoot(svxpath):
return fileroot
def caveifywallets():
def set_survexblocks(wallet):
if svxfiles := wallet.survexfiles(): # reads from JSON, should be cached
for svx in svxfiles:
# svx is a string, need to resolve to a survexfile object
#o = SurvexFile.objects.get(path=svx)
blocks = SurvexBlock.objects.filter(survexfile__path=svx)
for b in blocks:
if b.scanswallet == wallet:
pass
else:
b.scanswallet = wallet
b.save()
# print(f"setting {wallet} on {b.survexfile} : {b}")
def survexifywallets():
"""Gets the caves from the list of survexblocks
We seem to hve a LOT of blocks with no atatched scnaswallet. Is this because we are
not inheriting *ref properly in the survexfile ?
"""
print(f" - Update wallets with survex data")
start = time.time()
# if there is a wallet for a block, add the people to the wallet
sprsall = SurvexPersonRole.objects.all().select_related("person").select_related("survexblock")
for spr in sprsall:
w = spr.survexblock.scanswallet
if w:
w.persons.add(spr.person)
duration = time.time() - start
print(f" - TIME: add people to wallets {duration:7.2f} s", file=sys.stderr)
start = time.time()
wallets = Wallet.objects.all()
for w in wallets:
blocks = SurvexBlock.objects.filter(scanswallet=w)
set_survexblocks(w) # reads JSON, sets survexblocks if survexfiles specified on wallet JSON
duration = time.time() - start
print(f" - TIME: set survexblock:wallet using JSON survexfiles {duration:7.2f} s", file=sys.stderr)
start = time.time()
for w in wallets:
blocks = SurvexBlock.objects.filter(scanswallet=w).select_related("survexfile")
for b in blocks:
# NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath
if b.survexfile.cave:
w.caves.add(b.survexfile.cave)
w.save()
duration = time.time() - start
print(f" - TIME: add caves to wallets {duration:7.2f} s", file=sys.stderr)
start = time.time()
cuccblocks = set()
for spr in SurvexPersonRole.objects.all():
cuccblocks.add(spr.survexblock)
sentinelbad = Wallet.objects.get(walletname="1983#00")
for b in cuccblocks:
if b.date > date(2001, 1, 1): # do we care about older ones? 1999 certainly has different wallet system
if not b.scanswallet:
if b.parent.scanswallet:
if b.parent.scanswallet != sentinelbad:
b.scanswallet = b.parent.scanswallet
continue
message = f" ! *REF missing {b.date} {b.survexfile}.svx : '{b}'"
# print(message, file=sys.stderr)
url = get_offending_filename(b.survexfile.path)
DataIssue.objects.update_or_create(parser="ref", message=message, url=url)
duration = time.time() - start
print(f" - TIME: check missing *ref on survexblocks {duration:7.2f} s", file=sys.stderr)
start = time.time()
def LoadSurvexBlocks():
global dup_includes
@@ -2516,10 +2583,11 @@ def LoadSurvexBlocks():
global dataissues
dataissues = []
DataIssue.objects.filter(parser="survex").delete()
DataIssue.objects.filter(parser="svxdate").delete()
DataIssue.objects.filter(parser="xSvxDate").delete()
DataIssue.objects.filter(parser="survexleg").delete()
DataIssue.objects.filter(parser="survexunits").delete()
DataIssue.objects.filter(parser="survex team").delete()
DataIssue.objects.filter(parser="ref").delete()
# DataIssue.objects.filter(parser="xEntrances").delete()
print(" - survex Data Issues flushed")
mem1 = get_process_memory()