expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-12-15 03:27:26 +00:00
Code Issues 18 Projects 3 Activity

Revised (faster, less brain damaged) wallet pages

Browse Source
This commit is contained in:
Philip Sargent
2023-10-25 01:56:30 +03:00
parent 33477f2b40
commit 0843a27966
8 changed files with 151 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
parsers/survex.py
View File

@@ -138,7 +138,7 @@ def get_offending_filename(path):
"""
return "/survexfile/" + path + ".svx"
# THIS SHOULD NOT BE GLOBAL ! SHould be per instance of file loader..
trip_people_cache = {} # indexed by survexblock, so never needs cleaning out
def get_team_on_trip(survexblock):
"""Uses a cache to avoid a database query if it doesn't need to.
@@ -146,7 +146,8 @@ def get_team_on_trip(survexblock):
global trip_people_cache
if survexblock in trip_people_cache:
return trip_people_cache[survexblock]
if len(trip_people_cache[survexblock]) > 0:
return trip_people_cache[survexblock]
qpeople = SurvexPersonRole.objects.filter(survexblock=survexblock) # not very good Django style
trip_people_cache[survexblock] = qpeople # this is a query list
@@ -154,7 +155,8 @@ def get_team_on_trip(survexblock):
def get_people_on_trip(survexblock):
"""Gets the displayable names of the people on a survexbock trip.
Only used for complete team."""
Only used for complete team.
Seems to be only used for error messages."""
qpeople = get_team_on_trip(survexblock) # qpeople is a Query List
people = []
@@ -699,7 +701,7 @@ class LoadingSurvex:
else:
message = f"! DATE Warning LONG DATE '{oline}' ({survexblock}) {survexblock.survexfile.path}"
print(self.insp+message)
stash_data_issue(parser='svxdate', message=message, url=None, sb=(survexblock.survexfile.path))
stash_data_issue(parser='xSvxDate', message=message, url=None, sb=(survexblock.survexfile.path))
if len(line) == 10:
@@ -712,7 +714,7 @@ class LoadingSurvex:
message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m") # sets to first of month
elif len(line) == 4:
@@ -720,7 +722,7 @@ class LoadingSurvex:
message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
survexblock.date = datetime.strptime(line, "%Y") # sets to January 1st
elif len(line) == 9 or len(line) == 8:
@@ -728,7 +730,7 @@ class LoadingSurvex:
message = f"! DATE format WARNING, single digit day or month number,'{oline}' [{line[-5]}][{line[-2]}] ({survexblock}) {survexblock.survexfile.path}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
if line[-2] == "-" or line[-2] == ".":
line = line[:-1] + '0' + line[-1]
@@ -745,7 +747,7 @@ class LoadingSurvex:
)
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
else:
# these errors are reporting the wrong survexblock, which is actually a SurvexFile (!)
@@ -985,7 +987,7 @@ class LoadingSurvex:
perps = get_people_on_trip(survexblock)
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' malformed id '{args}' {perps}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
return
if not letterx:
@@ -994,24 +996,24 @@ class LoadingSurvex:
letterx = "X"
message = f" ! Wallet *REF has LETTER in '{survexblock.survexfile.path}' malformed id '{args}' {perps}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
if len(wallet) < 2:
wallet = "0" + wallet
if not (int(yr) > 1960 and int(yr) < 2050):
message = " ! Wallet year out of bounds {yr} '{refscan}' {survexblock.survexfile.path}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
refscan = f"{yr}#{letterx}{wallet}"
try:
if int(wallet) > 99:
message = f" ! Wallet *REF {refscan} - very big (more than 99) so probably wrong in '{survexblock.survexfile.path}'"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
except:
message = f" ! Wallet *REF {refscan} - not numeric in '{survexblock.survexfile.path}'"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
manywallets = Wallet.objects.filter(
walletname=refscan
@@ -1020,13 +1022,13 @@ class LoadingSurvex:
if len(manywallets) > 1:
message = f" ! Wallet *REF {refscan} - more than one found {len(manywallets)} wallets in db with same id {survexblock.survexfile.path}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
if survexblock.scanswallet:
if survexblock.scanswallet.walletname != refscan:
message = f" ! Wallet *REF {refscan} in {survexblock.survexfile.path} - Already a DIFFERENT wallet is set for this block '{survexblock.scanswallet.walletname}'"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
else:
survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
survexblock.save()
@@ -1037,7 +1039,7 @@ class LoadingSurvex:
perps = get_people_on_trip(survexblock)
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
def LoadSurvexDataNormal(self, survexblock, args):
"""Sets the order for data elements in this and following blocks, e.g.
@@ -2481,17 +2483,82 @@ def MakeFileRoot(svxpath):
return fileroot
def caveifywallets():
def set_survexblocks(wallet):
if svxfiles := wallet.survexfiles(): # reads from JSON, should be cached
for svx in svxfiles:
# svx is a string, need to resolve to a survexfile object
#o = SurvexFile.objects.get(path=svx)
blocks = SurvexBlock.objects.filter(survexfile__path=svx)
for b in blocks:
if b.scanswallet == wallet:
pass
else:
b.scanswallet = wallet
b.save()
# print(f"setting {wallet} on {b.survexfile} : {b}")
def survexifywallets():
"""Gets the caves from the list of survexblocks
We seem to hve a LOT of blocks with no atatched scnaswallet. Is this because we are
not inheriting *ref properly in the survexfile ?
"""
print(f" - Update wallets with survex data")
start = time.time()
# if there is a wallet for a block, add the people to the wallet
sprsall = SurvexPersonRole.objects.all().select_related("person").select_related("survexblock")
for spr in sprsall:
w = spr.survexblock.scanswallet
if w:
w.persons.add(spr.person)
duration = time.time() - start
print(f" - TIME: add people to wallets {duration:7.2f} s", file=sys.stderr)
start = time.time()
wallets = Wallet.objects.all()
for w in wallets:
blocks = SurvexBlock.objects.filter(scanswallet=w)
set_survexblocks(w) # reads JSON, sets survexblocks if survexfiles specified on wallet JSON
duration = time.time() - start
print(f" - TIME: set survexblock:wallet using JSON survexfiles {duration:7.2f} s", file=sys.stderr)
start = time.time()
for w in wallets:
blocks = SurvexBlock.objects.filter(scanswallet=w).select_related("survexfile")
for b in blocks:
# NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath
if b.survexfile.cave:
w.caves.add(b.survexfile.cave)
w.save()
duration = time.time() - start
print(f" - TIME: add caves to wallets {duration:7.2f} s", file=sys.stderr)
start = time.time()
cuccblocks = set()
for spr in SurvexPersonRole.objects.all():
cuccblocks.add(spr.survexblock)
sentinelbad = Wallet.objects.get(walletname="1983#00")
for b in cuccblocks:
if b.date > date(2001, 1, 1): # do we care about older ones? 1999 certainly has different wallet system
if not b.scanswallet:
if b.parent.scanswallet:
if b.parent.scanswallet != sentinelbad:
b.scanswallet = b.parent.scanswallet
continue
message = f" ! *REF missing {b.date} {b.survexfile}.svx : '{b}'"
# print(message, file=sys.stderr)
url = get_offending_filename(b.survexfile.path)
DataIssue.objects.update_or_create(parser="ref", message=message, url=url)
duration = time.time() - start
print(f" - TIME: check missing *ref on survexblocks {duration:7.2f} s", file=sys.stderr)
start = time.time()
def LoadSurvexBlocks():
global dup_includes
@@ -2516,10 +2583,11 @@ def LoadSurvexBlocks():
global dataissues
dataissues = []
DataIssue.objects.filter(parser="survex").delete()
DataIssue.objects.filter(parser="svxdate").delete()
DataIssue.objects.filter(parser="xSvxDate").delete()
DataIssue.objects.filter(parser="survexleg").delete()
DataIssue.objects.filter(parser="survexunits").delete()
DataIssue.objects.filter(parser="survex team").delete()
DataIssue.objects.filter(parser="ref").delete()
# DataIssue.objects.filter(parser="xEntrances").delete()
print(" - survex Data Issues flushed")
mem1 = get_process_memory()