2
0
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-24 16:21:53 +00:00

Revised (faster, less brain damaged) wallet pages

This commit is contained in:
Philip Sargent 2023-10-25 01:56:30 +03:00
parent 33477f2b40
commit 0843a27966
8 changed files with 151 additions and 92 deletions

View File

@ -99,8 +99,7 @@ class Wallet(models.Model):
walletdate = models.DateField(blank=True, null=True)
walletyear = models.DateField(blank=True, null=True)
caves = models.ManyToManyField("Cave", related_name="wallets")
# not yet
#persons = models.ManyToManyField("Person", related_name="wallets")
persons = models.ManyToManyField("Person", related_name="wallets")
class Meta:
ordering = ("walletname",)
@ -127,6 +126,8 @@ class Wallet(models.Model):
if stuff:
return [stuff] # single object, not a string, but now in a list.
return []
def get_absolute_url(self):
# we do not use URL_ROOT any more.
@ -221,24 +222,6 @@ class Wallet(models.Model):
print(f"FAIL adding cave to wallet.caves '{i}'")
pass
# if type(cavelist) is list:
# for i in cavelist:
# if i != "":
# i = i.replace("/", "-")
# caveobject = get_cave_leniently(i)
# self.caves.add(caveobject) # new many-to-many field
# else:
# # either single cave or the square barckets have been removed and it s a singoe string
# ids = cavelist.split(",")
# for i in ids:
# j = i.replace("'","").replace("/", "-").strip('[] "')
# if i != "":
# try:
# caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data
# self.caves.add(caveobject)
# except:
# print(f"FAIL adding cave to wallet.caves '{j}'")
# pass
def year(self):
"""This gets the year syntactically without opening and reading the JSON"""
if len(self.walletname) < 5:
@ -294,6 +277,15 @@ class Wallet(models.Model):
return None
return jsondata["name"]
def survexfiles(self):
if not (jsondata := self.get_json()): # WALRUS
return None
filelist = Wallet.input_to_list(jsondata["survex file"])
#print(f"'{self} {jsondata['survex file']}' => {filelist}")
return filelist
def get_fnames(self):
'''Filenames without the suffix, i.e. without the ".jpg"'''
dirpath = Path(settings.SCANS_ROOT, self.fpath) # does nowt as fpath is a rooted path already

View File

@ -27,8 +27,6 @@ Note that caveifywallet() etc do NOT save the object to the db. They are ephemer
manywallets dict.
TODO
cave for a wallet - just gets the last one, randomly. SHould make this a list or many:many ideally
-- add the participants on an explicit wallet list to .slugpeople so that they get proper URL-linked
on the per-person wallet report, and do the same thing for per-cave and per-year wallet reports
@ -36,19 +34,19 @@ on the per-person wallet report, and do the same thing for per-cave and per-year
add this file in to the todo list thinggy.
"""
def fix_manywallets(many):
for w in many:
fillblankpeople(w)
fillblankothers(w)
w.ticks = w.get_ticks() # the complaints in colour form
fixsurvextick(w, w.ticks)
def populatewallet(w):
"""Copy survex data here just for display, not permanently
"""Need to tidy this up, now all the work is done at parse time not here
Only gets data from the survex file when it was parsed on import, or edited (& thus parsed) online,
so doesn't work if there was no *ref value
"""
slugpeople = set()
blocks = SurvexBlock.objects.filter(scanswallet=w)
for b in blocks:
for personrole in b.survexpersonrole_set.all():
slugpeople.add(personrole.person) # Person objects, not the names anymore
w.slugpeople = slugpeople
w.slugpeople = w.persons.all()
def caveifywallet(w):
@ -104,10 +102,10 @@ def fillblankpeople(w):
if nobody(wp):
populatewallet(w) # sets w.slugpeople
else:
w.persons = parse_name_list(w)
w.peeps = parse_name_list(w)
populatewallet(w) # sets w.slugpeople
if hasattr(w, "slugpeople"):
w.persons = w.persons.difference(w.slugpeople)
w.peeps = w.peeps.difference(w.slugpeople)
return
@ -243,11 +241,7 @@ def walletslistperson(request, slug):
print(message)
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
for w in manywallets:
fillblankpeople(w)
fillblankothers(w)
w.ticks = w.get_ticks() # the complaints in colour form
fixsurvextick(w, w.ticks)
fix_manywallets(manywallets)
return manywallets
# print("-walletslistperson")
@ -257,6 +251,7 @@ def walletslistperson(request, slug):
expeditions = Expedition.objects.all()
length_ug = 0.0
for w in manywallets:
print(w.persons)
for sb in w.survexblock_set.all():
length_ug += sb.legslength
return render(
@ -284,25 +279,18 @@ def walletslistyear(request, year):
def ticksyearwallet(year):
manywallets = []
wallets = Wallet.objects.filter(walletyear__year=year)
for w in wallets:
manywallets.append(w)
fillblankpeople(w)
fillblankothers(w)
w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
fixsurvextick(w, w.ticks)
manywallets = Wallet.objects.filter(walletyear__year=year)
fix_manywallets(manywallets)
return manywallets
# print("-walletslistyear")
if year < 1976 or year > 2050:
return render(request, "errors/generic.html", {"message": "Year out of range. Must be between 1976 and 2050"})
# return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'})
year = str(year)
manywallets = ticksyearwallet(year)
expeditions = Expedition.objects.all() #bad Django style
expeditions = Expedition.objects.all()
expedition = expeditions.filter(year=year)
length_ug = 0.0
for w in manywallets:
@ -334,11 +322,8 @@ def cavewallets(request, caveid):
wallets = cave.wallets.all()
manywallets = list(set(wallets))
for w in manywallets:
fillblankpeople(w)
fillblankothers(w)
w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
fixsurvextick(w, w.ticks)
fix_manywallets(manywallets)
expeditions = Expedition.objects.all()
length_ug = 0.0
for w in manywallets:

View File

@ -56,7 +56,7 @@ def import_survex():
troggle.parsers.survex.LoadSurvexBlocks()
print(" - Survex entrances x/y/z Positions")
with transaction.atomic():
troggle.parsers.survex.caveifywallets()
troggle.parsers.survex.survexifywallets()
with transaction.atomic():
troggle.parsers.locations.LoadPositions()

View File

@ -7,6 +7,9 @@ from troggle.core.models.troggle import DataIssue
from troggle.core.models.wallets import Wallet
"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. Loads all the wallets .
todo - update the survexblock scanswallet field to refer to the wallet, if it isn't already.
But we can't do that, we only have the survexfile, not the blcok. But we could set it for ALL the blocks? Hmm.
"""
contentsjson = "contents.json"
@ -177,3 +180,4 @@ def load_all_scans():

View File

@ -138,7 +138,7 @@ def get_offending_filename(path):
"""
return "/survexfile/" + path + ".svx"
# THIS SHOULD NOT BE GLOBAL ! SHould be per instance of file loader..
trip_people_cache = {} # indexed by survexblock, so never needs cleaning out
def get_team_on_trip(survexblock):
"""Uses a cache to avoid a database query if it doesn't need to.
@ -146,6 +146,7 @@ def get_team_on_trip(survexblock):
global trip_people_cache
if survexblock in trip_people_cache:
if len(trip_people_cache[survexblock]) > 0:
return trip_people_cache[survexblock]
qpeople = SurvexPersonRole.objects.filter(survexblock=survexblock) # not very good Django style
@ -154,7 +155,8 @@ def get_team_on_trip(survexblock):
def get_people_on_trip(survexblock):
"""Gets the displayable names of the people on a survexbock trip.
Only used for complete team."""
Only used for complete team.
Seems to be only used for error messages."""
qpeople = get_team_on_trip(survexblock) # qpeople is a Query List
people = []
@ -699,7 +701,7 @@ class LoadingSurvex:
else:
message = f"! DATE Warning LONG DATE '{oline}' ({survexblock}) {survexblock.survexfile.path}"
print(self.insp+message)
stash_data_issue(parser='svxdate', message=message, url=None, sb=(survexblock.survexfile.path))
stash_data_issue(parser='xSvxDate', message=message, url=None, sb=(survexblock.survexfile.path))
if len(line) == 10:
@ -712,7 +714,7 @@ class LoadingSurvex:
message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m") # sets to first of month
elif len(line) == 4:
@ -720,7 +722,7 @@ class LoadingSurvex:
message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
survexblock.date = datetime.strptime(line, "%Y") # sets to January 1st
elif len(line) == 9 or len(line) == 8:
@ -728,7 +730,7 @@ class LoadingSurvex:
message = f"! DATE format WARNING, single digit day or month number,'{oline}' [{line[-5]}][{line[-2]}] ({survexblock}) {survexblock.survexfile.path}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
if line[-2] == "-" or line[-2] == ".":
line = line[:-1] + '0' + line[-1]
@ -745,7 +747,7 @@ class LoadingSurvex:
)
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path)
)
else:
# these errors are reporting the wrong survexblock, which is actually a SurvexFile (!)
@ -985,7 +987,7 @@ class LoadingSurvex:
perps = get_people_on_trip(survexblock)
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' malformed id '{args}' {perps}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
return
if not letterx:
@ -994,24 +996,24 @@ class LoadingSurvex:
letterx = "X"
message = f" ! Wallet *REF has LETTER in '{survexblock.survexfile.path}' malformed id '{args}' {perps}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
if len(wallet) < 2:
wallet = "0" + wallet
if not (int(yr) > 1960 and int(yr) < 2050):
message = " ! Wallet year out of bounds {yr} '{refscan}' {survexblock.survexfile.path}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
refscan = f"{yr}#{letterx}{wallet}"
try:
if int(wallet) > 99:
message = f" ! Wallet *REF {refscan} - very big (more than 99) so probably wrong in '{survexblock.survexfile.path}'"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
except:
message = f" ! Wallet *REF {refscan} - not numeric in '{survexblock.survexfile.path}'"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
manywallets = Wallet.objects.filter(
walletname=refscan
@ -1020,13 +1022,13 @@ class LoadingSurvex:
if len(manywallets) > 1:
message = f" ! Wallet *REF {refscan} - more than one found {len(manywallets)} wallets in db with same id {survexblock.survexfile.path}"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
if survexblock.scanswallet:
if survexblock.scanswallet.walletname != refscan:
message = f" ! Wallet *REF {refscan} in {survexblock.survexfile.path} - Already a DIFFERENT wallet is set for this block '{survexblock.scanswallet.walletname}'"
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
else:
survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
survexblock.save()
@ -1037,7 +1039,7 @@ class LoadingSurvex:
perps = get_people_on_trip(survexblock)
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."
print(self.insp + message)
stash_data_issue(parser="survex", message=message, url=url)
stash_data_issue(parser="ref", message=message, url=url)
def LoadSurvexDataNormal(self, survexblock, args):
"""Sets the order for data elements in this and following blocks, e.g.
@ -2481,16 +2483,81 @@ def MakeFileRoot(svxpath):
return fileroot
def caveifywallets():
def set_survexblocks(wallet):
if svxfiles := wallet.survexfiles(): # reads from JSON, should be cached
for svx in svxfiles:
# svx is a string, need to resolve to a survexfile object
#o = SurvexFile.objects.get(path=svx)
blocks = SurvexBlock.objects.filter(survexfile__path=svx)
for b in blocks:
if b.scanswallet == wallet:
pass
else:
b.scanswallet = wallet
b.save()
# print(f"setting {wallet} on {b.survexfile} : {b}")
def survexifywallets():
"""Gets the caves from the list of survexblocks
We seem to hve a LOT of blocks with no atatched scnaswallet. Is this because we are
not inheriting *ref properly in the survexfile ?
"""
print(f" - Update wallets with survex data")
start = time.time()
# if there is a wallet for a block, add the people to the wallet
sprsall = SurvexPersonRole.objects.all().select_related("person").select_related("survexblock")
for spr in sprsall:
w = spr.survexblock.scanswallet
if w:
w.persons.add(spr.person)
duration = time.time() - start
print(f" - TIME: add people to wallets {duration:7.2f} s", file=sys.stderr)
start = time.time()
wallets = Wallet.objects.all()
for w in wallets:
blocks = SurvexBlock.objects.filter(scanswallet=w)
set_survexblocks(w) # reads JSON, sets survexblocks if survexfiles specified on wallet JSON
duration = time.time() - start
print(f" - TIME: set survexblock:wallet using JSON survexfiles {duration:7.2f} s", file=sys.stderr)
start = time.time()
for w in wallets:
blocks = SurvexBlock.objects.filter(scanswallet=w).select_related("survexfile")
for b in blocks:
# NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath
if b.survexfile.cave:
w.caves.add(b.survexfile.cave)
w.save()
duration = time.time() - start
print(f" - TIME: add caves to wallets {duration:7.2f} s", file=sys.stderr)
start = time.time()
cuccblocks = set()
for spr in SurvexPersonRole.objects.all():
cuccblocks.add(spr.survexblock)
sentinelbad = Wallet.objects.get(walletname="1983#00")
for b in cuccblocks:
if b.date > date(2001, 1, 1): # do we care about older ones? 1999 certainly has different wallet system
if not b.scanswallet:
if b.parent.scanswallet:
if b.parent.scanswallet != sentinelbad:
b.scanswallet = b.parent.scanswallet
continue
message = f" ! *REF missing {b.date} {b.survexfile}.svx : '{b}'"
# print(message, file=sys.stderr)
url = get_offending_filename(b.survexfile.path)
DataIssue.objects.update_or_create(parser="ref", message=message, url=url)
duration = time.time() - start
print(f" - TIME: check missing *ref on survexblocks {duration:7.2f} s", file=sys.stderr)
start = time.time()
def LoadSurvexBlocks():
@ -2516,10 +2583,11 @@ def LoadSurvexBlocks():
global dataissues
dataissues = []
DataIssue.objects.filter(parser="survex").delete()
DataIssue.objects.filter(parser="svxdate").delete()
DataIssue.objects.filter(parser="xSvxDate").delete()
DataIssue.objects.filter(parser="survexleg").delete()
DataIssue.objects.filter(parser="survexunits").delete()
DataIssue.objects.filter(parser="survex team").delete()
DataIssue.objects.filter(parser="ref").delete()
# DataIssue.objects.filter(parser="xEntrances").delete()
print(" - survex Data Issues flushed")
mem1 = get_process_memory()

View File

@ -33,13 +33,15 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
<td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
<td style="padding:2px">{% if wallet.name %}{{wallet.name|truncatechars:20}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
<td style="padding:2px">
{% if wallet.slugpeople %}
<em>{%for p in wallet.slugpeople%}<a href="/person/{{p.slug}}">{{p.fullname}}</a>{%if not forloop.last %}, {% endif %}{% endfor %}</em>
{% endif %}
{%for p in wallet.persons%}
{%for p in wallet.peeps%}
<a href="/person/{{p.slug}}">{{p.fullname}}</a>{%if not forloop.last %}, {% endif %}
{% endfor %}</td> <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td>
{% endfor %}
</td>
<td style="padding:2px">
{% for survexblock in wallet.survexblock_set.all %}
<a href="{% url "svx" survexblock.survexfile.path %}">{{survexblock}}</a>

View File

@ -13,6 +13,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
<p>See also wallets
<ul>
<li>per person, e.g. <a href="/wallets/person/wookey">Wookey</a>, <a href="/wallets/person/chris-densham">Chris Densham</a>, <a href="/wallets/person/martin-green">Martin Green</a>, <a href="/wallets/person/andrew-atkinson">Andrew Atkinson</a>, <a href="/wallets/person/julian-todd">Julian Todd</a>, <a href="/wallets/person/frank-tully">FrankTully</a>, <a href="/wallets/person/dave-loeffler">Dave Loeffler</a>, <a href="/wallets/person/becka-lawson">Becka</a>
<li>per cave, e.g. <a href="/cave/scans/1623-161">1623-161</a>, <a href="/cave/scans/1626-359">1626-359</a>, <a href="/cave/scans/1623-290">1623-290</a>, <a href="/cave/scans/1623-291">1623-291</a>, <a href="/cave/scans/1623-264">1623-264</a>
<li>per year:
{% for otherexpedition in expeditions %}
@ -22,7 +23,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
{% include 'wallet_table.html' %}
<p>Note that names in italics are copied from the related survex file block name.
<br />
<p>Total underground survey length: {{length_ug|floatformat:"1g"}} m
<p>Total underground survey length: {{length_ug|floatformat:"1g"}} m - but this includes all the surveys associated with the wallet, including those where <em>{{person.fullname|safe}}</em> did not participate.
<br />
<table width=95%>
<tr><th>Wallet</th><th width=13%>Wallet Date</th><th>Wallet Name</th><th width=28%>Team</th><th width=8%>Cave(s)</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr>
@ -32,19 +33,25 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
<td style="padding:2px" >{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
<td style="padding:2px">{% if wallet.name %}{{wallet.name|truncatechars:20}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
<td style="padding:2px">
{% if wallet.slugpeople %}
<em>{%for p in wallet.slugpeople%}<a href="/person/{{p.slug}}">{{p.fullname}}</a>{%if not forloop.last %}, {% endif %}{% endfor %}</em>
{% endif %}
{%for p in wallet.persons%}
{%for p in wallet.peeps%}
<a href="/person/{{p.slug}}">{{p.fullname}}</a>{%if not forloop.last %}, {% endif %}
{% endfor %}</td>
{% endfor %}
</td>
<td style="padding:2px; font-family: monospace; font-size: 90%;">
{% for c in wallet.caves.all %}
<a href="/cave/scans/{{c.slug}}">{{c}}</a>
{% endfor %}
</td>
<td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td>
<td style="padding:2px">
{% for survexblock in wallet.survexblock_set.all %}

View File

@ -38,14 +38,15 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
<td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
<td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
<td style="padding:2px">{% if wallet.name %}{{wallet.name|truncatechars:20}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
<td style="padding:2px">
{% if wallet.slugpeople %}
<em>{%for p in wallet.slugpeople%}<a href="/person/{{p.slug}}">{{p.fullname}}</a>{%if not forloop.last %}, {% endif %}{% endfor %}</em>
{% endif %}
{%for p in wallet.persons%}
{%for p in wallet.peeps%}
<a href="/person/{{p.slug}}">{{p.fullname}}</a>{%if not forloop.last %}, {% endif %}
{% endfor %}</td>
{% endfor %}
<td style="padding:2px; font-family: monospace; font-size: 90%;">
{% for c in wallet.caves.all %}