xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

attempted speedup, explicit dates wallet objects

Browse Source
This commit is contained in:
Philip Sargent 2022-12-20 00:07:55 +00:00
parent bb14c94ab1
commit f24f283a07
9 changed files with 73 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
core/models/survex.py
View File

@ -172,6 +172,7 @@ class Wallet(models.Model):
fpath = models.CharField(max_length=200)
walletname = models.CharField(max_length=200)
walletdate = models.DateField(blank=True, null=True)
walletyear = models.DateField(blank=True, null=True)
class Meta:
ordering = ('walletname',)
@ -213,6 +214,8 @@ class Wallet(models.Model):
print(f' - {datestr=} ')
try:
thisdate = datetime.date.fromisoformat(datestr)
self.walletdate = thisdate
self.save()
try:
waldata["date"] = thisdate.isoformat()
except:
@ -220,20 +223,24 @@ class Wallet(models.Model):
from troggle.core.models.troggle import DataIssue
DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl)
except:
message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load fro, {jsonfile} JSON file"
message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
from troggle.core.models.troggle import DataIssue
DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl)
return waldata
def year(self):
'''This gets the year syntactically without opening and reading the JSON
'''
if len(self.walletname) < 5:
return None
if self.walletname[4] != "#":
return None
year = int(self.walletname[0:4])
if year < 1976 or year > 2050:
if year < 1975 or year > 2050:
return None
else:
self.walletyear = datetime.date(year, 1, 1)
self.save()
return str(year)
@ -243,7 +250,7 @@ class Wallet(models.Model):
return self.walletdate
if not self.get_json():
return None
jsondata = self.get_json()
jsondata = self.get_json() # use walrus operator?
datestr = jsondata["date"]
if not datestr:
@ -252,12 +259,13 @@ class Wallet(models.Model):
datestr = datestr.replace('.','-')
try:
samedate = datetime.date.fromisoformat(datestr)
self.walletdate = samedate.isoformat()
except:
try:
samedate = datetime.date.fromisoformat(datestr[:10])
self.walletdate = samedate.isoformat()
except:
samedate = None
self.walletdate = samedate.isoformat()
self.save()
return self.walletdate

43
core/views/scans.py
View File

@ -8,6 +8,7 @@ from urllib.request import urlopen
from django.conf import settings
from django.shortcuts import render
from django.http import HttpResponse
from django.db import transaction
from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock
from troggle.core.models.troggle import Person, Expedition
@ -48,6 +49,9 @@ def populatewallet(w):
w.persons = list(set(survexpeople))
def datewallet(w, earliest):
'''Gets the date of the youngest survexblock associated with the wallet
REFACTOR this to do the whole date-getting task
'''
first = earliest
blocks = SurvexBlock.objects.filter(scanswallet = w)
for b in blocks:
@ -59,6 +63,7 @@ def datewallet(w, earliest):
w.date = None
else:
w.date = first.isoformat()
return w.date
def caveifywallet(w):
'''Gets the cave from the list of survex files,
@ -95,9 +100,12 @@ def fillblankpeople(w):
populatewallet(w)
def fillblankothers(w):
if not w.walletdate:
earliest = datetime.datetime.now().date()
if not w.date():
datewallet(w, earliest)
if not w.date(): # sets .walletdate as a side-effect, gets it from JSON
d =datewallet(w, earliest) # if nothing in JASON, it looks at the survex blocks
w.walletdate = d
w.save()
Gcavelookup = GetCaveLookup()
@ -141,6 +149,8 @@ def walletslistperson(request, first_name, last_name):
fixsurvextick(w, w.ticks)
return manywallets
print(f"-walletslistperson")
try:
if last_name:
p = Person.objects.get(fullname= f'{first_name} {last_name}')
@ -153,37 +163,43 @@ def walletslistperson(request, first_name, last_name):
manywallets = tickspersonwallet(p)
expeditions = Expedition.objects.all()
print(f"--")
return render(request, 'personwallets.html', { 'manywallets':manywallets, 'settings': settings, 'person': p, 'expeditions': expeditions})
def setwalletsdates():
wallets = Wallet.objects.filter(walletdate=None)
print(f"undated wallets: {len(wallets)}")
for w in wallets:
w.walletdate = w.date()
w.save()
def walletslistyear(request, year):
'''Page which displays a list of all the wallets in a specific year
'''Page which displays a list of all the wallets in a specific year.
We have a field .walletyear, which we set on import.
'''
def ticksyearwallet(year):
manywallets = []
wallets = Wallet.objects.all()
wallets = Wallet.objects.filter(walletyear__year=year)
for w in wallets:
if year == w.year():
manywallets.append(w)
fillblankpeople(w)
fillblankothers(w)
w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
fixsurvextick(w, w.ticks)
else:
continue
return manywallets
print(f"-walletslistyear")
if year < 1976 or year > 2050:
return render(request, 'errors/generic.html', {'message': 'Year out of range. Must be between 1976 and 2050'})
else:
year = str(year)
#return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'})
year = str(year)
manywallets = ticksyearwallet(year)
expeditions = Expedition.objects.all()
expedition = Expedition.objects.filter(year=year)
expedition = expeditions.filter(year=year)
print(f"--")
return render(request, 'yearwallets.html', { 'manywallets':manywallets, 'settings': settings, 'year': year, 'expeditions': expeditions, 'expedition': expedition})
@ -191,6 +207,8 @@ def walletslistyear(request, year):
def cavewallets(request, caveid):
'''Returns all the wallets for just one cave
'''
print(f"-cavewalletsl")
Gcavelookup = GetCaveLookup()
if caveid in Gcavelookup:
cave = Gcavelookup[caveid]
@ -224,6 +242,7 @@ def cavewallets(request, caveid):
w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
fixsurvextick(w, w.ticks)
expeditions = Expedition.objects.all()
print(f"--")
return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave, 'expeditions': expeditions})

2
parsers/imports.py
View File

@ -41,7 +41,7 @@ def import_logbooks():
with transaction.atomic():
troggle.parsers.logbooks.LoadLogbooks()
def import_logbook(year=2018):
def import_logbook(year=2019):
print(f"-- Importing Logbook {year}")
with transaction.atomic():
troggle.parsers.logbooks.LoadLogbook(year)

16
parsers/scans.py
View File

@ -26,6 +26,8 @@ git = settings.GIT
# to do: Actually read all the JSON files and set the survex file field appropriately!
def setwalletyear(wallet):
_ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear
def load_all_scans():
'''This iterates through the scans directories (either here or on the remote server)
@ -85,7 +87,9 @@ def load_all_scans():
wallet = wallets[walletname]
else:
print("", flush=True, end='')
# Create the wallet object. But we don't have a date for it yet.
wallet = Wallet(fpath=fpath, walletname=walletname)
setwalletyear(wallet)
wallet.save()
wallets[walletname] = wallet
@ -110,7 +114,7 @@ def load_all_scans():
# but we also need to check if JSON exists, even if there are no uploaded scan files.
# Here we know there is a rigid folder structure, so no need to look for sub folders
print(f"\n - Checking for wallets where only JSON exists, but there are no actual uploaded scan files:")
print(f"\n - Checking for wallets where JSON exists, but there may be no uploaded scan files:")
print(' ', end='')
wjson = 0
contents_path = Path(settings.DRAWINGS_DATA, "walletjson")
@ -127,10 +131,18 @@ def load_all_scans():
print(f"{walletname} ", end='')
fpath = Path(settings.SCANS_ROOT, str(yeardir.stem), walletname)
# The wallets found from JSON should all have dates already
wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
wallets[walletname] = wallet
# could now also load the json and use it. check &ref is correct or missing too..
# could now also load the json but we don't. Do later, on-demand
# wallet.walletdate = wallet.date()
# could check if link to svx file is valid too.. but do on-demand later
# But we *do* set the walletyear:
setwalletyear(wallet)
if not created:
print(f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?")
wallet.save()
print(f'\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets')
wallets = Wallet.objects.filter(walletyear=None)
for w in wallets:
w.walletyear = datetime.date(1999, 1, 1)

2
parsers/survex.py
View File

@ -660,6 +660,8 @@ class LoadingSurvex():
else:
survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
survexblock.save()
# This is where we should check that the wallet JSON contains a link to the survexfile
# and that the JSON date and walletdate are set correctly to the survexblock date.
else:
perps = get_people_on_trip(survexblock)
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."

2
templates/cavewallets.html
View File

@ -28,7 +28,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
<tr>
<td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
<td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
<td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
<td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
<td style="padding:2px">{{wallet.persons }} </td>

2
templates/personwallets.html
View File

@ -27,7 +27,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
<tr>
<td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
<td style="padding:2px" >{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
<td style="padding:2px" >{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
<td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
<td style="padding:2px">{{wallet.persons}}</td>
<td style="padding:2px">

2
templates/wallet_table.html
View File

@ -21,7 +21,7 @@
<tr>
<td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
<td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
<td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
<td style="padding:2px">
{% if wallet.cave %}
{% if wallet.caveobj.slug %}

2
templates/yearwallets.html
View File

@ -35,7 +35,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
<tr>
<td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
<td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
<td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
<td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
<td style="padding:2px">{{wallet.persons}}</td>
<td style="padding:2px">