mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2024-11-22 07:11:52 +00:00
attempted speedup, explicit dates wallet objects
This commit is contained in:
parent
bb14c94ab1
commit
f24f283a07
@ -172,6 +172,7 @@ class Wallet(models.Model):
|
||||
fpath = models.CharField(max_length=200)
|
||||
walletname = models.CharField(max_length=200)
|
||||
walletdate = models.DateField(blank=True, null=True)
|
||||
walletyear = models.DateField(blank=True, null=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ('walletname',)
|
||||
@ -213,6 +214,8 @@ class Wallet(models.Model):
|
||||
print(f' - {datestr=} ')
|
||||
try:
|
||||
thisdate = datetime.date.fromisoformat(datestr)
|
||||
self.walletdate = thisdate
|
||||
self.save()
|
||||
try:
|
||||
waldata["date"] = thisdate.isoformat()
|
||||
except:
|
||||
@ -220,20 +223,24 @@ class Wallet(models.Model):
|
||||
from troggle.core.models.troggle import DataIssue
|
||||
DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl)
|
||||
except:
|
||||
message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load fro, {jsonfile} JSON file"
|
||||
message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
|
||||
from troggle.core.models.troggle import DataIssue
|
||||
DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl)
|
||||
return waldata
|
||||
|
||||
def year(self):
|
||||
'''This gets the year syntactically without opening and reading the JSON
|
||||
'''
|
||||
if len(self.walletname) < 5:
|
||||
return None
|
||||
if self.walletname[4] != "#":
|
||||
return None
|
||||
year = int(self.walletname[0:4])
|
||||
if year < 1976 or year > 2050:
|
||||
if year < 1975 or year > 2050:
|
||||
return None
|
||||
else:
|
||||
self.walletyear = datetime.date(year, 1, 1)
|
||||
self.save()
|
||||
return str(year)
|
||||
|
||||
|
||||
@ -243,7 +250,7 @@ class Wallet(models.Model):
|
||||
return self.walletdate
|
||||
if not self.get_json():
|
||||
return None
|
||||
jsondata = self.get_json()
|
||||
jsondata = self.get_json() # use walrus operator?
|
||||
|
||||
datestr = jsondata["date"]
|
||||
if not datestr:
|
||||
@ -252,12 +259,13 @@ class Wallet(models.Model):
|
||||
datestr = datestr.replace('.','-')
|
||||
try:
|
||||
samedate = datetime.date.fromisoformat(datestr)
|
||||
self.walletdate = samedate.isoformat()
|
||||
except:
|
||||
try:
|
||||
samedate = datetime.date.fromisoformat(datestr[:10])
|
||||
self.walletdate = samedate.isoformat()
|
||||
except:
|
||||
samedate = None
|
||||
self.walletdate = samedate.isoformat()
|
||||
self.save()
|
||||
return self.walletdate
|
||||
|
||||
|
||||
@ -8,6 +8,7 @@ from urllib.request import urlopen
|
||||
from django.conf import settings
|
||||
from django.shortcuts import render
|
||||
from django.http import HttpResponse
|
||||
from django.db import transaction
|
||||
|
||||
from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock
|
||||
from troggle.core.models.troggle import Person, Expedition
|
||||
@ -48,6 +49,9 @@ def populatewallet(w):
|
||||
w.persons = list(set(survexpeople))
|
||||
|
||||
def datewallet(w, earliest):
|
||||
'''Gets the date of the youngest survexblock associated with the wallet
|
||||
REFACTOR this to do the whole date-getting task
|
||||
'''
|
||||
first = earliest
|
||||
blocks = SurvexBlock.objects.filter(scanswallet = w)
|
||||
for b in blocks:
|
||||
@ -59,6 +63,7 @@ def datewallet(w, earliest):
|
||||
w.date = None
|
||||
else:
|
||||
w.date = first.isoformat()
|
||||
return w.date
|
||||
|
||||
def caveifywallet(w):
|
||||
'''Gets the cave from the list of survex files,
|
||||
@ -94,10 +99,13 @@ def fillblankpeople(w):
|
||||
# print(f' - {wp=} {nobody=}')
|
||||
populatewallet(w)
|
||||
|
||||
def fillblankothers(w):
|
||||
earliest = datetime.datetime.now().date()
|
||||
if not w.date():
|
||||
datewallet(w, earliest)
|
||||
def fillblankothers(w):
|
||||
if not w.walletdate:
|
||||
earliest = datetime.datetime.now().date()
|
||||
if not w.date(): # sets .walletdate as a side-effect, gets it from JSON
|
||||
d =datewallet(w, earliest) # if nothing in JASON, it looks at the survex blocks
|
||||
w.walletdate = d
|
||||
w.save()
|
||||
|
||||
Gcavelookup = GetCaveLookup()
|
||||
|
||||
@ -140,6 +148,8 @@ def walletslistperson(request, first_name, last_name):
|
||||
w.ticks = w.get_ticks() # the complaints in colour form
|
||||
fixsurvextick(w, w.ticks)
|
||||
return manywallets
|
||||
|
||||
print(f"-walletslistperson")
|
||||
|
||||
try:
|
||||
if last_name:
|
||||
@ -153,37 +163,43 @@ def walletslistperson(request, first_name, last_name):
|
||||
|
||||
manywallets = tickspersonwallet(p)
|
||||
expeditions = Expedition.objects.all()
|
||||
print(f"--")
|
||||
return render(request, 'personwallets.html', { 'manywallets':manywallets, 'settings': settings, 'person': p, 'expeditions': expeditions})
|
||||
|
||||
def setwalletsdates():
|
||||
wallets = Wallet.objects.filter(walletdate=None)
|
||||
print(f"undated wallets: {len(wallets)}")
|
||||
for w in wallets:
|
||||
w.walletdate = w.date()
|
||||
w.save()
|
||||
|
||||
|
||||
def walletslistyear(request, year):
|
||||
'''Page which displays a list of all the wallets in a specific year
|
||||
'''Page which displays a list of all the wallets in a specific year.
|
||||
We have a field .walletyear, which we set on import.
|
||||
'''
|
||||
def ticksyearwallet(year):
|
||||
manywallets = []
|
||||
wallets = Wallet.objects.all()
|
||||
wallets = Wallet.objects.filter(walletyear__year=year)
|
||||
for w in wallets:
|
||||
|
||||
if year == w.year():
|
||||
manywallets.append(w)
|
||||
fillblankpeople(w)
|
||||
fillblankothers(w)
|
||||
w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
|
||||
fixsurvextick(w, w.ticks)
|
||||
else:
|
||||
continue
|
||||
manywallets.append(w)
|
||||
fillblankpeople(w)
|
||||
fillblankothers(w)
|
||||
w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
|
||||
fixsurvextick(w, w.ticks)
|
||||
|
||||
return manywallets
|
||||
|
||||
print(f"-walletslistyear")
|
||||
if year < 1976 or year > 2050:
|
||||
return render(request, 'errors/generic.html', {'message': 'Year out of range. Must be between 1976 and 2050'})
|
||||
else:
|
||||
year = str(year)
|
||||
|
||||
#return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'})
|
||||
|
||||
|
||||
year = str(year)
|
||||
manywallets = ticksyearwallet(year)
|
||||
expeditions = Expedition.objects.all()
|
||||
expedition = Expedition.objects.filter(year=year)
|
||||
expedition = expeditions.filter(year=year)
|
||||
print(f"--")
|
||||
return render(request, 'yearwallets.html', { 'manywallets':manywallets, 'settings': settings, 'year': year, 'expeditions': expeditions, 'expedition': expedition})
|
||||
|
||||
|
||||
@ -191,6 +207,8 @@ def walletslistyear(request, year):
|
||||
def cavewallets(request, caveid):
|
||||
'''Returns all the wallets for just one cave
|
||||
'''
|
||||
print(f"-cavewalletsl")
|
||||
|
||||
Gcavelookup = GetCaveLookup()
|
||||
if caveid in Gcavelookup:
|
||||
cave = Gcavelookup[caveid]
|
||||
@ -224,6 +242,7 @@ def cavewallets(request, caveid):
|
||||
w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
|
||||
fixsurvextick(w, w.ticks)
|
||||
expeditions = Expedition.objects.all()
|
||||
print(f"--")
|
||||
return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave, 'expeditions': expeditions})
|
||||
|
||||
|
||||
|
||||
@ -41,7 +41,7 @@ def import_logbooks():
|
||||
with transaction.atomic():
|
||||
troggle.parsers.logbooks.LoadLogbooks()
|
||||
|
||||
def import_logbook(year=2018):
|
||||
def import_logbook(year=2019):
|
||||
print(f"-- Importing Logbook {year}")
|
||||
with transaction.atomic():
|
||||
troggle.parsers.logbooks.LoadLogbook(year)
|
||||
|
||||
@ -26,7 +26,9 @@ git = settings.GIT
|
||||
|
||||
# to do: Actually read all the JSON files and set the survex file field appropriately!
|
||||
|
||||
|
||||
def setwalletyear(wallet):
|
||||
_ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear
|
||||
|
||||
def load_all_scans():
|
||||
'''This iterates through the scans directories (either here or on the remote server)
|
||||
and builds up the models we can access later.
|
||||
@ -85,7 +87,9 @@ def load_all_scans():
|
||||
wallet = wallets[walletname]
|
||||
else:
|
||||
print("", flush=True, end='')
|
||||
# Create the wallet object. But we don't have a date for it yet.
|
||||
wallet = Wallet(fpath=fpath, walletname=walletname)
|
||||
setwalletyear(wallet)
|
||||
wallet.save()
|
||||
wallets[walletname] = wallet
|
||||
|
||||
@ -110,7 +114,7 @@ def load_all_scans():
|
||||
|
||||
# but we also need to check if JSON exists, even if there are no uploaded scan files.
|
||||
# Here we know there is a rigid folder structure, so no need to look for sub folders
|
||||
print(f"\n - Checking for wallets where only JSON exists, but there are no actual uploaded scan files:")
|
||||
print(f"\n - Checking for wallets where JSON exists, but there may be no uploaded scan files:")
|
||||
print(' ', end='')
|
||||
wjson = 0
|
||||
contents_path = Path(settings.DRAWINGS_DATA, "walletjson")
|
||||
@ -127,10 +131,18 @@ def load_all_scans():
|
||||
|
||||
print(f"{walletname} ", end='')
|
||||
fpath = Path(settings.SCANS_ROOT, str(yeardir.stem), walletname)
|
||||
# The wallets found from JSON should all have dates already
|
||||
wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
|
||||
wallets[walletname] = wallet
|
||||
# could now also load the json and use it. check &ref is correct or missing too..
|
||||
# could now also load the json but we don't. Do later, on-demand
|
||||
# wallet.walletdate = wallet.date()
|
||||
# could check if link to svx file is valid too.. but do on-demand later
|
||||
# But we *do* set the walletyear:
|
||||
setwalletyear(wallet)
|
||||
if not created:
|
||||
print(f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?")
|
||||
wallet.save()
|
||||
print(f'\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets')
|
||||
wallets = Wallet.objects.filter(walletyear=None)
|
||||
for w in wallets:
|
||||
w.walletyear = datetime.date(1999, 1, 1)
|
||||
|
||||
@ -660,6 +660,8 @@ class LoadingSurvex():
|
||||
else:
|
||||
survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
|
||||
survexblock.save()
|
||||
# This is where we should check that the wallet JSON contains a link to the survexfile
|
||||
# and that the JSON date and walletdate are set correctly to the survexblock date.
|
||||
else:
|
||||
perps = get_people_on_trip(survexblock)
|
||||
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."
|
||||
|
||||
@ -28,7 +28,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
|
||||
<tr>
|
||||
<td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
|
||||
|
||||
<td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
|
||||
<td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
|
||||
<td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
|
||||
<td style="padding:2px">{{wallet.persons }} </td>
|
||||
|
||||
|
||||
@ -27,7 +27,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
|
||||
<tr>
|
||||
<td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
|
||||
|
||||
<td style="padding:2px" >{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
|
||||
<td style="padding:2px" >{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
|
||||
<td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
|
||||
<td style="padding:2px">{{wallet.persons}}</td>
|
||||
<td style="padding:2px">
|
||||
|
||||
@ -21,7 +21,7 @@
|
||||
<tr>
|
||||
<td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
|
||||
|
||||
<td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
|
||||
<td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
|
||||
<td style="padding:2px">
|
||||
{% if wallet.cave %}
|
||||
{% if wallet.caveobj.slug %}
|
||||
|
||||
@ -35,7 +35,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
|
||||
<tr>
|
||||
<td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
|
||||
|
||||
<td style="padding:2px">{% if wallet.date %}{{wallet.date}}{% else %} {% endif %}</td>
|
||||
<td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
|
||||
<td style="padding:2px">{% if wallet.name %}{{wallet.name}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
|
||||
<td style="padding:2px">{{wallet.persons}}</td>
|
||||
<td style="padding:2px">
|
||||
|
||||
Loading…
Reference in New Issue
Block a user