expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-22 07:11:52 +00:00
Code Issues 19 Projects 3 Activity

Setting wallet dates earlier in the import process

Browse Source
This commit is contained in:
Philip Sargent 2023-02-01 23:43:05 +00:00
parent c7d88077ec
commit 9d1c0ac395
5 changed files with 49 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/models/wallets.py
View File

@ -99,7 +99,7 @@ class Wallet(models.Model):
# Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it
def date(self):
"""Reads all the JSON data just to get the JSNON date."""
"""Reads all the JSON data just to get the JSON date."""
if self.walletdate:
return self.walletdate
if not self.get_json():

31
core/views/scans.py
View File

@ -11,9 +11,9 @@ from troggle.core.models.survex import SingleScan, SurvexBlock
from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import DataIssue, Expedition, Person
from troggle.core.views.expo import getmimetype
from troggle.parsers.survex import set_walletdate
# from troggle.parsers.people import GetPersonExpeditionNameLookup
# import parsers.surveys
"""one of these views serves files as binary blobs, and simply set the mime type based on the file extension,
@ -23,7 +23,7 @@ by looking inside the file before being served.
need to check if inavlid query string is invalid, or produces multiple replies
and render a user-friendly error page.
Note that datewallet(), caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the
Note that caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the
manywallets dict.
TODO
@ -47,24 +47,7 @@ def populatewallet(w):
w.persons = list(set(survexpeople))
def datewallet(w, earliest):
"""Gets the date of the youngest survexblock associated with the wallet
REFACTOR this to do the whole date-getting task
"""
first = earliest
blocks = SurvexBlock.objects.filter(scanswallet=w)
for b in blocks:
if b.date:
if b.date < first:
first = b.date
if first == earliest:
# no date found
w.date = None
else:
w.date = first.isoformat()
return w.date
def caveifywallet(w):
"""Gets the cave from the list of survex files,
only selects one of them though. Only used for display.
@ -105,11 +88,7 @@ def fillblankpeople(w):
def fillblankothers(w):
if not w.walletdate:
earliest = datetime.datetime.now().date()
if not w.date(): # sets .walletdate as a side-effect, gets it from JSON
d = datewallet(w, earliest) # if nothing in JASON, it looks at the survex blocks
w.walletdate = d
w.save()
set_walletdate(w)
Gcavelookup = GetCaveLookup()
@ -180,6 +159,8 @@ def walletslistperson(request, first_name, last_name):
def setwalletsdates():
"""This sets all the undated wallets, but they should already all be dated on
import or on edit"""
wallets = Wallet.objects.filter(walletdate=None)
print(f"undated wallets: {len(wallets)}")
for w in wallets:

1
core/views/wallets_edit.py
View File

@ -842,6 +842,7 @@ def walletedit(request, path=None):
},
)
else: # no wallet data: should never happen as there should be default data in all cases
print(f"! - Should never get here ! Problem in wallet editing...")
context = {
"year": year,
"prev": prev,

21
parsers/scans.py
View File

@ -6,7 +6,7 @@ from troggle.core.models.survex import SingleScan
from troggle.core.models.troggle import DataIssue
from troggle.core.models.wallets import Wallet
"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. Loads all the wallets .
"""
contentsjson = "contents.json"
@ -16,9 +16,13 @@ git = settings.GIT
# to do: Actually read all the JSON files and set the survex file field appropriately!
def setwalletyear(wallet):
def set_walletyear(wallet):
_ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear
def set_JSONwalletdate(wallet):
"""At this point in the import process, the survex files have not been imported so
we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here."""
_ = wallet.date() # don't need return value. Sets .walletdate as side effect
def load_all_scans():
"""This iterates through the scans directories (either here or on the remote server)
@ -104,7 +108,7 @@ def load_all_scans():
print("", flush=True, end="")
# Create the wallet object. But we don't have a date for it yet.
wallet = Wallet(fpath=fpath, walletname=walletname)
setwalletyear(wallet)
set_walletyear(wallet)
wallet.save()
wallets[walletname] = wallet
@ -147,17 +151,18 @@ def load_all_scans():
# The wallets found from JSON should all have dates already
wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
wallets[walletname] = wallet
# could now also load the json but we don't. Do later, on-demand
# wallet.walletdate = wallet.date()
# could check if link to svx file is valid too.. but do on-demand later
# But we *do* set the walletyear:
setwalletyear(wallet)
# Now also load the json
# BUT can't check linked survex blocks as they haven't been imported yet
set_JSONwalletdate(wallet)
set_walletyear(wallet)
if not created:
print(
f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?"
)
wallet.save()
print(f"\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets")
# Only the 1999 wallets have filenames which mean that the walletyear will be unset:
wallets = Wallet.objects.filter(walletyear=None)
for w in wallets:
w.walletyear = datetime.date(1999, 1, 1)

28
parsers/survex.py
View File

@ -55,6 +55,33 @@ class SurvexLeg:
compass = 0.0
clino = 0.0
def datewallet(w, earliest):
"""Gets the date of the youngest survexblock associated with the wallet
REFACTOR this to do the whole date-getting task
Currently there is only one SurvexBlock, but this is in anticipation of
chnaging the schema to allow many.
"""
first = earliest
blocks = SurvexBlock.objects.filter(scanswallet=w) # only ONE I think ?!
for b in blocks:
if b.date:
if b.date < first:
first = b.date
if first == earliest:
# no date found
w.date = None
else:
w.date = first.isoformat()
return w.date
def set_walletdate(w):
earliest = datetime.now().date()
if not w.date(): # sets .walletdate as a side-effect if it gets it from JSON
d = datewallet(w, earliest) # Not in JSON, so checks all the survex blocks
w.walletdate = d
w.save()
def stash_data_issue(parser=None, message=None, url=None, sb=None):
"""Avoid hitting the database for error messages until the end of the import"""
global dataissues
@ -848,6 +875,7 @@ class LoadingSurvex:
survexblock.save()
# This is where we should check that the wallet JSON contains a link to the survexfile
# and that the JSON date and walletdate are set correctly to the survexblock date.
set_walletdate(survexblock.scanswallet)
else:
perps = get_people_on_trip(survexblock)
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."