forked from expo/troggle
Setting wallet dates earlier in the import process
This commit is contained in:
parent
c7d88077ec
commit
9d1c0ac395
@ -99,7 +99,7 @@ class Wallet(models.Model):
|
||||
|
||||
# Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it
|
||||
def date(self):
|
||||
"""Reads all the JSON data just to get the JSNON date."""
|
||||
"""Reads all the JSON data just to get the JSON date."""
|
||||
if self.walletdate:
|
||||
return self.walletdate
|
||||
if not self.get_json():
|
||||
|
||||
@ -11,9 +11,9 @@ from troggle.core.models.survex import SingleScan, SurvexBlock
|
||||
from troggle.core.models.wallets import Wallet
|
||||
from troggle.core.models.troggle import DataIssue, Expedition, Person
|
||||
from troggle.core.views.expo import getmimetype
|
||||
from troggle.parsers.survex import set_walletdate
|
||||
|
||||
# from troggle.parsers.people import GetPersonExpeditionNameLookup
|
||||
|
||||
# import parsers.surveys
|
||||
|
||||
"""one of these views serves files as binary blobs, and simply set the mime type based on the file extension,
|
||||
@ -23,7 +23,7 @@ by looking inside the file before being served.
|
||||
need to check if inavlid query string is invalid, or produces multiple replies
|
||||
and render a user-friendly error page.
|
||||
|
||||
Note that datewallet(), caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the
|
||||
Note that caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the
|
||||
manywallets dict.
|
||||
|
||||
TODO
|
||||
@ -47,24 +47,7 @@ def populatewallet(w):
|
||||
w.persons = list(set(survexpeople))
|
||||
|
||||
|
||||
def datewallet(w, earliest):
|
||||
"""Gets the date of the youngest survexblock associated with the wallet
|
||||
REFACTOR this to do the whole date-getting task
|
||||
"""
|
||||
first = earliest
|
||||
blocks = SurvexBlock.objects.filter(scanswallet=w)
|
||||
for b in blocks:
|
||||
if b.date:
|
||||
if b.date < first:
|
||||
first = b.date
|
||||
if first == earliest:
|
||||
# no date found
|
||||
w.date = None
|
||||
else:
|
||||
w.date = first.isoformat()
|
||||
return w.date
|
||||
|
||||
|
||||
|
||||
def caveifywallet(w):
|
||||
"""Gets the cave from the list of survex files,
|
||||
only selects one of them though. Only used for display.
|
||||
@ -105,11 +88,7 @@ def fillblankpeople(w):
|
||||
|
||||
def fillblankothers(w):
|
||||
if not w.walletdate:
|
||||
earliest = datetime.datetime.now().date()
|
||||
if not w.date(): # sets .walletdate as a side-effect, gets it from JSON
|
||||
d = datewallet(w, earliest) # if nothing in JASON, it looks at the survex blocks
|
||||
w.walletdate = d
|
||||
w.save()
|
||||
set_walletdate(w)
|
||||
|
||||
Gcavelookup = GetCaveLookup()
|
||||
|
||||
@ -180,6 +159,8 @@ def walletslistperson(request, first_name, last_name):
|
||||
|
||||
|
||||
def setwalletsdates():
|
||||
"""This sets all the undated wallets, but they should already all be dated on
|
||||
import or on edit"""
|
||||
wallets = Wallet.objects.filter(walletdate=None)
|
||||
print(f"undated wallets: {len(wallets)}")
|
||||
for w in wallets:
|
||||
|
||||
@ -842,6 +842,7 @@ def walletedit(request, path=None):
|
||||
},
|
||||
)
|
||||
else: # no wallet data: should never happen as there should be default data in all cases
|
||||
print(f"! - Should never get here ! Problem in wallet editing...")
|
||||
context = {
|
||||
"year": year,
|
||||
"prev": prev,
|
||||
|
||||
@ -6,7 +6,7 @@ from troggle.core.models.survex import SingleScan
|
||||
from troggle.core.models.troggle import DataIssue
|
||||
from troggle.core.models.wallets import Wallet
|
||||
|
||||
"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
|
||||
"""Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. Loads all the wallets .
|
||||
"""
|
||||
|
||||
contentsjson = "contents.json"
|
||||
@ -16,9 +16,13 @@ git = settings.GIT
|
||||
# to do: Actually read all the JSON files and set the survex file field appropriately!
|
||||
|
||||
|
||||
def setwalletyear(wallet):
|
||||
def set_walletyear(wallet):
|
||||
_ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear
|
||||
|
||||
def set_JSONwalletdate(wallet):
|
||||
"""At this point in the import process, the survex files have not been imported so
|
||||
we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here."""
|
||||
_ = wallet.date() # don't need return value. Sets .walletdate as side effect
|
||||
|
||||
def load_all_scans():
|
||||
"""This iterates through the scans directories (either here or on the remote server)
|
||||
@ -104,7 +108,7 @@ def load_all_scans():
|
||||
print("", flush=True, end="")
|
||||
# Create the wallet object. But we don't have a date for it yet.
|
||||
wallet = Wallet(fpath=fpath, walletname=walletname)
|
||||
setwalletyear(wallet)
|
||||
set_walletyear(wallet)
|
||||
wallet.save()
|
||||
wallets[walletname] = wallet
|
||||
|
||||
@ -147,17 +151,18 @@ def load_all_scans():
|
||||
# The wallets found from JSON should all have dates already
|
||||
wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
|
||||
wallets[walletname] = wallet
|
||||
# could now also load the json but we don't. Do later, on-demand
|
||||
# wallet.walletdate = wallet.date()
|
||||
# could check if link to svx file is valid too.. but do on-demand later
|
||||
# But we *do* set the walletyear:
|
||||
setwalletyear(wallet)
|
||||
# Now also load the json
|
||||
# BUT can't check linked survex blocks as they haven't been imported yet
|
||||
set_JSONwalletdate(wallet)
|
||||
set_walletyear(wallet)
|
||||
if not created:
|
||||
print(
|
||||
f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?"
|
||||
)
|
||||
wallet.save()
|
||||
print(f"\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets")
|
||||
|
||||
# Only the 1999 wallets have filenames which mean that the walletyear will be unset:
|
||||
wallets = Wallet.objects.filter(walletyear=None)
|
||||
for w in wallets:
|
||||
w.walletyear = datetime.date(1999, 1, 1)
|
||||
|
||||
@ -55,6 +55,33 @@ class SurvexLeg:
|
||||
compass = 0.0
|
||||
clino = 0.0
|
||||
|
||||
def datewallet(w, earliest):
|
||||
"""Gets the date of the youngest survexblock associated with the wallet
|
||||
REFACTOR this to do the whole date-getting task
|
||||
|
||||
Currently there is only one SurvexBlock, but this is in anticipation of
|
||||
chnaging the schema to allow many.
|
||||
"""
|
||||
first = earliest
|
||||
blocks = SurvexBlock.objects.filter(scanswallet=w) # only ONE I think ?!
|
||||
for b in blocks:
|
||||
if b.date:
|
||||
if b.date < first:
|
||||
first = b.date
|
||||
if first == earliest:
|
||||
# no date found
|
||||
w.date = None
|
||||
else:
|
||||
w.date = first.isoformat()
|
||||
return w.date
|
||||
|
||||
def set_walletdate(w):
|
||||
earliest = datetime.now().date()
|
||||
if not w.date(): # sets .walletdate as a side-effect if it gets it from JSON
|
||||
d = datewallet(w, earliest) # Not in JSON, so checks all the survex blocks
|
||||
w.walletdate = d
|
||||
w.save()
|
||||
|
||||
def stash_data_issue(parser=None, message=None, url=None, sb=None):
|
||||
"""Avoid hitting the database for error messages until the end of the import"""
|
||||
global dataissues
|
||||
@ -848,6 +875,7 @@ class LoadingSurvex:
|
||||
survexblock.save()
|
||||
# This is where we should check that the wallet JSON contains a link to the survexfile
|
||||
# and that the JSON date and walletdate are set correctly to the survexblock date.
|
||||
set_walletdate(survexblock.scanswallet)
|
||||
else:
|
||||
perps = get_people_on_trip(survexblock)
|
||||
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."
|
||||
|
||||
Loading…
Reference in New Issue
Block a user