xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

attempted speedup, explicit dates wallet objects

Browse Source
This commit is contained in:
Philip Sargent
2022-12-20 00:07:55 +00:00
parent bb14c94ab1
commit f24f283a07
9 changed files with 73 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
parsers/scans.py
View File

@@ -26,7 +26,9 @@ git = settings.GIT
# to do: Actually read all the JSON files and set the survex file field appropriately!
def setwalletyear(wallet):
_ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear
def load_all_scans():
'''This iterates through the scans directories (either here or on the remote server)
and builds up the models we can access later.
@@ -85,7 +87,9 @@ def load_all_scans():
wallet = wallets[walletname]
else:
print("", flush=True, end='')
# Create the wallet object. But we don't have a date for it yet.
wallet = Wallet(fpath=fpath, walletname=walletname)
setwalletyear(wallet)
wallet.save()
wallets[walletname] = wallet
@@ -110,7 +114,7 @@ def load_all_scans():
# but we also need to check if JSON exists, even if there are no uploaded scan files.
# Here we know there is a rigid folder structure, so no need to look for sub folders
print(f"\n - Checking for wallets where only JSON exists, but there are no actual uploaded scan files:")
print(f"\n - Checking for wallets where JSON exists, but there may be no uploaded scan files:")
print(' ', end='')
wjson = 0
contents_path = Path(settings.DRAWINGS_DATA, "walletjson")
@@ -127,10 +131,18 @@ def load_all_scans():
print(f"{walletname} ", end='')
fpath = Path(settings.SCANS_ROOT, str(yeardir.stem), walletname)
# The wallets found from JSON should all have dates already
wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
wallets[walletname] = wallet
# could now also load the json and use it. check &ref is correct or missing too..
# could now also load the json but we don't. Do later, on-demand
# wallet.walletdate = wallet.date()
# could check if link to svx file is valid too.. but do on-demand later
# But we *do* set the walletyear:
setwalletyear(wallet)
if not created:
print(f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?")
wallet.save()
print(f'\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets')
wallets = Wallet.objects.filter(walletyear=None)
for w in wallets:
w.walletyear = datetime.date(1999, 1, 1)