expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-21 23:01:52 +00:00
Code Issues 19 Projects 3 Activity

cleaned up wallets import process

Browse Source
This commit is contained in:
Philip Sargent 2024-08-11 18:41:27 +03:00
parent fc145648e6
commit 3e198fc410
3 changed files with 39 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
core/models/wallets.py
View File

@ -143,43 +143,41 @@ class Wallet(models.Model):
Reads JSON date and sets w.walletdate Reads JSON date and sets w.walletdate
This repeats a lot of stuff done in the initial parsing job:
traversing the file system.
Needs to be refactored as special handling of subdirectories is duplicated
import DataIssue locally to prevent import cycle problem""" import DataIssue locally to prevent import cycle problem"""
if hasattr(self, "JSONdata"): if hasattr(self, "JSONdata"):
return self.JSONdata return self.JSONdata
scans_path = Path(settings.SCANS_ROOT) scans_path = Path(settings.SCANS_ROOT)
wurl = self.get_url()
# :drawings: walletjson/2022/2022#01/contents.json # :drawings: walletjson/2022/2022#01/contents.json
# fpath = /mnt/d/EXPO/expofiles/surveyscans/1999/1999#02 # fpath = /mnt/d/EXPO/expofiles/surveyscans/1999/1999#02
# This does not work if there are subdirectories in the surveyscan folder
fp = Path(self.fpath) fp = Path(self.fpath)
if fp.name in archaic_wallets: if fp.name in archaic_wallets:
return None return None
if fp.parent.parent.parent.parent == scans_path:
print(f" ! ", end="")
if fp.parent.parent.parent.parent.parent == scans_path:
print(f" !! ", end="")
if fp.parent.parent.parent.parent == scans_path: if fp.parent.parent.parent.parent == scans_path:
subfolder = fp.parent.name subfolder = fp.parent.name
wname = fp.parent.parent.name wname = fp.parent.parent.name
wyear = fp.parent.parent.parent.name wyear = fp.parent.parent.parent.name
print(f"\n - Subfolder {subfolder} one deep, detected in {wname}") # print(f" - Subfolder {subfolder} two deep, detected in {wname=} {wyear=} {wurl=}")
else: else:
wname = fp.name wname = fp.name
wyear = fp.parent.name wyear = fp.parent.name
try: try:
y = int(wyear) y = int(wyear)
except: except:
message = f"\n! 2 or more deep subfolder detected?:\n {fp.parent.name=}\n {fp.name=} \n {self.fpath=}" message = f"! 3 or more deep subfolder detected?:\n {fp.parent.name=}\n {fp.name=} \n {self.fpath=} {wurl=}"
print(message) print(message)
# this is a hack, work down from /surveyscans instead. To be fixed # this is a hack, work down from /surveyscans instead. To be fixed
wname = fp.parent.parent.parent.name wname = fp.parent.parent.parent.name
wyear = fp.parent.parent.parent.parent.name wyear = fp.parent.parent.parent.parent.name
wurl = self.get_url()
if len(wyear) != 4 or len(wname) !=6: if len(wyear) != 4 or len(wname) !=6:
# no contents.json for old-style wallets # no contents.json for old-style wallets
# but this ruined all the tick-list displays.. why?! # but this ruined all the tick-list displays.. why?!
@ -188,6 +186,9 @@ class Wallet(models.Model):
jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json" jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
if not Path(jsonfile).is_file(): if not Path(jsonfile).is_file():
message = f"! {jsonfile} is not a file:\n {wyear=} (should be eg. '2035')\n {wname=} (should be eg. '2035#13')\n {self.fpath=}"
print(message)
if wname not in archaic_wallets: if wname not in archaic_wallets:
message = f"! {jsonfile} is not a file:\n {wyear=} (should be eg. '2023')\n {wname=} (should be eg. '2023#13')\n {self.fpath=}" message = f"! {jsonfile} is not a file:\n {wyear=} (should be eg. '2023')\n {wname=} (should be eg. '2023#13')\n {self.fpath=}"
print(message) print(message)

30
parsers/scans.py
View File

@ -81,6 +81,9 @@ def load_all_scans():
# they are if they are /2010/2010#33 # they are if they are /2010/2010#33
# or /1996-1999NotKHbook/ # or /1996-1999NotKHbook/
# but not if they are /2010/2010#33/therion/ : the wallet is /2010#33/ not /therion/ # but not if they are /2010/2010#33/therion/ : the wallet is /2010#33/ not /therion/
# READ THE FUNCTION get_json(self) in models/wallets which ALSO does this SAME job
# needs refactoring
print(" ", end="") print(" ", end="")
scans_path = Path(settings.SCANS_ROOT) scans_path = Path(settings.SCANS_ROOT)
seen = set() seen = set()
@ -96,19 +99,25 @@ def load_all_scans():
pass pass
else: else:
c += 1 c += 1
if c % 60 == 0: if c % 120 == 0:
print(".", end="") print(".", end="")
if c % 3000 == 0: if c % 6000 == 0:
print("\n ", end="") print("\n ", end="")
if p.parent.parent.parent.parent == scans_path: # Finding the wallet from deeply hidden subdirectories
# the wallet has subfolders, which we are uninterested in at this point. if p.parent.parent.parent.parent.parent.parent == scans_path:
# print(f"too deep {p}", end='\n') fpath = p.parent.parent.parent.parent
walletname = p.parent.parent.parent.parent.name
elif p.parent.parent.parent.parent.parent == scans_path:
fpath = p.parent.parent.parent
walletname = p.parent.parent.parent.name
elif p.parent.parent.parent.parent == scans_path:
fpath = p.parent.parent fpath = p.parent.parent
walletname = p.parent.parent.name # wallet is one level higher walletname = p.parent.parent.name
else: else:
fpath = p.parent fpath = p.parent
walletname = p.parent.name walletname = p.parent.name
if walletname in wallets: # assumes all walletnames are unique if walletname in wallets: # assumes all walletnames are unique
wallet = wallets[walletname] wallet = wallets[walletname]
@ -129,17 +138,14 @@ def load_all_scans():
if len(tag) > 4: if len(tag) > 4:
if tag[4] != "#": if tag[4] != "#":
relative_path = p.relative_to(scans_path, walk_up=False) relative_path = p.relative_to(scans_path, walk_up=False)
seen.add(str(relative_path.parent)) seen.add((str(relative_path.parent)+"/", walletname))
wjson = 0 wjson = 0
print("\n ", end="")
seenlist = list(seen) seenlist = list(seen)
seenlist.sort() seenlist.sort()
print("\n")
for tag in seenlist: for tag in seenlist:
wjson += 1 wjson += 1
if wjson % 10 == 0: print(f" {tag} ", end="\n")
print("\n ", end="")
print(f" {tag} ", end="")
print(f"\n - found and loaded {c:,} acceptable scan files in {len(wallets):,} wallets") print(f"\n - found and loaded {c:,} acceptable scan files in {len(wallets):,} wallets")
# but we also need to check if JSON exists, even if there are no uploaded scan files. # but we also need to check if JSON exists, even if there are no uploaded scan files.

4
parsers/survex.py
View File

@ -2709,12 +2709,12 @@ def set_survexblocks():
pass pass
elif b.scanswallet: elif b.scanswallet:
if b.date > date(2024, 1, 1) and b.date < date(2025, 1, 1): if b.date > date(2024, 1, 1) and b.date < date(2025, 1, 1):
print(f"2019 not set{wallet} on {b.survexfile} for block {b} as set explicitly to {b.scanswallet}") print(f"2024-check not set{wallet} on {b.survexfile} for block {b} as set explicitly to {b.scanswallet}")
else: else:
b.scanswallet = wallet b.scanswallet = wallet
b.save() b.save()
if b.date > date(2024, 1, 1) and b.date < date(2025, 1, 1): if b.date > date(2024, 1, 1) and b.date < date(2025, 1, 1):
print(f"2019 setting {wallet} on {b.survexfile} for block {b}") print(f"2024-check setting {wallet} on {b.survexfile} for block {b}")
def survexifywallets(): def survexifywallets():
"""Gets the caves from the list of survexblocks """Gets the caves from the list of survexblocks