expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-22 07:11:52 +00:00
Code Issues 19 Projects 3 Activity

Rearranging wallets setting from scanning surveyscans

Browse Source
This commit is contained in:
Philip Sargent 2024-08-11 17:02:59 +03:00
parent 6e1c83ec6e
commit fc145648e6
1 changed files with 21 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
parsers/scans.py
View File

@ -40,12 +40,13 @@ def load_all_scans():
SingleScan.objects.all().delete() SingleScan.objects.all().delete()
Wallet.objects.all().delete() Wallet.objects.all().delete()
print(" - deleting all Wallet and SingleScan objects, and resize error messages") # print(" - deleting all Wallet and SingleScan objects, and resize error messages")
DataIssue.objects.filter(parser="scans").delete() DataIssue.objects.filter(parser="scans").delete()
DataIssue.objects.filter(parser="wallets").delete() DataIssue.objects.filter(parser="wallets").delete()
DataIssue.objects.filter(parser="mogrify").delete() DataIssue.objects.filter(parser="mogrify").delete()
# These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet. # These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet.
# But we ignore this list.. anything goes..
valids = [ valids = [
".top", ".top",
".txt", ".txt",
@ -82,7 +83,7 @@ def load_all_scans():
# but not if they are /2010/2010#33/therion/ : the wallet is /2010#33/ not /therion/ # but not if they are /2010/2010#33/therion/ : the wallet is /2010#33/ not /therion/
print(" ", end="") print(" ", end="")
scans_path = Path(settings.SCANS_ROOT) scans_path = Path(settings.SCANS_ROOT)
seen = [] seen = set()
c = 0 c = 0
wallets = {} wallets = {}
for p in scans_path.rglob("*"): # this implicitly ignoires empty folders as it looks only for files for p in scans_path.rglob("*"): # this implicitly ignoires empty folders as it looks only for files
@ -95,9 +96,9 @@ def load_all_scans():
pass pass
else: else:
c += 1 c += 1
if c % 15 == 0: if c % 60 == 0:
print(".", end="") print(".", end="")
if c % 750 == 0: if c % 3000 == 0:
print("\n ", end="") print("\n ", end="")
if p.parent.parent.parent.parent == scans_path: if p.parent.parent.parent.parent == scans_path:
@ -109,7 +110,7 @@ def load_all_scans():
fpath = p.parent fpath = p.parent
walletname = p.parent.name walletname = p.parent.name
if walletname in wallets: if walletname in wallets: # assumes all walletnames are unique
wallet = wallets[walletname] wallet = wallets[walletname]
else: else:
print("", flush=True, end="") print("", flush=True, end="")
@ -124,17 +125,21 @@ def load_all_scans():
singlescan.save() singlescan.save()
# only printing progress: # only printing progress:
tag = p.parent tag = str(p.parent.name) # folder containing the scan file
if len(walletname) > 4: if len(tag) > 4:
if walletname[4] == "#": if tag[4] != "#":
tag = p.parent.parent relative_path = p.relative_to(scans_path, walk_up=False)
seen.add(str(relative_path.parent))
if tag not in seen: wjson = 0
print(f" {tag.name} ", end="") print("\n ", end="")
if len(str(tag.name)) > 17: seenlist = list(seen)
print("\n ", end="") seenlist.sort()
seen.append(tag)
for tag in seenlist:
wjson += 1
if wjson % 10 == 0:
print("\n ", end="")
print(f" {tag} ", end="")
print(f"\n - found and loaded {c:,} acceptable scan files in {len(wallets):,} wallets") print(f"\n - found and loaded {c:,} acceptable scan files in {len(wallets):,} wallets")
# but we also need to check if JSON exists, even if there are no uploaded scan files. # but we also need to check if JSON exists, even if there are no uploaded scan files.