diff --git a/parsers/scans.py b/parsers/scans.py index e5138c3..947278e 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -40,12 +40,13 @@ def load_all_scans(): SingleScan.objects.all().delete() Wallet.objects.all().delete() - print(" - deleting all Wallet and SingleScan objects, and resize error messages") + # print(" - deleting all Wallet and SingleScan objects, and resize error messages") DataIssue.objects.filter(parser="scans").delete() DataIssue.objects.filter(parser="wallets").delete() DataIssue.objects.filter(parser="mogrify").delete() # These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet. + # But we ignore this list.. anything goes.. valids = [ ".top", ".txt", @@ -82,7 +83,7 @@ def load_all_scans(): # but not if they are /2010/2010#33/therion/ : the wallet is /2010#33/ not /therion/ print(" ", end="") scans_path = Path(settings.SCANS_ROOT) - seen = [] + seen = set() c = 0 wallets = {} for p in scans_path.rglob("*"): # this implicitly ignoires empty folders as it looks only for files @@ -95,9 +96,9 @@ def load_all_scans(): pass else: c += 1 - if c % 15 == 0: + if c % 60 == 0: print(".", end="") - if c % 750 == 0: + if c % 3000 == 0: print("\n ", end="") if p.parent.parent.parent.parent == scans_path: @@ -109,7 +110,7 @@ def load_all_scans(): fpath = p.parent walletname = p.parent.name - if walletname in wallets: + if walletname in wallets: # assumes all walletnames are unique wallet = wallets[walletname] else: print("", flush=True, end="") @@ -124,17 +125,21 @@ def load_all_scans(): singlescan.save() # only printing progress: - tag = p.parent - if len(walletname) > 4: - if walletname[4] == "#": - tag = p.parent.parent - - if tag not in seen: - print(f" {tag.name} ", end="") - if len(str(tag.name)) > 17: - print("\n ", end="") - seen.append(tag) - + tag = str(p.parent.name) # folder containing the scan file + if len(tag) > 4: + if tag[4] != "#": + relative_path = p.relative_to(scans_path, walk_up=False) + seen.add(str(relative_path.parent)) + wjson = 0 + print("\n ", end="") + seenlist = list(seen) + seenlist.sort() + + for tag in seenlist: + wjson += 1 + if wjson % 10 == 0: + print("\n ", end="") + print(f" {tag} ", end="") print(f"\n - found and loaded {c:,} acceptable scan files in {len(wallets):,} wallets") # but we also need to check if JSON exists, even if there are no uploaded scan files.