From 74ce8894d2c0fa154f58b2dc2afb6e3332120284 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@gmail.com>
Date: Sun, 11 Aug 2024 15:19:59 +0300
Subject: [PATCH] wallets subfolders fiddling

---
 core/models/wallets.py | 32 +++++++++++++++++++++++++-------
 parsers/scans.py       | 13 +++++++------
 2 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/core/models/wallets.py b/core/models/wallets.py
index 7dacc7d..d94d048 100644
--- a/core/models/wallets.py
+++ b/core/models/wallets.py
@@ -147,19 +147,37 @@ class Wallet(models.Model):
         
         if hasattr(self, "JSONdata"):
             return self.JSONdata
+            
+        scans_path = Path(settings.SCANS_ROOT)
   
         # :drawings: walletjson/2022/2022#01/contents.json
         # fpath = /mnt/d/EXPO/expofiles/surveyscans/1999/1999#02
         # This does not work if there are subdirectories in the surveyscan folder
         fp = Path(self.fpath)
-        wname = fp.name
-        wyear = fp.parent.name
+        if fp.name in archaic_wallets:
+            return None
         
-        try:
-            y = int(wyear)
-        except:
-            message = f"! Subfolder detected?:\n {wyear=} (should be eg. '2023')\n {wname=} \n {self.fpath=}"
-            print(message)
+        if fp.parent.parent.parent.parent == scans_path:
+            print(f" ! ", end="")
+        if fp.parent.parent.parent.parent.parent == scans_path:
+            print(f" !! ", end="")
+            
+        if fp.parent.parent.parent.parent == scans_path:
+            subfolder = fp.parent.name
+            wname = fp.parent.parent.name
+            wyear = fp.parent.parent.parent.name
+            print(f"\n - Subfolder {subfolder} one deep, detected in {wname}")           
+        else:
+            wname = fp.name
+            wyear = fp.parent.name
+            try:
+                y = int(wyear)
+            except:
+                message = f"\n! 2 or more deep subfolder detected?:\n {fp.parent.name=}\n {fp.name=} \n {self.fpath=}"
+                print(message)
+                # this is a hack, work down from /surveyscans instead. To be fixed
+                wname = fp.parent.parent.parent.name
+                wyear = fp.parent.parent.parent.parent.name
         wurl = self.get_url()
 
         if len(wyear) != 4 or len(wname) !=6: 
diff --git a/parsers/scans.py b/parsers/scans.py
index ad9c110..e5138c3 100644
--- a/parsers/scans.py
+++ b/parsers/scans.py
@@ -85,15 +85,15 @@ def load_all_scans():
     seen = []
     c = 0
     wallets = {}
-    for p in scans_path.rglob("*"):
+    for p in scans_path.rglob("*"): # this implicitly ignoires empty folders as it looks only for files
         if p.is_file():
-            if p.suffix.lower() not in valids and p.name.lower() not in validnames:
+            # if p.suffix.lower() not in valids and p.name.lower() not in validnames:
+                # # we do not care how the files are called. Many have no suffices at all.
                 # print(f"'{p}'", end='\n')
-                pass
-            elif p.parent == scans_path:  # skip files directly in /surveyscans/
+                # pass
+            if p.parent == scans_path:  # skip files directly in /surveyscans/
                 pass
             else:
-
                 c += 1
                 if c % 15 == 0:
                     print(".", end="")
@@ -101,6 +101,7 @@ def load_all_scans():
                     print("\n   ", end="")
 
                 if p.parent.parent.parent.parent == scans_path:
+                    # the wallet has subfolders, which we are uninterested in at this point.
                     # print(f"too deep {p}", end='\n')
                     fpath = p.parent.parent
                     walletname = p.parent.parent.name  # wallet is one level higher
@@ -167,7 +168,7 @@ def load_all_scans():
                             )
     print(f"\n  - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets")
 
-    # Only the 1999 wallets have filenames which mean that the walletyear will be unset:
+    # Only the 1999 wallets have filepaths which would mean that the walletyear will be unset:
     wallets = Wallet.objects.filter(walletyear=None)
     for w in wallets:
         w.walletyear = datetime.date(1999, 1, 1)