removing technical debt, replace convoluted code

core/utils.py

@@ -74,6 +74,8 @@ def chaosmonkey(n):
 def GetListDir(sdir):
     '''handles url or file, so we can refer to a set of scans (not drawings) on another server
     returns a list of f (file), ff (file full path), is_dir (bool)
+    
+    REPLACE all use of this with Path.rglob() !
     '''
     res = [ ]
     if type(sdir) is str and sdir[:7] == "http://":

parsers/scans.py

@@ -66,16 +66,18 @@ def LoadListScansFile(wallet):
             if c>=10:
                 print(".", end='')
                 c = 0
-        
+    
 def load_all_scans():
     '''This iterates through the scans directories (either here or on the remote server)
     and builds up the models we can access later.
     It does NOT read or validate anything in the JSON data attached to each wallet. Those checks
     are done at runtime, when a wallet is accessed, not at import time.
     
-    NOTE that parsers/survex.py does NOT create a wallet if it finds an unrecognised *REF wallet.
+    Replace GetListDir with a more modern Path.iter idiom
-    Instead it reports an error in DataIssues. But it does make a link in the db between the
+    path = Path("scans")
-    existing wallet (probably no JSON, just a folder containing scans) and the survex file.
+    for p in path.rglob("*"):
+        print(p.name)
+    
     '''
     print(' - Loading Survey Scans')
 
@@ -83,49 +85,68 @@ def load_all_scans():
     Wallet.objects.all().delete()
     print(' - deleting all Wallet and SingleScan objects')
     DataIssue.objects.filter(parser='scans').delete()
-
+    
-    # first do the smkhs (large kh survey scans) directory
+    valids = [".top",".txt",".tif",".png",".jpg",".jpeg",".pdf",".svg",".gif",".xvi",
-    # this seems to be never used ?!
+        ".json",".autosave",".sxd",".svx",".th",".th2",".tdr",".sql",".zip",".dxf",".3d",
-    #We should load all the scans, even for nonstandard names. 
+        ".ods",".csv",".xcf",".xml"]
-    manywallets_smkhs = Wallet(fpath=os.path.join(settings.SCANS_ROOT, "../surveys/smkhs"), walletname="smkhs") 
+    validnames = ["thconfig","manifest"]
-    print("smkhs", end=' ')
-    if os.path.isdir(manywallets_smkhs.fpath):
-        manywallets_smkhs.save()
-        LoadListScansFile(manywallets_smkhs)
-    else:
-        print("smkhs NOT LOADED", end=' ')
 
     # iterate into the surveyscans directory
+    # Not all folders with files in them are wallets.
+    # they are if they   are /2010/2010#33 
+    #     or /1996-1999NotKHbook/
+    #     but not if they are /2010/1010#33/therion  or /1998/
     print(' - ', end=' ')
-    for walletname, fpath, fisdir in GetListDir(settings.SCANS_ROOT):
+    scans_path = Path(settings.SCANS_ROOT) 
+    seen = []
+    for p in scans_path.rglob('*'):
+        if p.is_file():
+            if p.suffix.lower() not in valids and p.name.lower() not in validnames:
+                # print(f"'{p}'", end='\n')
+                pass
+            elif p.parent == scans_path: # skip files directly in /surveyscans/
+                pass
+            else:
+                if p.parent.parent.parent.parent == scans_path:
+                    # print(f"too deep {p}", end='\n')
+                    fpath = p.parent.parent
+                    walletname = p.parent.parent.name # wallet is one level higher
+                else: 
+                    fpath = p.parent
+                    walletname = p.parent.name
+                    
+                # UNFINISHED
+                tag = p.parent
+                if len(walletname)>4:
+                    if walletname[4] == "#":
+                        tag = p.parent.parent
+                    
+                if tag not in seen:
+                    print(f"{tag.name}", end=' ')
+                    seen.append(tag)
+                     #wallet = Wallet(fpath=fpath, walletname=walletname)
+                    
+    
+    print('\n UNFINISHED  \n\n--- ')
+    for topfolder, fpath, fisdir in GetListDir(settings.SCANS_ROOT):
         if not fisdir:
             continue
 
-        for walletname, fpath, fisdir in GetListDir(fpath):
+        # do the year folders
-            if fisdir:
+        if re.match(r"\d\d\d\d$", topfolder):
-                wallet = Wallet(fpath=fpath, walletname=walletname)
+            print(f"{topfolder}", end=' ')
-                # this is where we should record the year explicitly
+            for walletname, fpath, fisdir in GetListDir(fpath):
-                # line 347 of view/uploads.py and needs refactoring for loading contentsjson
+                if fisdir:
-                CheckEmptyDate(wallet)
+                    wallet = Wallet(fpath=fpath, walletname=walletname)
-                CheckEmptyPeople(wallet)
+                    # this is where we should record the year explicitly
-                wallet.save()
+                    # line 347 of view/uploads.py and needs refactoring for loading contentsjson
-                LoadListScansFile(wallet)
+                    CheckEmptyDate(wallet)
-      
+                    CheckEmptyPeople(wallet)
-        # # do the year folders
+                    wallet.save()
-        # if re.match(r"\d\d\d\d$", walletname):
+                    LoadListScansFile(wallet)
-            # print(f"{walletname}", end=' ')
+        else:
-            # for walletname, fpath, fisdir in GetListDir(fpath):
+            # but We *should* load all the scans, even for nonstandard names. 
-                # if fisdir:
+            print(f'\n - IGNORE {walletname} - {fpath}')
-                    # wallet = Wallet(fpath=fpath, walletname=walletname)
-                    # # this is where we should record the year explicitly
-                    # # line 347 of view/uploads.py and needs refactoring for loading contentsjson
-                    # CheckEmptyDate(wallet)
-                    # CheckEmptyPeople(wallet)
-                    # wallet.save()
-                    # LoadListScansFile(wallet)
-        # else:
-            # # but We *should* load all the scans, even for nonstandard names. 
-            # print(f'\n - IGNORE {walletname} - {fpath}')
  
     # but we also need to check if JSON exists, even if there are no uploaded scan files
     contents_path = Path(settings.DRAWINGS_DATA, "walletjson")