Big refactor of cave lookuo & creation. Works.

_deploy/wsl/requirements-p11d4.txt

@@ -1,16 +1,21 @@
-asgiref==3.6.0
+asgiref==3.6
+beautifulsoup4=4.12
 black==23.1.0
+chardet=5.1
 click==8.1.3
 coverage==7.1.0
+deptry=0.12
 Django==4.2
 docutils==0.19
-isort==5.12.0
+isort==5.12
 mypy-extensions==1.0.0
 packaging==23.0
-pathspec==0.11.0
+pathspec==0.11
 Pillow==9.4.0
-platformdirs==3.0.0
+platformdirs==3.0
 pytz==2022.7
 ruff==0.0.245
-sqlparse==0.4.3
+setuptools=67.7
-Unidecode==1.3.6
+soupsieve=2.5
+sqlparse==0.4
+Unidecode==1.3

core/models/caves.py

@@ -546,6 +546,7 @@ def GetCaveLookup():
 
     # These might alse create more duplicate entries
     # Yes, this should be set in, and imported from, settings.py
+    # yes we should move to always using the 1623- prefix too.
     aliases = [
         ("1987-02", "267"),
         ("1990-01", "171"),

parsers/caves.py

@@ -133,6 +133,7 @@ def get_area(areanum):
 def create_new_cave(svxpath):
     """This is called only when a new survex file is edited online which has a path on the
     :loser: repo which is not recognised as a known cave.
+    ALSO called by survex parser when it finds a cave it doesn't recognise
     """
     # e.g. svxpath = "caves-1623/666/antig"
     print(f"Create new cave at {svxpath}")
@@ -171,7 +172,8 @@ def create_new_cave(svxpath):
         print(message)
         raise    
     
-    # we know what the survex file is, we don't need to use the guess
+    # we know what the survex file is, we don't need to use the guess.
+    # But this sets the survex file on  he cave from the first one we find, not necessarily the best survex file for this cave
     cave.survex_file=survex_file
     cave.save()
     return cave

parsers/survex.py

@@ -1165,33 +1165,34 @@ class LoadingSurvex:
 
     def IdentifyCave(self, cavepath, svxid, depth):
         """Given a file path for a survex file, e.g. /1626/107/107.svx, or a survex-block path,
-        return the cave object
+        return the cave object       
-        
-        REWRITE ALL THIS and make a methoid on the class
         """
-        caveslist = GetCaveLookup()
+        path = cavepath.lower()
-        if cavepath.lower() in caveslist: # will only work after we load in full paths as indexes, see below
+        if path in self.caveslist: # primed with GCaveLookup
-            return caveslist[cavepath.lower()]
+            return self.caveslist[path]
         #     rx_cave = re.compile(r"(?i)caves-(\d\d\d\d)/([-\d\w]+|\d\d\d\d-?\w+-\d+)")
-        path_match = self.rx_cave.search(cavepath) # use as Class method. 
+        path_match = self.rx_cave.search(cavepath) 
         if path_match:
             sluggy = f"{path_match.group(1)}-{path_match.group(2)}"
-            # guesses = [sluggy.lower(), path_match.group(2).lower()] # this looks for JUST "107" and ignores 1626..
             guesses = [sluggy.lower()] # full 1626-107 search, don;t use short-forms
             for g in guesses:
-                if g in caveslist:
+                if g in self.caveslist:
-                    caveslist[cavepath] = caveslist[g] # set "caves-1626/107/107.svx" as index to cave 1626-107
+                    self.caveslist[cavepath] = self.caveslist[g] # set "caves-1626/107/107" as index to cave 1626-107
-                    return caveslist[g]
+                    return self.caveslist[g]
-            print(f"    ! Failed to find cave for {cavepath.lower()}", file=sys.stderr)
+            cave = create_new_cave(cavepath) # uses the pending stuff to create pending cave descriptions
+            self.caveslist[cavepath] = cave
+            message = f"    ! MAKING cave for {cavepath=} {svxid=}"
+            stash_data_issue(parser="survex", message=message, url=None, sb=(svxid))
+            if not cavepath.startswith("caves-1624") or cavepath.startswith("caves-1626"):
+                print(message, file=sys.stderr)
         else:
-            # not a cave, but that is fine.
+            # isn't all this pointless...??
-            if self.is_it_already_pending(cavepath, svxid, depth):
+            if self.is_it_already_pending(cavepath, svxid, depth): # but pending will already have been created as Cave objects
                 pass
             else:
                 # It is too late to add it to the pending caves list here, they were already 
                 # processed in parsers/caves.py So we have to do a bespoke creation.
                 cave = create_new_cave(svxid)
-                
                 message = f" ! Warning: cave identifier '{caveid}'or {id} (guessed from file path)  is not a known cave.  Need to add to expoweb/cave_data/pendingcaves.txt ?  In '{includelabel}.svx' at depth:[{len(depth)}]."
                 print("\n" + message)
                 print("\n" + message, file=sys.stderr)
@@ -1238,6 +1239,8 @@ class LoadingSurvex:
         if id in self.pending:
             print(f"! ALREADY PENDING id {id}", file=sys.stderr)
             return True
+            
+        return False
 
     def LoadSurvexFile(self, svxid):
         """Creates SurvexFile in the database, and SurvexDirectory if needed