Now creating ARGE caves properly

parsers/caves.py

@@ -178,13 +178,67 @@ def create_new_cave(svxpath, msg=None):
     cave.save()
     return cave
 
-def do_pending_cave(k, caveid, url, area, msg=None):
+def do_ARGE_cave(slug, caveid, url, area, svxid):
+    """Only called by survex parser. 
+    Creates a new Cave object, but with abbreviated data as the survex file (from ARGE) is all we have.
+    We already know the survex file.
+    We already know that it doesn't exist.
+    """
+    
+    default_note = "This is an ARGE cave where we only have the survex file and no other information"
+
+    urltest = Cave.objects.filter(url=url)
+    if urltest:
+        message = f" ! Cave {urltest[0]} already exists with this url {url}. Can't create new ARGE cave {slug}"
+        DataIssue.objects.create(parser="caves", message=message, url=url)
+        print(message)
+        return urltest[0]
+        
+    numtest = Cave.objects.filter(unofficial_number=caveid.upper())
+    if numtest:
+        message = f" ! Cave {numtest[0]} already exists with this unofficial_number {caveid.upper()}. Can't create new ARGE cave {slug}"
+        DataIssue.objects.create(parser="caves", message=message, url=url)
+        print(message)
+        return numtest[0]
+
+    cave = Cave(
+        unofficial_number=caveid.upper(),
+        underground_description="ARGE cave.",
+        survex_file= f"{svxid}.svx",
+        url=url,
+        notes=default_note,
+    )
+    if cave:
+        cave.save()  # must save to have id before foreign keys work. This is also a ManyToMany key.
+        # cave.area.add(area) 
+        # cave.save() # crashes entire transaction with foreign key error.
+        # The 'caves' list page uses the area__short_name to select for the area, so these ARGE caves do not appear.
+        
+        # message = f" ! {slug:18} ARGE cave  url: {url} "
+        # DataIssue.objects.create(parser="caves", message=message, url=url)
+        # print(message)
+
+        try:  # Now create a cave slug ID
+            CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False)
+        except:
+            message = f" ! {slug:11s}  ARGE CaveSLUG create failure {caveid=} {url=} {area=} {svxid=}"
+            DataIssue.objects.create(parser="caves", message=message)
+            print(message)
+    else:
+        message = f" ! {slug:11s}  ARGE cave create failure {caveid=} {url=} {area=} {svxid=}"
+        DataIssue.objects.create(parser="caves", message=message)
+        print(message)
+
+    return cave
+    
+def do_pending_cave(slug, caveid, url, area, msg=None):
     """
     default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists
     in expoweb/cave_data/1623-"k".html
 
-    Note that at this point in importing the data we have not yet seen the survex files, so we can't
+    Note that at this point (parsing caves) in importing the data we have not yet seen the survex files, so we can't
     look inside the relevant survex file to find the year and so we con't provide helpful links.
+    This also gets called when parsing survex files, when we do have this info.
     """
 
     def get_survex_file(k):
@@ -222,12 +276,10 @@ def do_pending_cave(k, caveid, url, area, msg=None):
             # print(message)
         return survex_file
 
-    slug = k
-
     g = GetCaveLookup()
     with transaction.atomic():
         if slug in g:
-            message = f" ! {k:18} cave listed in pendingcaves.txt already exists. - {msg}"
+            message = f" ! {slug:18} cave listed in pendingcaves.txt already exists. - {msg}"
             DataIssue.objects.create(parser="caves", message=message, url=url)
             print(message)
             return
@@ -274,7 +326,7 @@ def do_pending_cave(k, caveid, url, area, msg=None):
             print(message)
             return urltest[0]
 
-        survex_file = get_survex_file(k)
+        survex_file = get_survex_file(slug)
 
         cave = Cave(
             unofficial_number=caveid.upper(),
@@ -287,18 +339,18 @@ def do_pending_cave(k, caveid, url, area, msg=None):
             cave.save()  # must save to have id before foreign keys work. This is also a ManyToMany key.
             cave.area.add(area)
             cave.save()
-            message = f" ! {k:18} Pending cave write-up url: {url} - {msg}"
+            message = f" ! {slug:18} Pending cave write-up url: {url} - {msg}"
             DataIssue.objects.create(parser="caves", message=message, url=url)
             print(message)
 
             try:  # Now create a cave slug ID
                 CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False)
             except:
-                message = f" ! {k:11s}  PENDING CaveSLUG {slug} create failure - {msg}"
+                message = f" ! {slug:11s}  PENDING CaveSLUG {slug} create failure - {msg}"
                 DataIssue.objects.create(parser="caves", message=message)
                 print(message)
         else:
-            message = f" ! {k:11s}  PENDING cave create failure - {msg}"
+            message = f" ! {slug:11s}  PENDING cave create failure - {msg}"
             DataIssue.objects.create(parser="caves", message=message)
             print(message)
 

parsers/survex.py

@@ -14,7 +14,7 @@ from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
 from troggle.core.models.wallets import Wallet
 from troggle.core.models.troggle import DataIssue, Expedition
 from troggle.core.utils import chaosmonkey, get_process_memory
-from troggle.parsers.caves import create_new_cave
+from troggle.parsers.caves import create_new_cave, do_ARGE_cave
 from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner
 
 """Imports the tree of survex files following from a defined root .svx file
@@ -1175,10 +1175,10 @@ class LoadingSurvex:
             for key in self.caveslist:
                 cave = self.caveslist[key]
                 if type(cave) != Cave:
-                    print(f"BAD CAVE TYPE '{key}'  --  {type(cave)}'{cave}'")
+                    print(f"BAD CAVE TYPE '{type(cave)}'{cave}' -- {key}'")
             for key in self.caveslist:
                 cave = self.caveslist[key]
-                print(f"{key}  --  Cave<{cave}>")
+                print(f"Cave<{cave}> -- {key}")
                 
         for i in self.ignoreprefix:
             if cavepath.lower().startswith(i):
@@ -1192,19 +1192,23 @@ class LoadingSurvex:
         #     rx_cave = re.compile(r"(?i)caves-(\d\d\d\d)/([-\d\w]+|\d\d\d\d-?\w+-\d+)")
         path_match = self.rx_cave.search(cavepath) 
         if path_match:
-            sluggy = f"{path_match.group(1)}-{path_match.group(2)}"
+            area = path_match.group(1)
+            caveid = path_match.group(2)
+            sluggy = f"{area}-{caveid}"
             seek = [sluggy, sluggy.replace("1623-","")] # to catch '2023-kt-02' etc
             for s in seek:
                 if s in self.caveslist:
                     self.caveslist[cavepath] = self.caveslist[s] # set "caves-1626/107/107" as index to cave 1626-107
                     return self.caveslist[s]
             
+            if (cavepath.startswith("caves-1624") or cavepath.startswith("caves-1626")):
+                url = f"/survexfile/{svxid}.svx"
+                return do_ARGE_cave(sluggy, caveid, url, area, svxid)
+                
             cave = create_new_cave(cavepath, f"Make cave found in survex file {svxid}") # uses the pending code to create pending cave descriptions
             self.caveslist[cavepath] = cave
-            message = f"\n    ! MAKING cave {sluggy} for {cavepath=} {svxid=} (not reporting this for 1624 or 1626)"
+            message = f"\n    ! MAKING cave {sluggy} for {cavepath=} {svxid=}"
             # stash_data_issue(parser="survex", message=message, url="/survexfile/{svxid}.svx", sb=(svxid))
-            if not (cavepath.startswith("caves-1624") or cavepath.startswith("caves-1626")):
-                print(message, file=sys.stderr)
             return cave
         else:
             path_match = rx_svxcollection.search(svxid) 
@@ -1220,39 +1224,6 @@ class LoadingSurvex:
                 stash_data_issue(parser="survex", message=message, url="{svxid}.svx", sb=(svxid))
                 return None
 
-    def is_it_already_pending(self, headpath, includelabel, depth):
-        """Ignore surface, kataser and gpx *include survex files"""
-        if not self.pending:
-            self.pending = set()
-            fpending = Path(settings.CAVEDESCRIPTIONS, "pendingcaves.txt")
-            if fpending.is_file():
-                with open(fpending, "r") as fo:
-                    cids = fo.readlines()
-                for cid in cids:
-                    id = cid.strip().rstrip("\n").upper()
-                    if cid.startswith("162"):
-                        self.pending.add(id)
-                    else:
-                        self.pending.add("1623-" + id)
-
-        if headpath in self.ignorenoncave:
-            message = f" - {headpath} is <ignorenoncave> (while creating '{includelabel}' sfile & sdirectory)"
-            # print("\n"+message)
-            # print("\n"+message,file=sys.stderr)
-            return True
-
-        caveid = f"{headpath[6:10]}-{headpath[11:]}".upper()
-        if caveid in self.pending:
-            # Yes we didn't find this cave, but we know it is a pending one. So not an error.
-            print(f'! ALREADY PENDING caveid {caveid}',file=sys.stderr)
-            return True
-        id = caveid[5:]
-        if id in self.pending:
-            print(f"! ALREADY PENDING id {id}", file=sys.stderr)
-            return True
-            
-        return False
-
     def LoadSurvexFile(self, svxid):
         """Creates SurvexFile in the database, and SurvexDirectory if needed
         Creates a new current survexfile and valid .survexdirectory