fixing pending caves system to be cleaner

parsers/caves.py

@@ -84,6 +84,15 @@ def do_pending_cave(k, url, area):
     in expoweb/cave_data/1623-"k".html 
     '''
     slug = k
+    
+    g = GetCaveLookup()
+    if slug in g:
+        message = f" ! {k} cave listed in pendingcaves.txt already exists."
+        DataIssue.objects.create(parser='caves', message=message, url=url)
+        print(message)
+        return
+
+    
 
     default_note = f"_Survex file found in loser repo but no description in expoweb <br><br><br>\n" 
     default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then " 
@@ -111,7 +120,7 @@ def do_pending_cave(k, url, area):
     cave = Cave(
             unofficial_number = k, 
             underground_description = "Pending cave write-up - creating as empty object. No XML file available yet.",
-            survex_file = f"caves-{area.short_name}/{k}/{k}.svx",
+            survex_file = f"caves-{area.short_name}/{k[5:]}/{k[5:]}.svx",
             url = url,
             notes = default_note)
     if cave:
@@ -458,27 +467,6 @@ def readcaves():
         print(" - Saving Area 1626")
         area_1626.save()
 
-        print (" - Setting pending caves")
-        # Do this first, so that these empty entries are overwritten as they get properly created.
-
-        for k in pending:
-            
-            area = area_1623
-            areanum = k[0:4]
-            url = areanum + "/" +  k[5:] # Note we are not appending the .htm as we are modern folks now.
-            if areanum == "1623":
-                area = area_1623
-            if areanum == "1624":
-                area = area_1624
-            if areanum == "1626":
-                area = area_1626
-            try:    
-                do_pending_cave(k[5:], url, area)
-            except:
-                message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum}"
-                DataIssue.objects.create(parser='caves', message=message)
-                print(message)
-                raise
 
     with transaction.atomic():
         print(" - settings.CAVEDESCRIPTIONS: ", CAVEDESCRIPTIONS)
@@ -498,4 +486,27 @@ def readcaves():
 
     print (" - Setting up all the variously useful alias names")
     mycavelookup = GetCaveLookup()
+    
+    print (" - Setting pending caves")
+    # Do this last, so we can detect if they are created and no longer 'pending'
+
+    for k in pending:
+        
+        area = area_1623
+        areanum = k[0:4]
+        url = areanum + "/" +  k[5:] # Note we are not appending the .htm as we are modern folks now.
+        if areanum == "1623":
+            area = area_1623
+        if areanum == "1624":
+            area = area_1624
+        if areanum == "1626":
+            area = area_1626
+        try:    
+            do_pending_cave(k, url, area)
+        except:
+            message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum}"
+            DataIssue.objects.create(parser='caves', message=message)
+            print(message)
+            raise
+
 

parsers/survex.py

@@ -184,6 +184,7 @@ class LoadingSurvex():
     currentcave = None
     caverndate = None
     currentpersonexped = []
+    pending = []
 
     def __init__(self):
         self.caveslist = GetCaveLookup()
@@ -721,6 +722,15 @@ class LoadingSurvex():
     def ReportNonCaveIncludes(self, headpath, includelabel, depth):
         """Ignore surface, kataser and gpx *include survex files
         """
+        if not self.pending:
+            self.pending = set()
+            fpending = Path(settings.CAVEDESCRIPTIONS, "pendingcaves.txt")
+            if fpending.is_file():
+                with open(fpending, "r") as fo:
+                    cids = fo.readlines()
+                for cid in cids:
+                    self.pending.add(cid.rstrip('\n').upper())
+
         if headpath in self.ignorenoncave:
             message = f" - {headpath} is <ignorenoncave> (while creating '{includelabel}' sfile & sdirectory)"
             #print("\n"+message)
@@ -732,20 +742,26 @@ class LoadingSurvex():
                 # print("\n"+message)
                 # print("\n"+message,file=sys.stderr)
                 return
-        message = f" ! Error: not a cave nor ignorable. headpath:'{headpath}' while creating '{includelabel=}' at depth:[{len(depth)}].  ignore prefix list:'{self.ignoreprefix}'"
-        # getting this triggered for gpx/2018 (cavern error) but not for gpx/2017 (no content).
+        caveid = f'{headpath[6:10]}-{headpath[11:]}'.upper()
+        if caveid in self.pending:
+           # Yes we didn't find this cave, but we know it is a pending one. So not an error.
+           # print(f'! ALREADY PENDING {caveid}',file=sys.stderr)
+           return
+            
+        message = f" ! Error: not a cave nor ignorable. headpath:'{headpath}' while parsing '{includelabel=}.svx' at depth:[{len(depth)}].  ignore prefix list:'{self.ignoreprefix}'"
         print("\n"+message)
         print("\n"+message,file=sys.stderr)
         DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(headpath))
         print(f' # datastack in  LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr)
         for dict in self.datastack:
-            print(f'{dict["type"].upper()}   ', end="",file=sys.stderr)
+            print(f'<{dict["type"].upper()}   >', end="",file=sys.stderr)
         
 
     def LoadSurvexFile(self, svxid):
         """Creates SurvexFile in the database, and SurvexDirectory if needed
         with links to 'cave'
         Creates a new current survexfile and valid .survexdirectory
+        Inspects the parent folder of the survexfile and uses that to decide if this is a cave we know
         The survexblock passed-in is not necessarily the parent. FIX THIS.
         """
         if debugprint: