Cave import & reports fixes

parsers/caves.py

@@ -6,10 +6,10 @@ from pathlib import Path
 from django.conf import settings
 from django.db import transaction
 
-from troggle.settings import SURVEX_DATA, EXPOWEB
+from troggle.settings import SURVEX_DATA, EXPOWEB, CAVEDESCRIPTIONS, ENTRANCEDESCRIPTIONS
 from troggle.core.models.troggle import DataIssue
 from troggle.core.models.caves import Area, Cave, Entrance, CaveSlug, EntranceSlug, CaveAndEntrance
-'''Reads all the cave description data by parsing the xml files (stored as e.g. :EXPOWEB:/cave-data/1623-161.html )
+'''Reads all the cave description data by parsing the xml files (stored as e.g. :EXPOWEB:/cave_data/1623-161.html )
 and creating the various Cave, Entrance and necessary Area objects.
 
 BUT in Django 2.0 and later we cannot do any queries on data we have just entered 
@@ -23,7 +23,7 @@ todo='''- db Update does not work when a cave id is in the pending list but a pr
    and is being imported. It should work. But currently Django aborts and he file is not read in.
    
  - Cannot use Edit This Page for pendingcaves.txt_edit as Edit This Page is expecting an html file.
-   So we will need a separate file-editing capability just for this configuration file.
+   So we will need a separate file-editing capability just for this configuration file ?!
 '''
 entrances_xslug = {}
 caves_xslug = {}
@@ -140,7 +140,7 @@ def readcaves():
     # For those caves which do not have cave_data/1623-xxx.html XML files even though they exist and have surveys
     # should put this in a simple list which can be edited using 'Edit this file'
     pending = set()
-    fpending = Path(settings.CAVEDESCRIPTIONS, "pendingcaves.txt")
+    fpending = Path(CAVEDESCRIPTIONS, "pendingcaves.txt")
     if fpending.is_file():
         with open(fpending, "r") as fo:
             cids = fo.readlines()
@@ -153,6 +153,7 @@ def readcaves():
         Entrance.objects.all().delete()
         # Clear the cave data issues and the caves as we are reloading
         DataIssue.objects.filter(parser='caves').delete()
+        DataIssue.objects.filter(parser='caves ok').delete()
         DataIssue.objects.filter(parser='entrances').delete()
         
         area_1623 = Area.objects.update_or_create(short_name = "1623", parent = None)
@@ -176,9 +177,9 @@ def readcaves():
                 raise
 
     with transaction.atomic():
-        print(" - settings.CAVEDESCRIPTIONS: ", settings.CAVEDESCRIPTIONS)
+        print(" - settings.CAVEDESCRIPTIONS: ", CAVEDESCRIPTIONS)
         print(" - Reading Entrances from entrance descriptions xml files")
-        for filename in next(os.walk(settings.ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
+        for filename in next(os.walk(ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
             # if filename.endswith('.html'):
                 # if Path(filename).stem[5:] in pending:
                         # print(f'Skipping pending entrance dummy file <{filename}>')
@@ -187,7 +188,7 @@ def readcaves():
             readentrance(filename)
 
         print(" - Reading Caves from cave descriptions xml files")
-        for filename in next(os.walk(settings.CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
+        for filename in next(os.walk(CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
             if filename.endswith('.html'):
                 readcave(filename)
 
@@ -197,10 +198,10 @@ def readentrance(filename):
     global areas_xslug
 
     # Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
-    with open(os.path.join(settings.ENTRANCEDESCRIPTIONS, filename)) as f:
+    with open(os.path.join(ENTRANCEDESCRIPTIONS, filename)) as f:
         contents = f.read()
     context = filename
-    #print("Reading file ENTRANCE {} / {}".format(settings.ENTRANCEDESCRIPTIONS, filename))
+    #print("Reading file ENTRANCE {} / {}".format(ENTRANCEDESCRIPTIONS, filename))
     entrancecontentslist = getXML(contents, "entrance", maxItems = 1, context = context)
     if len(entrancecontentslist) != 1:
         message = f'! BAD ENTRANCE at "{filename}"'
@@ -293,7 +294,7 @@ def readcave(filename):
     global areas_xslug
     
     # Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
-    with open(os.path.join(settings.CAVEDESCRIPTIONS, filename)) as f:
+    with open(os.path.join(CAVEDESCRIPTIONS, filename)) as f:
         contents = f.read()
     context = filename
     cavecontentslist = getXML(contents, "cave", maxItems = 1, context = context)
@@ -419,9 +420,14 @@ def readcave(filename):
                     DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/')
                     print(message)
                     
-            if description_file[0]:
+
+            if description_file[0]: # if not an empty string
+                message = f' - {slug:12} complex description filename "{description_file[0]}" inside "{CAVEDESCRIPTIONS}/{filename}"'
+                DataIssue.objects.create(parser='caves ok', message=message, url=f'/cave/{slug}/edit/')
+                print(message)
+
                 if not (Path(EXPOWEB) / description_file[0]).is_file():
-                    message = f' ! {slug:12} description filename does not exist :{EXPOWEB}:"{description_file[0]}" in "{filename}"'
+                    message = f' ! {slug:12} description filename  "{EXPOWEB}/{description_file[0]}" does not refer to a real file'
                     DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/')
                     print(message)
                     #c.description_file="" # done only once, to clear out cruft.