debugging Cave page links..

core/forms.py

@@ -25,7 +25,7 @@ todo = """
 
 class CaveForm(ModelForm):
     """Only those fields for which we want to override defaults are listed here
-    the other fields are present on the form, but use the default presentation style
+    the other fields of the class Cave are present on the form, but use the default presentation style
     """
 
     official_name = forms.CharField(required=False, widget=forms.TextInput(attrs={"size": "45"}))
@@ -74,19 +74,11 @@ class CaveForm(ModelForm):
         model = Cave
         exclude = ("filename",)
        
-    field_order = ['area', 'unofficial_number', 'kataster_number', 'official_name', 'underground_description', 'explorers', 'equipment', 'survey', 'kataster_status', 'underground_centre_line', 'notes', 'references', 'description_file', 'survex_file', 'url', 'length', 'depth', 'extent']
+    field_order = ['unofficial_number', 'kataster_number', 'official_name', 'underground_description', 'explorers', 'equipment', 'survey', 'kataster_status', 'kataster_code','underground_centre_line', 'notes', 'references', 'description_file', 'survex_file', 'areacode', 'url', 'length', 'depth', 'extent']
-    
+   
-    def get_area(self):
-        for a in self.cleaned_data["area"]:
-            if a.kat_area():
-                return a.kat_area()
-    
     def clean_cave_slug(self):
         if self.cleaned_data["cave_slug"] == "":
-            myArea = ""
+            myArea = self.cleaned_data["areacode"]
-            for a in self.cleaned_data["area"]:
-                if a.kat_area():
-                    myArea = a.kat_area()
             if self.data["kataster_number"]:
                 cave_slug = f"{myArea}-{self.cleaned_data['kataster_number']}"
             else:
@@ -112,8 +104,8 @@ class CaveForm(ModelForm):
             )
         #        if self.cleaned_data.get("kataster_number") != "" and self.cleaned_data.get("official_name") == "":
         #            self._errors["official_name"] = self.error_class(["This field is required when there is a kataster number."])
-        if cleaned_data.get("area") == []:
+        if cleaned_data.get("url") == []:
-            self._errors["area"] = self.error_class(["This field is required."])
+            self._errors["url"] = self.error_class(["This field is required."])
         if cleaned_data.get("url") and cleaned_data.get("url").startswith("/"):
             self._errors["url"] = self.error_class(["This field cannot start with a /."])
         return cleaned_data

core/models/caves.py

@@ -65,7 +65,7 @@ class CaveAndEntrance(models.Model):
 class Cave(TroggleModel):
     # too much here perhaps,
     areacode = models.CharField(max_length=4, blank=True, null=True) # could use models.IntegerChoices
-    subarea = models.CharField(max_length=10, blank=True, null=True) # 9, 8c etc.
+    subarea = models.CharField(max_length=25, blank=True, null=True) # 9, 8c etc.
     depth = models.CharField(max_length=100, blank=True, null=True)
     description_file = models.CharField(max_length=200, blank=True, null=True)
     entrances = models.ManyToManyField("Entrance", through="CaveAndEntrance")
@@ -89,34 +89,12 @@ class Cave(TroggleModel):
 
     # class Meta:
     #    unique_together = (("area", "kataster_number"), ("area", "unofficial_number"))
-    # FIXME Kataster Areas and CUCC defined sub areas need seperating
 
     # href    = models.CharField(max_length=100)
 
     class Meta:
         ordering = ("kataster_code", "unofficial_number")
 
-    def hassurvey(self):
-        """This is almost certainly a fossil - needs checking...
-        """
-        if not self.underground_centre_line:
-            return "No"
-        if (
-            self.survey.find("<img") > -1
-            or self.survey.find("<a") > -1
-            or self.survey.find("<IMG") > -1
-            or self.survey.find("<A") > -1
-        ):
-            return "Yes"
-        return "Missing"
-
-    def hassurveydata(self):
-        if not self.underground_centre_line:
-            return "No"
-        if self.survex_filcavee:
-            return "Yes"
-        return "Missing"
-
     def slug(self):
         primarySlugs = self.caveslug_set.filter(primary=True)
         if primarySlugs:
@@ -135,8 +113,8 @@ class Cave(TroggleModel):
         else:
             return self.unofficial_number
 
-    def reference(self):
+    def reference(self): # tidy this up, no longer used?
-        return f"{self.kat_area()}-{self.number()}"
+        return f"{self.areacode}-{self.number()}"
 
     def get_absolute_url(self):
         if self.kataster_number:
@@ -148,7 +126,10 @@ class Cave(TroggleModel):
         return Path(settings.URL_ROOT) / self.url  # not good Django style.. NEEDS actual URL
 
     def url_parent(self):
-        return self.url.rsplit("/", 1)[0]
+        if self.url:
+            return self.url.rsplit("/", 1)[0]
+        else:
+            return "NO cave.url"
 
     def __str__(self, sep=": "):
         return str(self.slug())
@@ -174,9 +155,6 @@ class Cave(TroggleModel):
         qms = self.get_open_QMs() | self.get_ticked_QMs() # set union operation
         return qms  # a QuerySet
         
-    def kat_area(self):
-        return self.areacode
-
     def entrances(self):
         return CaveAndEntrance.objects.filter(cave=self)
 

core/views/caves.py

@@ -353,7 +353,7 @@ def cavepage(request, karea=None, subpath=None, slug=None):
 def edit_cave(request, path="", slug=None):
     """This is the form that edits all the cave data and writes out an XML file in the :expoweb: repo folder
     The format for the file being saved is in templates/dataformat/cave.xml
-    Warning. This uses Django deep magic.
+    Warning. This uses Django deep magic in the CaveForm processing.
 
     It saves the data into into the database and into the html file, which it then commits to git.
     """
@@ -372,9 +372,9 @@ def edit_cave(request, path="", slug=None):
             # print(f'! POST is valid. {cave}')
             cave = form.save(commit=False)
             if not cave.filename:
-                cave.filename = form.get_area() + "-" + cave.number() + ".html"
+                cave.filename = cave.areacode + "-" + cave.number() + ".html"
             if not cave.url:
-                cave.url = form.get_area() + "/" + cave.number() + ".html"
+                cave.url = cave.areacode + "/" + cave.number() 
             cave.save()
             form.save_m2m()
             if slug is None:

core/views/survex.py

@@ -557,7 +557,8 @@ def get_survexareapath(area):
 # direct local non-database browsing through the svx file repositories
 # every time the page is viewed! Should cache this.
 def survexcaveslist(request):
-    """This reads the entire list of caves in the Loser repo directory and produces a complete report.
+    """NEEDS REWRITING COMPLETELY
+    This reads the entire list of caves in the Loser repo directory and produces a complete report.
     It can find caves which have not yet been properly registered in the system by Databasereset.py because
     someone may have uploaded the survex files with git without doing the rest of the integration process.
     
@@ -566,6 +567,8 @@ def survexcaveslist(request):
     what is already in the db, and just construct:  onefilecaves,  multifilecaves, subdircaves.
     
     It uses very impenetrable code in identifycavedircontents()
+    
+    It is MISSING some caves, e.g. "LA34" should appear twice, as 1623 and 1626.
     """
     # TO DO - filter out the non-public caves  from display UNLESS LOGGED IN
     # This is very impenetrable code, original from Aaron Curtis I think.
@@ -692,11 +695,11 @@ def check_cave_registered(areacode, survex_cave):
     A serious bodge.
     """
     try:
-        cave = Cave.objects.get(kataster_number=survex_cave)
+        cave = Cave.objects.get(areacode=areacode, kataster_number=survex_cave)
         return str(cave)
 
     except MultipleObjectsReturned:
-        caves = Cave.objects.filter(kataster_number=survex_cave)
+        caves = Cave.objects.filter(areacode=areacode, kataster_number=survex_cave)
         for c in caves:
             if str(c) == areacode + "-" + survex_cave:
                 return str(c)  # just get the first that matches
@@ -706,11 +709,14 @@ def check_cave_registered(areacode, survex_cave):
         pass
 
     try:
-        cave = Cave.objects.get(unofficial_number=survex_cave)  # should be unique!
+        cave = Cave.objects.get(areacode=areacode, unofficial_number=survex_cave)  # Not unique, e.g. LA34
-        if cave.kataster_number:
+        return str(cave)
-            return str(cave)
+    except MultipleObjectsReturned:
-        else:
+        caves = Cave.objects.filter(areacode=areacode,unofficial_number=survex_cave)
-            return None
+        for c in caves:
+            if str(c) == areacode + "-" + survex_cave:
+                return str(c)  # just get the first that matches
+        return None  # many returned but none in correct area
     except ObjectDoesNotExist:
         pass
 

parsers/caves.py

@@ -27,6 +27,9 @@ So is the first thing that creates tables.
 """
 
 todo = """  
+- When reading cave data, to start off wit we do not know the cave id (slug) so we can't give a useful url in
+  the error message, but we do have the filename. Systematize this, and the same thing with reading entrance files.
+  
 - Cannot use Edit This Page for pendingcaves.txt_edit as Edit This Page is expecting an html file.
    So we will need a separate file-editing capability just for this configuration file ?!
    
@@ -35,15 +38,15 @@ todo = """
 - rewrite archaic regex
   re.findall("<%(itemname)s>(.*?)</%(itemname)s>" % {"itemname": itemname}, text, re.S)
   in modern form and pre-compile it. 
-
-- Semi-automagically import all the 1627- pending caves and create HTML files for them to be
-  edited individually. (These are caves we only know about because we have German survex files.)
    
 - crashes on MariaDB in databasereset.py on server when deleting Caves and complains Area needs a 
   non null parent, But this is not true.   The only solution we have found is to let it crash, then 
   stop and restart MariaDB (requires a logon able to sudo)   and then restart the databasereset.py 
-  again. (status as of July 2022)
+  again. (status as of July 2022). May not happen now that class Area is removed (Sept.2023).
 """
+AREACODES = {"1623", "1624", "1626", "1627"} # NB set not dict
+ARGEAREAS = {"1626", "1627"}
+
 entrances_xslug = {}
 caves_xslug = {}
 areas_xslug = {}
@@ -105,7 +108,7 @@ def create_new_cave(svxpath, msg=None):
     :loser: repo which is not recognised as a known cave.
     ALSO called by survex parser when it finds a cave it doesn't recognise
     """
-    # e.g. svxpath = "caves-1623/666/antig"
+    # e.g. svxpath = "caves-1623/666/beast" .svx
     print(f"Create new cave at {svxpath} - {msg}")
     #
     survex_file = svxpath + ".svx"
@@ -117,16 +120,13 @@ def create_new_cave(svxpath, msg=None):
     if a[0:3] == "162":
         areacode = a[0:4]
         url = f"{areacode}/{caveid}.html"  # Note we are appending the .html as we are believe in backwards compatability.
-        #url = f"{areacode}/{a[5:]}.html"  # This is original code, but a above is only defined as being 4 characters long, so it did not make sense and produced non unique urls
     else: 
         print(f"WARNING: parsers/caves/create_new_cave called with svxpath '{svxpath}'.  Surely it should start 'caves-162*'? {msg}")
         areacode = "1623"
         url = f"1623/{caveid}.html"
-        #url = f"1623/{k}.html" # This is original code, but a above is only defined as being 4 characters long, so it did not make sense and produced non unique urls
+         
-        
     k = f"{areacode}-{caveid}"
 
-
     caves = Cave.objects.filter(unofficial_number=caveid, areacode =areacode) 
     if caves:
         message = f" ! Already exists, caveid:{k} in areacode {areacode} {caves} - {msg}"
@@ -149,14 +149,15 @@ def create_new_cave(svxpath, msg=None):
     cave.save()
     return cave
 
-def do_ARGE_cave(slug, caveid, url, areacode, svxid):
+def do_ARGE_cave(slug, caveid, svxurl, areacode, svxid):
-    """Only called by survex parser. 
+    """Only called by survex parser, NOT the cave parser.
     Creates a new Cave object, but with abbreviated data as the survex file (from ARGE) is all we have.
     We already know the survex file.
-    We already know that it doesn't exist... though there are bugs..
+    We already know that the cave doesn't exist... though there are bugs..
     """
     
     default_note = "This is an ARGE cave where we only have the survex file and no other information"
+    url = f"{areacode}/{caveid}.html"
 
     urltest = Cave.objects.filter(url=url)
     if urltest:
@@ -171,11 +172,21 @@ def do_ARGE_cave(slug, caveid, url, areacode, svxid):
         DataIssue.objects.create(parser="caves", message=message, url=url)
         print(message)
         return numtest[0]
-
+    
+    sv = Path(settings.SURVEX_DATA, svxid + ".svx")
+    if sv.is_file:
+        with open(sv, "r") as s:
+            line1 = s.readline()
+            line2 = s.readline()
+            line3 = s.readline()
+    else:
+        print(f"not correct {sv}", file=sys.stderr)
+        
     cave = Cave( 
-    underground_description="ARGE cave.",
+    underground_description="ARGE cave.\n3 lines of the survexfile:\n" + line1 +line2 +line3,
-    survex_file= f"{svxid}.svx", # or is this svxurl ?!
+    unofficial_number="ARGE",
-    # url=url, No, the url spplied is that of the survexfile not of the cave file, e.g. /1626/254/254
+    survex_file= svxurl,
+    url=url, 
     notes=default_note,
     areacode=areacode,
     )
@@ -427,7 +438,7 @@ def read_entrance(filename, ent=None):
         DataIssue.objects.create(parser="entrances", message=message, url=f"/cave/{slug}/edit/")
         print(message)
         
-    lastvisit = getXML(entrancecontents, "lastvisit", maxItems=1, minItems=0, context=context)
+    lastvisit = getXML(entrancecontents, "lastvisit", maxItems=1, minItems=0, context=f"/cave/{slug}/edit/")
 
     alt = getXMLmax1("alt")
     approach = getXMLmax1("approach")
@@ -611,12 +622,12 @@ def read_cave(filename, cave=None):
     
     # Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
     fn = settings.CAVEDESCRIPTIONS / filename
-    context = filename
+    context = f"/cave_data/{filename}_edit"
     
     # print(f" - Reading Cave from cave descriptions file {fn}")
     if not fn.exists():
         message = f" ! Cave_data file reading problem filename:'cave_data/{filename}'"
-        DataIssue.objects.create(parser="caves", message=message, url=f"/cave_data/{filename}_edit")
+        DataIssue.objects.create(parser="caves", message=message, url=context)
         print(message)
         return None
 
@@ -626,7 +637,7 @@ def read_cave(filename, cave=None):
 
     if len(cavecontentslist) != 1:
         message = f'! BAD CAVE DATA in "{filename}". More than one cave. Edit file manually, click.'
-        DataIssue.objects.create(parser="caves", message=message, url=f"/cave_data/{filename}_edit")
+        DataIssue.objects.create(parser="caves", message=message, url=context)
         print(message)
         return None
         
@@ -634,10 +645,11 @@ def read_cave(filename, cave=None):
     slugs = getXML(cavecontents, "caveslug", maxItems=1, context=context)
     if len(slugs) > 1:
         message = f" ! - More than one slug for a cave: {cave}, slugs: {slugs}. Ignoring all except first."
-        DataIssue.objects.create(parser="caves", message=message, url=f"{cave.url}_edit/")
+        DataIssue.objects.create(parser="caves", message=message, url=context)
         print(message)
     slug = slugs[0]
-
+    context = url=f"/{slug[0:4]}/{slug}_cave_edit/"
+ 
     non_public = getXMLmax1("non_public")
     official_name = getXMLmax1("official_name")
     kataster_code = getXMLmax1("kataster_code")
@@ -668,7 +680,7 @@ def read_cave(filename, cave=None):
             kaves = Cave.objects.all().filter(filename=filename) # replace with slug when CaveSlug tidied up
             for k in kaves:
                 message = " ! - DUPLICATES in db. kataster:" + str(k.kataster_number) + ", slug:" + str(k.slug())
-                DataIssue.objects.create(parser="caves", message=message)
+                DataIssue.objects.create(parser="caves", message=message, url=context)
                 print(message)
             for k in kaves:
                 if k.slug() is not None:
@@ -679,6 +691,7 @@ def read_cave(filename, cave=None):
     # From here on the code applies to both edited and newly-imported caves (mostly!)
     do_caveslugstuff() # needs cave!=None
     
+    
     cave.non_public=boolify(non_public)
     cave.official_name=official_name[0]
     cave.kataster_code=kataster_code[0]
@@ -701,11 +714,14 @@ def read_cave(filename, cave=None):
             
     areas = getXML(cavecontents, "area", context=context) # can be multiple <area> tags
     for area_slug in areas:
-        if area_slug in ["1623", "1624", "1626", "1627"]: # ignore sub areas which are in another <area> tag
+        if area_slug in AREACODES: # ignore sub areas which are in another <area> tag
             cave.areacode = area_slug 
         else:
             cave.subarea = area_slug
- 
+    if not cave.areacode:
+         if slug[0:4] in AREACODES:
+            cave.areacode = slug[0:4]
+            
     entrances = getXML(cavecontents, "entrance", context=context)
     do_entrances()
     # print(f"- {entrances_xslug=}")

parsers/survex.py

@@ -14,7 +14,7 @@ from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
 from troggle.core.models.wallets import Wallet
 from troggle.core.models.troggle import DataIssue, Expedition
 from troggle.core.utils import chaosmonkey, get_process_memory
-from troggle.parsers.caves import create_new_cave, do_ARGE_cave
+from troggle.parsers.caves import create_new_cave, do_ARGE_cave, AREACODES, ARGEAREAS
 from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner
 
 """Imports the tree of survex files following from a defined root .svx file
@@ -1201,7 +1201,7 @@ class LoadingSurvex:
                     self.caveslist[cavepath] = self.caveslist[s] # set "caves-1626/107/107" as index to cave 1626-107
                     return self.caveslist[s]
             
-            if (cavepath.startswith("caves-1624") or cavepath.startswith("caves-1626")):
+            if cavepath[6:10] in ARGEAREAS:
                 url = f"/survexfile/{svxid}.svx"
                 return do_ARGE_cave(sluggy, caveid, url, area, svxid)
                 

templates/caveindex.html

@@ -49,7 +49,7 @@
     </p>
 
 <h3>1626</h3>
-<div style="column-count: 2;">
+<div style="column-count: 3;">
 <table class="searchable">
     {% for cave in caves1626 %}
     <tr><td><a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }} {{cave.official_name|safe}}</a> {% if cave.unofficial_number %}({{cave.unofficial_number }}){% endif %}{% else %}{{cave.unofficial_number }} {{cave.official_name|safe}}</a> {% endif %} </td></tr>
@@ -64,7 +64,7 @@
   </p>
 
 <h3>1627</h3>
-<div style="column-count: 2;">
+<div style="column-count: 3;">
 <table class="searchable">
     {% for cave in caves1627 %}
     <tr><td><a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }} {{cave.official_name|safe}}</a> {% if cave.unofficial_number %}({{cave.unofficial_number }}){% endif %}{% else %}{{cave.unofficial_number }} {{cave.official_name|safe}}</a> {% endif %} </td></tr>

templates/dataformat/cave.xml

@@ -19,7 +19,7 @@ the form documented at
 <non_public>{{ cave.non_public }}</non_public>{% for slug in cave.caveslug_set.all %}<!-- 'False' or 'True'. True if the cave should only be visible to logged-in users. Caves are normally public, so enter 'False' unless you know otherwise. -->
 <caveslug>{{ slug.slug|default_if_none:"1623-000"|safe }}</caveslug>{% endfor %}<!--(Required). Internal I.D. used to refer to this cave in entrance data files. Typically the same as the filebase, e.g. '1623-195'  -->
 <official_name>{{ cave.official_name|default_if_none:""|safe }}</official_name><!-- Name of the cave (normally in German)  Use &uuml; for u+Umlaut and &ouml; for o+umlaut eg H&ouml;hle for Hohle and Gl&uuml;ck for Gluck-->
-{% for area in cave.area.all %}<area>{{ area.short_name|default_if_none:""|safe }}</area>{% endfor %}<!-- the CUCC-defined areas shown in http://expo.survex.com/areas.htm -->
+<area>{{cave.areacode |safe }}</area><!-- e.g. "1623" -->
 <kataster_code>{{ cave.kataster_code|default_if_none:""|safe }}</kataster_code><!-- 'length-or-depth/type exploration'
 code used in the Austrian kataster e.g '1/S +' - https://expo/.survex.com/katast.htm
 

templates/prospecting.html

@@ -40,8 +40,6 @@
      <td><a href="/{{ cave.url }}">{{ cave }}</a></td>
      <td>{{ cave.official_name|safe }}</td>
      <td>{{ cave.kataster_code }}</td>
-     <td>{{ cave.hassurveydata }}</td>
-     <td>{{ cave.hassurvey }}</td>
      <td>{{ cave.caveandentrance_set.all.0.entrance.marking_val }}</td>
      <td>{{ cave.caveandentrance_set.all.0.entrance.has_photo }} </td>
      <td>{{ cave.caveandentrance_set.all.0.entrance.find_location }}</td>