Survex 1624, 1626, 1627 now displayed robustly

core/views_survex.py

@@ -11,7 +11,7 @@ from django.shortcuts import render_to_response, render
 from django.template.context_processors import csrf
 from django.http import HttpResponse, Http404
 
-from django.core.exceptions import ObjectDoesNotExist
+from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 
 import troggle.settings as settings
 import parsers.survex
@@ -298,49 +298,32 @@ def identifycavedircontents(gcavedir):
         subsvx.insert(0, primesvx)
     return subdirs, subsvx
 
-def check_cave_registered(survex_cave):
+ 
-    '''Checks whether a cave has been properly registered when it is found in the Loser repo
+def get_survexareapath(area):
-    This should be called by Databasereset not here in a view
+    return survexdatasetpath /  str("caves-" + area)
-    Currently Caves are only registered if they are listed in :expoweb: settings.CAVEDESCRIPTIONS
-    so we need to add in any mroe here.
-    '''
-    try:
-        cave = Cave.objects.get(kataster_number=survex_cave)
-        return survex_cave
-    except ObjectDoesNotExist:
-        pass
-    try:
-        cave = Cave.objects.get(unofficial_number=survex_cave)
-        if cave.kataster_number:
-            return cave.kataster_number
-        else:
-            return None
-    except ObjectDoesNotExist:
-        pass
-
 
 # direct local non-database browsing through the svx file repositories
-# perhaps should use the database and have a reload button for it
+# every time the page is viewed! Should cache this.
-# why is caves-1623 HARD CODED here ?! That must be wrong..
 def survexcaveslist(request):
     '''This reads the entire list of caves in the Loser repo directory and produces a complete report.
     It can find caves which have not yet been properly registered in the system by Databasereset.py because
     someone may have uploaded the survex files without doing the rest of the integration process.
     '''
-    cavesdir = survexdatasetpath / "caves-1623"
+    # TO DO - filter out the non-public caves  from display UNLESS LOGGED INS
-    #cavesdircontents = { }
-    
     onefilecaves = [ ]
     multifilecaves = [ ]
     subdircaves = [ ]
+    fnumlist = [ ]
     
-    # first sort the file list
+    for area in ["1623", "1626", "1624", "1627"]:
-    fnumlist = sorted([ (-int(re.match(r"\d*", f).group(0) or "0"), f)  for f in os.listdir(cavesdir) ])
+        cavesdir = get_survexareapath(area)
+        arealist = sorted([ (area, -int(re.match(r"\d*", f).group(0) or "0"), f)  for f in os.listdir(cavesdir) ])
+        fnumlist += arealist
     
-    #print(fnumlist)
+    print(fnumlist)
     
     # go through the list and identify the contents of each cave directory
-    for num, cavedir in fnumlist:
+    for area, num, cavedir in fnumlist:
         
         # these have sub dirs /cucc/ /arge/ /old/ but that is no reason to hide them in this webpage
         # so these are now treated the same as 142 and 113 which also have a /cucc/ sub dir
@@ -350,14 +333,15 @@ def survexcaveslist(request):
         # This all assumes that the first .svx file has the same name as the cave name, 
         # which usually but not always true. e.g. caves-1623/78/allkaese.svx not caves-1623/78/78.svx
         # which is why we now also pass through the cavedir
+        cavesdir = get_survexareapath(area)
         gcavedir = os.path.join(cavesdir, cavedir)
         if os.path.isdir(gcavedir) and cavedir[0] != ".":
             subdirs, subsvx = identifycavedircontents(gcavedir)
             
-            katast = check_cave_registered(cavedir) # should do this only once per database load or it will be slow
+            caveid = check_cave_registered(area, cavedir) # should do this only once per database load or it will be slow
             survdirobj = [ ]
             for lsubsvx in subsvx:
-                survdirobj.append(("caves-1623/"+cavedir+"/"+lsubsvx, lsubsvx))
+                survdirobj.append(("caves-" +area+ "/" +cavedir+"/"+lsubsvx, lsubsvx))
             
             # caves with subdirectories
             if subdirs:
@@ -367,7 +351,7 @@ def survexcaveslist(request):
                     # assert not dsubdirs # handle case of empty sub directory
                     lsurvdirobj = [ ]
                     for lsubsvx in dsubsvx:
-                        lsurvdirobj.append(("caves-1623/"+cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))
+                        lsurvdirobj.append(("caves-" +area+ "/" +cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))
                     if len(dsubsvx) >= 1:
                         subsurvdirs.append((subdir,lsurvdirobj[0], lsurvdirobj[0:])) # list now includes the first item too
                 subdircaves.append((cavedir, (survdirobj[0], survdirobj[1:]), subsurvdirs))
@@ -384,24 +368,63 @@ def survexcaveslist(request):
 def survexcavesingle(request, survex_cave):
     '''parsing all the survex files of a single cave and showing that it's consistent and can find all 
     the files and people.   Should explicity fix the kataster number thing.
+    kataster numbers are not unique across areas. Fix this.
     '''
     sc = survex_cave
+
     try:
-        cave = Cave.objects.get(kataster_number=sc)
+        cave = Cave.objects.get(kataster_number=sc) # This may not be unique.
         return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })
+
     except ObjectDoesNotExist:
         # can get here if the survex file is in a directory labelled with unofficial number not kataster number.
         # maybe - and _ mixed up, or CUCC-2017- instead of 2017-CUCC-, or CUCC2015DL01 . Let's not get carried away..
-        for unoff in [sc, sc.replace('-','_'), sc.replace('_','-')]:
+        for unoff in [sc, sc.replace('-','_'), sc.replace('_','-'), sc.replace('-',''), sc.replace('_','')]:
             try:
-                cave = Cave.objects.get(unofficial_number=unoff)
+                cave = Cave.objects.get(unofficial_number=unoff) # return on first one we find
                 return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })
             except ObjectDoesNotExist:
-                continue
+                continue # next attempt in for loop
         return render_to_response('svxcavesingle404.html', {'settings': settings, "cave":sc })
 
+    except MultipleObjectsReturned:
+        caves = Cave.objects.filter(kataster_number=survex_cave) 
+        return render_to_response('svxcaveseveral.html', {'settings': settings, "caves":caves })
+
     except:
         return render_to_response('svxcavesingle404.html', {'settings': settings, "cave":sc })
 
  
+def check_cave_registered(area, survex_cave):
+    '''Checks whether a cave has been properly registered when it is found in the Loser repo
+    This should be called by Databasereset not here in a view
+    Currently Caves are only registered if they are listed in :expoweb: settings.CAVEDESCRIPTIONS
+    so we need to add in any more here.
+    
+    This function runs but does not seem to be used?!
+    '''
+    try:
+        cave = Cave.objects.get(kataster_number=survex_cave) 
+        return str(cave)
+
+    except MultipleObjectsReturned:
+        caves = Cave.objects.filter(kataster_number=survex_cave) 
+        for c in caves: 
+            if str(c) == area + "-" + survex_cave :
+                return str(c) # just get the first that matches
+        return None # many returned but none in correct area
+ 
+    except ObjectDoesNotExist:
+        pass
+
+    try:
+        cave = Cave.objects.get(unofficial_number=survex_cave) # should be unique!
+        if cave.kataster_number:
+            return str(cave)
+        else:
+            return None
+    except ObjectDoesNotExist:
+        pass
+        
+    return None
         

localsettingsWSL.py

@@ -62,7 +62,7 @@ print(MEDIA_ROOT)
 # --------------------- MEDIA redirections END --------------------- 
 
 PUBLIC_SITE = True
-DEBUG = True # Always keep this, even when on public server. Otherwise NO ERROR MESSAGES !
+DEBUG = True # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES !
 
 # executables:
 CAVERN = 'cavern'
@@ -79,8 +79,8 @@ DATABASES = {
         'PORT' : '',            # Set to empty string for default. Not used with sqlite3.
     }
 }
-# add in 290, 291, 358 when they don't make it crash horribly
+# add in 358 when they don't make it crash horribly
-NOTABLECAVESHREFS = [ "264", "258", "204", "76", "107"]
+NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"]
 
 PYTHON_PATH = REPOS_ROOT_PATH / 'troggle'
 sys.path.append(os.fspath(REPOS_ROOT_PATH))
@@ -110,7 +110,7 @@ TEMPLATES = [
             'loaders': [
                 'django.template.loaders.filesystem.Loader',
                 'django.template.loaders.app_directories.Loader',
-                # insert your TEMPLATE_LOADERS here
+                # insert your own TEMPLATE_LOADERS here
            ]
        },
     },
@@ -130,7 +130,7 @@ SURVEX_DATA = REPOS_ROOT_PATH / "loser"
 TUNNEL_DATA = REPOS_ROOT_PATH / "drawings"
 THREEDCACHEDIR  =  REPOS_ROOT_PATH / 'expowebcache' / '3d'
 
-EXPOWEB = REPOS_ROOT_PATH / "expoweb"
+EXPOWEB = REPOS_ROOT_PATH / "expoweb" 
 SURVEYS = REPOS_ROOT_PATH
 SURVEY_SCANS = '/mnt/f/expofiles/surveyscans/'
 FILES = '/mnt/f/expofiles/'
@@ -140,5 +140,15 @@ EXPOWEB_URL = ''
 SURVEYS_URL = '/survey_scans/'
 EXPOFILES ='/mnt/f/expofiles/'
 
-
+# Sanitise these to be strings as all other code is expecting strings
+# and we have not made the chnage to pathlib Path type in the other localsettings-* variants yet.
+CAVEDESCRIPTIONS = os.fspath(CAVEDESCRIPTIONS)
+ENTRANCEDESCRIPTIONS = os.fspath(ENTRANCEDESCRIPTIONS)
+LOGFILE = os.fspath(LOGFILE)
+SURVEYS = os.fspath(SURVEYS)
+EXPOWEB = os.fspath(EXPOWEB)
+THREEDCACHEDIR = os.fspath(THREEDCACHEDIR)
+TUNNEL_DATA = os.fspath(TUNNEL_DATA)
+SURVEX_DATA = os.fspath(SURVEX_DATA)
+REPOS_ROOT_PATH = os.fspath(REPOS_ROOT_PATH)
 print(" + finished importing troggle/localsettings.py")

localsettingsserver.py

@@ -31,7 +31,7 @@ DATABASES = {
 
 
 EXPOUSER = 'expo'
-EXPOUSERPASS = 'nnn:gggggg'
+EXPOUSERPASS = "nnn:gggggg"
 EXPOUSER_EMAIL = 'wookey@wookware.org'
 
 REPOS_ROOT_PATH = '/home/expo/'

templates/svxcaveseveral.html

@@ -0,0 +1,97 @@
+{% extends "base.html" %}
+{% load wiki_markup %}
+{% load link %}
+
+{% block title %}List of survex files{% endblock %}
+
+{% block content %}
+
+
+{% for cave in caves %}
+
+    {% autoescape off %}
+    <h1>Surveys for <a href="/{{cave.url}}">{{cave.official_name}}</a> - kataster:{{cave}}</h1>
+    {% endautoescape %}
+    <!-- the only thing passed into this temnplate is the object identifier for a cave.
+    All the processing to extract the survex subdriectories and survex files is done in this template -->
+
+    <p>Cave description: <a href="/{{cave.url}}">{{cave.url}}</a>
+    </p>
+    <p>
+    {% for survexdirectory in cave.survexdirectory_set.all %}
+      <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>
+    {% empty %}
+    <p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps
+    because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because the survex (.svx) files have been stored on the server in the 
+    <a href="/handbook/computing/repos.html"><var>:loser:</var></a> repository 
+    but whoever was 
+    responsible has not yet created the appropriate entries in the XML file in the 
+    <a href="/handbook/computing/repos.html"><var>:expoweb:</var></a> repository.
+    It is the XML file which registers the cave description and ties together
+    the survex files with everything else.
+    <p>The process for registering a new cave is documented in 
+    <a href="/handbook/survey/caveentry.html">this part of the survey handbook</a>.
+    {% endfor %}
+    </p>
+    {% for survexdirectory in cave.survexdirectory_set.all %}
+        <h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
+        <table>
+        <tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr>
+        {% for survexfile in survexdirectory.survexfile_set.all %}
+        <tr>
+          {% if survexfile.exists %}
+            <td rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
+          {% else %}
+            <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
+          {% endif %}
+
+          {% ifequal survexfile survexdirectory.primarysurvexfile %}
+            <a href="{% url "svx" survexfile.path %}"><b>{% url "svx" survexfile.path %}</b></a>
+          {% else %}
+            <a href="{% url "svx" survexfile.path %}"><i><small>{% url "svx" survexfile.path %}</small></i></a><!-- would like to extract only the last bit. Some javascript useful ?-->
+          {% endifequal %}
+          </td>
+        </tr>
+            {% for survexblock in survexfile.survexblock_set.all %}
+                <tr>
+                  <!-- Block -->   
+                 <td  style="width:10 em"><a href="{% url "svx" survexfile.path %}">{{survexblock.name}}</a></td>
+                  
+                  <!-- Date -->
+                  <td  style="white-space:nowrap">
+                  {% if survexblock.expedition %}
+                    <a href="{{survexblock.expedition.get_absolute_url}}">{{survexblock.date|date:"D d M Y"}}</a>
+                  {% else %}
+                    <!--{{survexblock.date}}-->
+                  {% endif %} 
+                  </td>
+                  <!-- Explorers -->
+                  <td>
+                  {% for personrole in survexblock.survexpersonrole_set.all %}
+                    {% if personrole.personexpedition %}
+                      <a href="{{personrole.personexpedition.get_absolute_url}}">{{personrole.personname}}</a>
+                    {% else %}
+                      {{personrole.personname}}
+                    {% endif %}
+                  {% endfor %}
+                  </td>
+                  <!-- length -->
+                  <td style="padding-right: 3px; text-align:right">{{survexblock.legslength|stringformat:".1f"}}</td>
+                 
+                  <!-- Titles -->
+                  <td style="padding-left: 3px;">
+                  {{survexblock.title}}
+                  </td>
+                  <!-- Scans -->
+                  <td>
+                  {% if survexblock.scansfolder %}
+                    <b><a href="{{survexblock.scansfolder.get_absolute_url}}">{{survexblock.scansfolder.walletname}}</a></b>
+                  {% endif %}
+                  </td>
+                </tr>
+            {% endfor %}      
+        {% endfor %}
+        </table>
+    {% endfor %}
+{% endfor %}
+{% endblock %}

templates/svxcavesingle.html

@@ -17,7 +17,18 @@ All the processing to extract the survex subdriectories and survex files is done
 <p>
 {% for survexdirectory in cave.survexdirectory_set.all %}
   <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>
-{% endfor %}
+    {% empty %}
+    <p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps
+    because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because the survex (.svx) files have been stored on the server in the 
+    <a href="/handbook/computing/repos.html"><var>:loser:</var></a> repository 
+    but whoever was 
+    responsible has not yet created the appropriate entries in the XML file in the 
+    <a href="/handbook/computing/repos.html"><var>:expoweb:</var></a> repository.
+    It is the XML file which registers the cave description and ties together
+    the survex files with everything else.
+    <p>The process for registering a new cave is documented in 
+    <a href="/handbook/survey/caveentry.html">this part of the survey handbook</a>.
+  {% endfor %}
 </p>
 {% for survexdirectory in cave.survexdirectory_set.all %}
     <h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>

templates/svxcavesingle404.html

@@ -6,78 +6,26 @@
 
 {% block content %}
 
-<h1>Surveys for cave - Cave not found</h1>
+<h1>Cave not found in database</h1>
 
 <h3>Cave number looked for: '{{cave}}'</h3>
-<h3>kataster number like this not found</h3>
+<ul>
-<h3>unofficial number like this not found</h3>
+<li>- unofficial number like this not found</li>
+<li>- kataster number like this not found</li>
+</ul>
 
+<h3>You probably got here because a survex (.svx) file has been stored on the server in the 
+<a href="/handbook/computing/repos.html"><var>:loser:</var></a> repository 
+but whoever was 
+responsible has not yet created the appropriate XML file the 
+<a href="/handbook/computing/repos.html"><var>:expoweb:</var></a> repository 
+which registers the cave description and ties together
+the survex files with everything else.
+</h3>
+
+<p>The process for registering a new cave is documented in 
+<a href="/handbook/survey/caveentry.html">this part of the survey handbook</a>.
 <p>
-{% for survexdirectory in cave.survexdirectory_set.all %}
-  <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>
-{% endfor %}
-</p>
-
-{% for survexdirectory in cave.survexdirectory_set.all %}
-<h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
-
-<table>
-<tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr>
-
-{% for survexfile in survexdirectory.survexfile_set.all %}
-<tr>
-  {% if survexfile.exists %}
-    <td rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
-  {% else %}
-    <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
-  {% endif %}
-
-  {% ifequal survexfile survexdirectory.primarysurvexfile %}
-    <a href="{% url "svx" survexfile.path %}"><b>{{survexfile.path}}</b></a>
-  {% else %}
-    <a href="{% url "svx" survexfile.path %}">{{survexfile.path}}</a>
-  {% endifequal %}
-  </td>
-</tr>
-
-{% for survexblock in survexfile.survexblock_set.all %}
-<tr>
-  <td  style="width:10 em">{{survexblock.name}}</td>
-  <td  style="white-space:nowrap">
-  {% if survexblock.expedition %}
-    <a href="{{survexblock.expedition.get_absolute_url}}">{{survexblock.date|date:"D d M Y"}}</a>
-  {% else %}
-    <!--{{survexblock.date}}-->
-  {% endif %} 
-  </td>
-
-  <td>
-  {% for personrole in survexblock.survexpersonrole_set.all %}
-    {% if personrole.personexpedition %}
-      <a href="{{personrole.personexpedition.get_absolute_url}}">{{personrole.personname}}</a>
-    {% else %}
-      {{personrole.personname}}
-    {% endif %}
-  {% endfor %}
-  </td>
-
-  <td style="padding-right: 3px; text-align:right">{{survexblock.legslength|stringformat:".1f"}}</td>
-
-  <td style="padding-left: 3px;">
-  {{survexblock.title}}
-  </td>
-
-  <td>
-  {% if survexblock.scansfolder %}
-    <b><a href="{{survexblock.scansfolder.get_absolute_url}}">{{survexblock.scansfolder.walletname}}</a></b>
-  {% endif %}
-  </td>
-</tr>
-{% endfor %}
-{% endfor %}
-</table>
-
-{% endfor %}
 
 {% endblock %}