From 3487c22da3b2da03a7d6a0814ea87ffb0a32229f Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@klebos.com>
Date: Fri, 26 Mar 2021 02:01:29 +0000
Subject: [PATCH] Survex 1624, 1626, 1627 now displayed robustly
---
core/views_survex.py | 97 ++++++++++++++++++++-------------
localsettingsWSL.py | 22 ++++++--
localsettingsserver.py | 2 +-
templates/svxcaveseveral.html | 97 +++++++++++++++++++++++++++++++++
templates/svxcavesingle.html | 13 ++++-
templates/svxcavesingle404.html | 84 ++++++----------------------
6 files changed, 202 insertions(+), 113 deletions(-)
create mode 100644 templates/svxcaveseveral.html
diff --git a/core/views_survex.py b/core/views_survex.py
index 464b870..491d20d 100644
--- a/core/views_survex.py
+++ b/core/views_survex.py
@@ -11,7 +11,7 @@ from django.shortcuts import render_to_response, render
from django.template.context_processors import csrf
from django.http import HttpResponse, Http404
-from django.core.exceptions import ObjectDoesNotExist
+from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
import troggle.settings as settings
import parsers.survex
@@ -298,49 +298,32 @@ def identifycavedircontents(gcavedir):
subsvx.insert(0, primesvx)
return subdirs, subsvx
-def check_cave_registered(survex_cave):
- '''Checks whether a cave has been properly registered when it is found in the Loser repo
- This should be called by Databasereset not here in a view
- Currently Caves are only registered if they are listed in :expoweb: settings.CAVEDESCRIPTIONS
- so we need to add in any mroe here.
- '''
- try:
- cave = Cave.objects.get(kataster_number=survex_cave)
- return survex_cave
- except ObjectDoesNotExist:
- pass
- try:
- cave = Cave.objects.get(unofficial_number=survex_cave)
- if cave.kataster_number:
- return cave.kataster_number
- else:
- return None
- except ObjectDoesNotExist:
- pass
-
+
+def get_survexareapath(area):
+ return survexdatasetpath / str("caves-" + area)
# direct local non-database browsing through the svx file repositories
-# perhaps should use the database and have a reload button for it
-# why is caves-1623 HARD CODED here ?! That must be wrong..
+# every time the page is viewed! Should cache this.
def survexcaveslist(request):
'''This reads the entire list of caves in the Loser repo directory and produces a complete report.
It can find caves which have not yet been properly registered in the system by Databasereset.py because
someone may have uploaded the survex files without doing the rest of the integration process.
'''
- cavesdir = survexdatasetpath / "caves-1623"
- #cavesdircontents = { }
-
+ # TO DO - filter out the non-public caves from display UNLESS LOGGED INS
onefilecaves = [ ]
multifilecaves = [ ]
subdircaves = [ ]
+ fnumlist = [ ]
- # first sort the file list
- fnumlist = sorted([ (-int(re.match(r"\d*", f).group(0) or "0"), f) for f in os.listdir(cavesdir) ])
+ for area in ["1623", "1626", "1624", "1627"]:
+ cavesdir = get_survexareapath(area)
+ arealist = sorted([ (area, -int(re.match(r"\d*", f).group(0) or "0"), f) for f in os.listdir(cavesdir) ])
+ fnumlist += arealist
- #print(fnumlist)
+ print(fnumlist)
# go through the list and identify the contents of each cave directory
- for num, cavedir in fnumlist:
+ for area, num, cavedir in fnumlist:
# these have sub dirs /cucc/ /arge/ /old/ but that is no reason to hide them in this webpage
# so these are now treated the same as 142 and 113 which also have a /cucc/ sub dir
@@ -350,14 +333,15 @@ def survexcaveslist(request):
# This all assumes that the first .svx file has the same name as the cave name,
# which usually but not always true. e.g. caves-1623/78/allkaese.svx not caves-1623/78/78.svx
# which is why we now also pass through the cavedir
+ cavesdir = get_survexareapath(area)
gcavedir = os.path.join(cavesdir, cavedir)
if os.path.isdir(gcavedir) and cavedir[0] != ".":
subdirs, subsvx = identifycavedircontents(gcavedir)
- katast = check_cave_registered(cavedir) # should do this only once per database load or it will be slow
+ caveid = check_cave_registered(area, cavedir) # should do this only once per database load or it will be slow
survdirobj = [ ]
for lsubsvx in subsvx:
- survdirobj.append(("caves-1623/"+cavedir+"/"+lsubsvx, lsubsvx))
+ survdirobj.append(("caves-" +area+ "/" +cavedir+"/"+lsubsvx, lsubsvx))
# caves with subdirectories
if subdirs:
@@ -367,7 +351,7 @@ def survexcaveslist(request):
# assert not dsubdirs # handle case of empty sub directory
lsurvdirobj = [ ]
for lsubsvx in dsubsvx:
- lsurvdirobj.append(("caves-1623/"+cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))
+ lsurvdirobj.append(("caves-" +area+ "/" +cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))
if len(dsubsvx) >= 1:
subsurvdirs.append((subdir,lsurvdirobj[0], lsurvdirobj[0:])) # list now includes the first item too
subdircaves.append((cavedir, (survdirobj[0], survdirobj[1:]), subsurvdirs))
@@ -384,24 +368,63 @@ def survexcaveslist(request):
def survexcavesingle(request, survex_cave):
'''parsing all the survex files of a single cave and showing that it's consistent and can find all
the files and people. Should explicity fix the kataster number thing.
+ kataster numbers are not unique across areas. Fix this.
'''
sc = survex_cave
+
try:
- cave = Cave.objects.get(kataster_number=sc)
+ cave = Cave.objects.get(kataster_number=sc) # This may not be unique.
return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })
+
except ObjectDoesNotExist:
# can get here if the survex file is in a directory labelled with unofficial number not kataster number.
# maybe - and _ mixed up, or CUCC-2017- instead of 2017-CUCC-, or CUCC2015DL01 . Let's not get carried away..
- for unoff in [sc, sc.replace('-','_'), sc.replace('_','-')]:
+ for unoff in [sc, sc.replace('-','_'), sc.replace('_','-'), sc.replace('-',''), sc.replace('_','')]:
try:
- cave = Cave.objects.get(unofficial_number=unoff)
+ cave = Cave.objects.get(unofficial_number=unoff) # return on first one we find
return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })
except ObjectDoesNotExist:
- continue
+ continue # next attempt in for loop
return render_to_response('svxcavesingle404.html', {'settings': settings, "cave":sc })
+ except MultipleObjectsReturned:
+ caves = Cave.objects.filter(kataster_number=survex_cave)
+ return render_to_response('svxcaveseveral.html', {'settings': settings, "caves":caves })
+
except:
return render_to_response('svxcavesingle404.html', {'settings': settings, "cave":sc })
+def check_cave_registered(area, survex_cave):
+ '''Checks whether a cave has been properly registered when it is found in the Loser repo
+ This should be called by Databasereset not here in a view
+ Currently Caves are only registered if they are listed in :expoweb: settings.CAVEDESCRIPTIONS
+ so we need to add in any more here.
+
+ This function runs but does not seem to be used?!
+ '''
+ try:
+ cave = Cave.objects.get(kataster_number=survex_cave)
+ return str(cave)
+
+ except MultipleObjectsReturned:
+ caves = Cave.objects.filter(kataster_number=survex_cave)
+ for c in caves:
+ if str(c) == area + "-" + survex_cave :
+ return str(c) # just get the first that matches
+ return None # many returned but none in correct area
+
+ except ObjectDoesNotExist:
+ pass
+
+ try:
+ cave = Cave.objects.get(unofficial_number=survex_cave) # should be unique!
+ if cave.kataster_number:
+ return str(cave)
+ else:
+ return None
+ except ObjectDoesNotExist:
+ pass
+
+ return None
diff --git a/localsettingsWSL.py b/localsettingsWSL.py
index cae8080..d9b7d47 100644
--- a/localsettingsWSL.py
+++ b/localsettingsWSL.py
@@ -62,7 +62,7 @@ print(MEDIA_ROOT)
# --------------------- MEDIA redirections END ---------------------
PUBLIC_SITE = True
-DEBUG = True # Always keep this, even when on public server. Otherwise NO ERROR MESSAGES !
+DEBUG = True # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES !
# executables:
CAVERN = 'cavern'
@@ -79,8 +79,8 @@ DATABASES = {
'PORT' : '', # Set to empty string for default. Not used with sqlite3.
}
}
-# add in 290, 291, 358 when they don't make it crash horribly
-NOTABLECAVESHREFS = [ "264", "258", "204", "76", "107"]
+# add in 358 when they don't make it crash horribly
+NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"]
PYTHON_PATH = REPOS_ROOT_PATH / 'troggle'
sys.path.append(os.fspath(REPOS_ROOT_PATH))
@@ -110,7 +110,7 @@ TEMPLATES = [
'loaders': [
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
- # insert your TEMPLATE_LOADERS here
+ # insert your own TEMPLATE_LOADERS here
]
},
},
@@ -130,7 +130,7 @@ SURVEX_DATA = REPOS_ROOT_PATH / "loser"
TUNNEL_DATA = REPOS_ROOT_PATH / "drawings"
THREEDCACHEDIR = REPOS_ROOT_PATH / 'expowebcache' / '3d'
-EXPOWEB = REPOS_ROOT_PATH / "expoweb"
+EXPOWEB = REPOS_ROOT_PATH / "expoweb"
SURVEYS = REPOS_ROOT_PATH
SURVEY_SCANS = '/mnt/f/expofiles/surveyscans/'
FILES = '/mnt/f/expofiles/'
@@ -140,5 +140,15 @@ EXPOWEB_URL = ''
SURVEYS_URL = '/survey_scans/'
EXPOFILES ='/mnt/f/expofiles/'
-
+# Sanitise these to be strings as all other code is expecting strings
+# and we have not made the chnage to pathlib Path type in the other localsettings-* variants yet.
+CAVEDESCRIPTIONS = os.fspath(CAVEDESCRIPTIONS)
+ENTRANCEDESCRIPTIONS = os.fspath(ENTRANCEDESCRIPTIONS)
+LOGFILE = os.fspath(LOGFILE)
+SURVEYS = os.fspath(SURVEYS)
+EXPOWEB = os.fspath(EXPOWEB)
+THREEDCACHEDIR = os.fspath(THREEDCACHEDIR)
+TUNNEL_DATA = os.fspath(TUNNEL_DATA)
+SURVEX_DATA = os.fspath(SURVEX_DATA)
+REPOS_ROOT_PATH = os.fspath(REPOS_ROOT_PATH)
print(" + finished importing troggle/localsettings.py")
diff --git a/localsettingsserver.py b/localsettingsserver.py
index b87754d..8c2f3df 100644
--- a/localsettingsserver.py
+++ b/localsettingsserver.py
@@ -31,7 +31,7 @@ DATABASES = {
EXPOUSER = 'expo'
-EXPOUSERPASS = 'nnn:gggggg'
+EXPOUSERPASS = "nnn:gggggg"
EXPOUSER_EMAIL = 'wookey@wookware.org'
REPOS_ROOT_PATH = '/home/expo/'
diff --git a/templates/svxcaveseveral.html b/templates/svxcaveseveral.html
new file mode 100644
index 0000000..27131d5
--- /dev/null
+++ b/templates/svxcaveseveral.html
@@ -0,0 +1,97 @@
+{% extends "base.html" %}
+{% load wiki_markup %}
+{% load link %}
+
+{% block title %}List of survex files{% endblock %}
+
+{% block content %}
+
+
+{% for cave in caves %}
+
+ {% autoescape off %}
+ <h1>Surveys for <a href="/{{cave.url}}">{{cave.official_name}}</a> - kataster:{{cave}}</h1>
+ {% endautoescape %}
+ <!-- the only thing passed into this temnplate is the object identifier for a cave.
+ All the processing to extract the survex subdriectories and survex files is done in this template -->
+
+ <p>Cave description: <a href="/{{cave.url}}">{{cave.url}}</a>
+ </p>
+ <p>
+ {% for survexdirectory in cave.survexdirectory_set.all %}
+ <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>
+ {% empty %}
+ <p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps
+ because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because the survex (.svx) files have been stored on the server in the
+ <a href="/handbook/computing/repos.html"><var>:loser:</var></a> repository
+ but whoever was
+ responsible has not yet created the appropriate entries in the XML file in the
+ <a href="/handbook/computing/repos.html"><var>:expoweb:</var></a> repository.
+ It is the XML file which registers the cave description and ties together
+ the survex files with everything else.
+ <p>The process for registering a new cave is documented in
+ <a href="/handbook/survey/caveentry.html">this part of the survey handbook</a>.
+ {% endfor %}
+ </p>
+ {% for survexdirectory in cave.survexdirectory_set.all %}
+ <h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
+ <table>
+ <tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr>
+ {% for survexfile in survexdirectory.survexfile_set.all %}
+ <tr>
+ {% if survexfile.exists %}
+ <td rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
+ {% else %}
+ <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
+ {% endif %}
+
+ {% ifequal survexfile survexdirectory.primarysurvexfile %}
+ <a href="{% url "svx" survexfile.path %}"><b>{% url "svx" survexfile.path %}</b></a>
+ {% else %}
+ <a href="{% url "svx" survexfile.path %}"><i><small>{% url "svx" survexfile.path %}</small></i></a><!-- would like to extract only the last bit. Some javascript useful ?-->
+ {% endifequal %}
+ </td>
+ </tr>
+ {% for survexblock in survexfile.survexblock_set.all %}
+ <tr>
+ <!-- Block -->
+ <td style="width:10 em"><a href="{% url "svx" survexfile.path %}">{{survexblock.name}}</a></td>
+
+ <!-- Date -->
+ <td style="white-space:nowrap">
+ {% if survexblock.expedition %}
+ <a href="{{survexblock.expedition.get_absolute_url}}">{{survexblock.date|date:"D d M Y"}}</a>
+ {% else %}
+ <!--{{survexblock.date}}-->
+ {% endif %}
+ </td>
+ <!-- Explorers -->
+ <td>
+ {% for personrole in survexblock.survexpersonrole_set.all %}
+ {% if personrole.personexpedition %}
+ <a href="{{personrole.personexpedition.get_absolute_url}}">{{personrole.personname}}</a>
+ {% else %}
+ {{personrole.personname}}
+ {% endif %}
+ {% endfor %}
+ </td>
+ <!-- length -->
+ <td style="padding-right: 3px; text-align:right">{{survexblock.legslength|stringformat:".1f"}}</td>
+
+ <!-- Titles -->
+ <td style="padding-left: 3px;">
+ {{survexblock.title}}
+ </td>
+ <!-- Scans -->
+ <td>
+ {% if survexblock.scansfolder %}
+ <b><a href="{{survexblock.scansfolder.get_absolute_url}}">{{survexblock.scansfolder.walletname}}</a></b>
+ {% endif %}
+ </td>
+ </tr>
+ {% endfor %}
+ {% endfor %}
+ </table>
+ {% endfor %}
+{% endfor %}
+{% endblock %}
diff --git a/templates/svxcavesingle.html b/templates/svxcavesingle.html
index 319cb4b..7ec5e92 100644
--- a/templates/svxcavesingle.html
+++ b/templates/svxcavesingle.html
@@ -17,7 +17,18 @@ All the processing to extract the survex subdriectories and survex files is done
<p>
{% for survexdirectory in cave.survexdirectory_set.all %}
<a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>
-{% endfor %}
+ {% empty %}
+ <p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps
+ because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because the survex (.svx) files have been stored on the server in the
+ <a href="/handbook/computing/repos.html"><var>:loser:</var></a> repository
+ but whoever was
+ responsible has not yet created the appropriate entries in the XML file in the
+ <a href="/handbook/computing/repos.html"><var>:expoweb:</var></a> repository.
+ It is the XML file which registers the cave description and ties together
+ the survex files with everything else.
+ <p>The process for registering a new cave is documented in
+ <a href="/handbook/survey/caveentry.html">this part of the survey handbook</a>.
+ {% endfor %}
</p>
{% for survexdirectory in cave.survexdirectory_set.all %}
<h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
diff --git a/templates/svxcavesingle404.html b/templates/svxcavesingle404.html
index 7409785..75ef9ac 100644
--- a/templates/svxcavesingle404.html
+++ b/templates/svxcavesingle404.html
@@ -6,78 +6,26 @@
{% block content %}
-<h1>Surveys for cave - Cave not found</h1>
+<h1>Cave not found in database</h1>
<h3>Cave number looked for: '{{cave}}'</h3>
-<h3>kataster number like this not found</h3>
-<h3>unofficial number like this not found</h3>
+<ul>
+<li>- unofficial number like this not found</li>
+<li>- kataster number like this not found</li>
+</ul>
+<h3>You probably got here because a survex (.svx) file has been stored on the server in the
+<a href="/handbook/computing/repos.html"><var>:loser:</var></a> repository
+but whoever was
+responsible has not yet created the appropriate XML file the
+<a href="/handbook/computing/repos.html"><var>:expoweb:</var></a> repository
+which registers the cave description and ties together
+the survex files with everything else.
+</h3>
+
+<p>The process for registering a new cave is documented in
+<a href="/handbook/survey/caveentry.html">this part of the survey handbook</a>.
<p>
-{% for survexdirectory in cave.survexdirectory_set.all %}
- <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>
-{% endfor %}
-</p>
-
-{% for survexdirectory in cave.survexdirectory_set.all %}
-<h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
-
-<table>
-<tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr>
-
-{% for survexfile in survexdirectory.survexfile_set.all %}
-<tr>
- {% if survexfile.exists %}
- <td rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
- {% else %}
- <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
- {% endif %}
-
- {% ifequal survexfile survexdirectory.primarysurvexfile %}
- <a href="{% url "svx" survexfile.path %}"><b>{{survexfile.path}}</b></a>
- {% else %}
- <a href="{% url "svx" survexfile.path %}">{{survexfile.path}}</a>
- {% endifequal %}
- </td>
-</tr>
-
-{% for survexblock in survexfile.survexblock_set.all %}
-<tr>
- <td style="width:10 em">{{survexblock.name}}</td>
- <td style="white-space:nowrap">
- {% if survexblock.expedition %}
- <a href="{{survexblock.expedition.get_absolute_url}}">{{survexblock.date|date:"D d M Y"}}</a>
- {% else %}
- <!--{{survexblock.date}}-->
- {% endif %}
- </td>
-
- <td>
- {% for personrole in survexblock.survexpersonrole_set.all %}
- {% if personrole.personexpedition %}
- <a href="{{personrole.personexpedition.get_absolute_url}}">{{personrole.personname}}</a>
- {% else %}
- {{personrole.personname}}
- {% endif %}
- {% endfor %}
- </td>
-
- <td style="padding-right: 3px; text-align:right">{{survexblock.legslength|stringformat:".1f"}}</td>
-
- <td style="padding-left: 3px;">
- {{survexblock.title}}
- </td>
-
- <td>
- {% if survexblock.scansfolder %}
- <b><a href="{{survexblock.scansfolder.get_absolute_url}}">{{survexblock.scansfolder.walletname}}</a></b>
- {% endif %}
- </td>
-</tr>
-{% endfor %}
-{% endfor %}
-</table>
-
-{% endfor %}
{% endblock %}