survex_file field inconsistency detection & edit

core/models_caves.py

@@ -189,7 +189,8 @@ class Cave(TroggleModel):
             subprocess.call(settings.FIX_PERMISSIONS) 
             f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "wb")
         t = loader.get_template('dataformat/cave.xml')
-        c = Context({'cave': self})
+        #c = Context({'cave': self})
+        c = dict({'cave': self})
         u = t.render(c)
         u8 = u.encode("utf-8")
         f.write(u8)

core/views/caves.py

@@ -1,11 +1,12 @@
 import os
 import string
-import subprocess
 import re
 import settings
 import urllib.parse
-
+import subprocess
+from pathlib import Path
 from PIL import Image, ImageDraw, ImageFont
+
 from django import forms
 from django.conf import settings
 from django.urls import reverse
@@ -59,6 +60,23 @@ class MapLocations(object):
     def __str__(self):
         return "{} map locations".format(len(self.p))
 
+def getCaves(cave_id):
+    '''Only gets called if a call to getCave() raises a MultipleObjects exception
+    
+    TO DO: search GCavelookup first, which should raise a MultpleObjectsReturned exception if there
+    are duplicates'''
+    try:
+        caves = Cave.objects.filter(kataster_number=cave_id)
+        caveset = set(caves)
+        
+        Gcavelookup = GetCaveLookup() # dictionary makes strings to Cave objects
+        if cave_id in Gcavelookup:
+            caveset.add(Gcavelookup[cave_id])
+        return list(caveset)
+    except:
+        return []
+
+
 def getCave(cave_id):
     '''Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm.
     
@@ -110,7 +128,6 @@ def getnotablecaves():
                     notablecaves.append(c)
     return notablecaves
 
-
 def caveindex(request):
     caves = Cave.objects.all()
     caves1623 = list(Cave.objects.filter(area__short_name = "1623"))
@@ -120,23 +137,51 @@ def caveindex(request):
     return render(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':getnotablecaves(), 'cavepage': True})
 
 def cave3d(request, cave_id=''):
+    '''This is used to create a download url in templates/cave.html if anyone wants to download the .3d file 
+    The caller template tries kataster first, then unofficial_number if that kataster number does not exist
+    but only if Cave.survex_file is non-empty
+    '''
     try:
         cave = getCave(cave_id) 
-    except Cave.MultipleObjectsReturned: # entirely the wrong action, REPLACE with the right display
+    except ObjectDoesNotExist:
-        caves = Cave.objects.filter(kataster_number=cave_id) 
+        return None
-        return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves })
+    except Cave.MultipleObjectsReturned: 
+        # But only one might have survex data? So scan and return the first that works. 
+        caves = getCaves(cave_id) 
+        for c in caves: 
+            if c.survex_file: 
+                # exists, but may not be a valid file path to a valid .svx file in the Loser repo
+                return file3d(request, c, c.slug)
+    else:
+        return file3d(request, cave, cave_id)
 
-    survexfilename = settings.SURVEX_DATA + cave.survex_file
+def file3d(request, cave, cave_id):
-    threedfilename = settings.THREEDCACHEDIR + '%s.3d' % cave_id
+    '''Produces a .3d file directly for download.
-    if True or os.path.getmtime(survexfilename) > os.path.getmtime(threedfilename):
+    survex_file should be in format 'caves-1623/264/264.svx' but it might be mis-entered as simply '2012-ns-10.svx'
-        subprocess.call(["cavern", "--output=%s" % threedfilename, survexfilename])
+    Here we only use the stem of the last part anyway.
-    test_file = open(threedfilename, 'rb')
+    
-    response = HttpResponse(content=test_file, content_type='application/3d')#mimetype is replaced by content_type for django 1.7
+    TO DO properly decide whether we want to use the stem of the .svx file or the cave slug . This assumes they are the same...
-    response['Content-Disposition'] = 'attachment; filename=%s.3d' % cave_id
+    '''
+    survexfilename = Path(settings.SURVEX_DATA, cave.survex_file)
+    threedfilename = Path(settings.THREEDCACHEDIR, cave_id +'.3d') # assumes cave_id is stem of survex_file. oops.
+    threedcachedir = Path(settings.THREEDCACHEDIR)
+    
+    if not threedfilename.is_file() or os.path.getmtime(survexfilename) > os.path.getmtime(threedfilename):
+        try:
+            op = subprocess.check_output([settings.CAVERN, "--log", "--output={}".format(threedcachedir), "{}".format(survexfilename)])
+        except OSError as ex:
+            # propagate this to caller
+            raise OSError(op) from ex
+    
+    if threedfilename.is_file():
+        response = HttpResponse(content=open(threedfilename, 'rb'), content_type='application/3d')
+        response['Content-Disposition'] = 'attachment; filename={}.3d'.format(cave_id)
         # response['X-Sendfile'] = "%s.3d" % cave_id
         # It's usually a good idea to set the 'Content-Length' header too.
         # You can also set any other required headers: Cache-Control, etc.
         return response
+    else:
+        return None 
 
 def cavepage(request, karea, subpath):
     '''Displays a cave description page
@@ -217,6 +262,9 @@ def caveLogbook(request, slug):
 
 @login_required_if_public
 def edit_cave(request, slug=None):
+    '''This is the form that edits all the cave data and writes out an XML file in the :expoweb: repo folder
+    The format for the file being saved is in templates/dataformat/cave.xml
+    '''
     if  slug is not None:
         cave = Cave.objects.get(caveslug__slug = slug)
     else:
@@ -339,10 +387,6 @@ def surveyindex(request):
     expeditions=Expedition.objects.order_by("-year")
     return render(request,'survey.html',locals())
 
-# def cave_description(request, cavedescription_name):
-    # cave_description = get_object_or_404(CaveDescription, short_name = cavedescription_name)
-    # return render(request,'cave_description.html', locals())
-
 def get_entrances(request, caveslug):
     cave = Cave.objects.get(caveslug__slug = caveslug)
     return render(request,'options.html', {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]})

localsettingsWSL.py

@@ -9,7 +9,8 @@ System for Linux (WSL), on the main server or in the potato hut,
 using SQLite or mariaDB.
 
 It sets the directory locations for the major parts of the system so
-that e.g. expofiles can be on a different filesystem.
+that e.g. expofiles can be on a different filesystem, or /javascript/ can be in
+a system-wide location rather than just a local directory.
 
 This file is included at the end of the main troggle/settings.py file so that
 it overwrites defaults in that file.
@@ -22,24 +23,26 @@ print(" * importing troggle/localsettings.py")
 #  - have you checked that credentials.py is in .gitignore ?
 #  - we don't want to have to change the expo system password !
 #-----------------------------------------------------------------
-# default values, then get overwritten by real secrets
+# default values, then get overwritten by real secrets imported from credentials.py
 EXPOUSERPASS = "nnn:gggggg"
 EMAIL_HOST_PASSWORD = "insert-real-email-password-here"
 from credentials import EXPOUSERPASS
 from credentials import EMAIL_HOST_PASSWORD
 
-SERVERPORT = '8000'
 EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate.
 #SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely
 
+SERVERPORT = '8000' # not needed
+
 # --------------------- MEDIA redirections BEGIN --------------------- 
 #REPOS_ROOT_PATH  = '/mnt/d/CUCC-Expo/t37/'
 REPOS_ROOT_PATH = Path(__file__).parent.parent
-LIBDIR  =  REPOS_ROOT_PATH / 'lib' / 'python3.7'
+LIBDIR  =  REPOS_ROOT_PATH / 'lib' / 'python3.7' # should be finding this automatically: python --version etc.
 
 TROGGLE_PATH = Path(__file__).parent
-TEMPLATE_PATH     = os.fspath(TROGGLE_PATH / 'templates')
+TEMPLATE_PATH     = TROGGLE_PATH / 'templates'
-MEDIA_ROOT        = os.fspath(TROGGLE_PATH / 'media')
+MEDIA_ROOT        = TROGGLE_PATH / 'media'
+JSLIB_ROOT        = TROGGLE_PATH / 'media' / 'jslib' # used for CaveViewer JS utility
 
 FILES = Path('/mnt/f/expofiles/')
 EXPOFILES = Path('/mnt/f/expofiles/')
@@ -52,7 +55,7 @@ MEDIA_URL = '/site-media/'
 
 DIR_ROOT = ''#this should end in / if a value is given
 URL_ROOT = '/'
-URL_ROOT = 'http://localhost:'+ SERVERPORT +'/'
+# URL_ROOT = 'http://localhost:'+ SERVERPORT +'/'
 
 MEDIA_URL   = urllib.parse.urljoin(URL_ROOT , '/site_media/')
 SURVEYS_URL = urllib.parse.urljoin(URL_ROOT , '/survey_scans/')
@@ -61,7 +64,7 @@ SVX_URL     = urllib.parse.urljoin(URL_ROOT , '/survex/')
 
 
 STATIC_URL = urllib.parse.urljoin(URL_ROOT , '/static/') # used by Django admin pages. Do not delete.
-JSLIB_URL  = urllib.parse.urljoin(URL_ROOT , '/javascript/') # always fails, try to revive it ?
+JSLIB_URL  = urllib.parse.urljoin(URL_ROOT , '/javascript/') # used for CaveViewer JS utility
 
 #STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py
 # --------------------- MEDIA redirections END --------------------- 
@@ -115,21 +118,22 @@ TEMPLATES = [
                 'django.contrib.messages.context_processors.messages',
             ],
             'loaders': [
-                'django.template.loaders.filesystem.Loader',
+                'django.template.loaders.filesystem.Loader', # default lcation is troggle/templates/
-                'django.template.loaders.app_directories.Loader', #For each app, inc admin,  in INSTALLED_APPS, loader looks for /templates 
+                'django.template.loaders.app_directories.Loader', # needed for admin 'app' 
-                # insert your own TEMPLATE_LOADERS here
            ]
        },
     },
 ]
 
+# Passwords are loaded from credentials.py by settings.py
+EXPOUSERPASS = "nnn:gggggg" # overwritten by loading from credentials.py 
+EMAIL_HOST_PASSWORD = "insert-real-email-password-here" # overwritten by loading from credentials.py
+
 EXPOUSER = 'expo'
-# EXPOUSERPASS = "nnn:gggggg" # loaded from credentials.py
 EXPOUSER_EMAIL = 'philip.sargent@gmail.com'
 
 EMAIL_HOST = "smtp-auth.mythic-beasts.com"
 EMAIL_HOST_USER = "django-test@klebos.net" # Philip Sargent really
-# EMAIL_HOST_PASSWORD = "insert-real-email-password-here" # loaded from credentials.py
 EMAIL_PORT=587
 EMAIL_USE_TLS = True
 DEFAULT_FROM_EMAIL = 'django-test@klebos.net'
@@ -156,4 +160,8 @@ THREEDCACHEDIR = os.fspath(THREEDCACHEDIR)
 TUNNEL_DATA     = os.fspath(TUNNEL_DATA)
 SURVEX_DATA     = os.fspath(SURVEX_DATA)
 REPOS_ROOT_PATH = os.fspath(REPOS_ROOT_PATH)
+TEMPLATE_PATH   = os.fspath(TROGGLE_PATH)
+MEDIA_ROOT      = os.fspath(MEDIA_ROOT)
+JSLIB_ROOT      = os.fspath(JSLIB_ROOT)
+
 print(" + finished importing troggle/localsettings.py")

templates/dataformat/cave.xml

@@ -1,5 +1,7 @@
+<!DOCTYPE html>
+<!-- This file is generated using the form documented at /handbook/survey/caveentry.html -->
 <!-- Only put one cave in this file -->
-<!-- If you edit this file, make sure you update the websites database -->
+<!-- If you edit this file by hand, make sure you update the database by doing a full data import -->
 <html lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>

templates/dataformat/entrance.xml

@@ -1,5 +1,7 @@
+<!DOCTYPE html>
 <!-- Only put one entrance in this file -->
-<!-- If you edit this file, make sure you update the websites database -->
+<!-- This file is generated using the form documented at /handbook/survey/caveentry.html -->
+<!-- If you edit this file by hand, make sure you update the database by doing a full data import -->
 <html lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>

templates/svxcaveseveral.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 {% block title %}List of survex files{% endblock %}
-
+{% load wiki_markup %}
 {% block content %}
     <!-- the only thing passed into this template is a list of object identifiers for caves.
     All the processing to extract the survex subdriectories and survex files is done in this template -->
@@ -14,7 +14,7 @@
     </p>
     <p>
     {% for survexdirectory in cave.survexdirectory_set.all %}
-      <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>
+      This cave has a linked survexdirectory: <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a> in the database.  See the table below.
     {% empty %}
     <p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps
     because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because the survex (.svx) files have been stored on the server in the 
@@ -28,6 +28,15 @@
     <a href="/handbook/survey/caveentry.html">this part of the survey handbook</a>.
     {% endfor %}
     </p>
+    
+    <p>If you can see a filename here: [&nbsp;<a href="/survexfile/{{cave.survex_file}}">{{cave.survex_file}}</a> &nbsp;] 
+    which does <em>not</em>  match any in the list below <em>including the directories beginning with </em><strong>caves-162X/</strong></em> 
+    not just the filename</em>, then (if logged on) you can
+    <strong>click here <var><a href="/cave/{{cave.slug}}/edit/">/cave{{cave.slug}}/edit</a></var></strong>
+    to go to a form to correct the online data. 
+    Instructions for filling in this form are in this part 
+    <a href="/handbook/survey/caveentry.html"> of the survey handbook</a>.
+    </p>
     {% for survexdirectory in cave.survexdirectory_set.all %}
         <h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
         <table>
@@ -86,7 +95,7 @@
                 </tr>
             {% endfor %}      
         {% endfor %}
-        </table>
+        </table><p>
     {% endfor %}
 {% endfor %}
 {% endblock %}

templates/svxcavesingle.html

@@ -1,10 +1,10 @@
 {% extends "base.html" %}
 {% block title %}List of survex files{% endblock %}
-
+{% load wiki_markup %}
 {% block content %}
 
 {% autoescape off %}
-<h1>Surveys for <a href="/{{cave.url}}">{{cave.official_name}}</a> - kataster:{{cave}}</h1>
+<h1>Surveys for <a href="/{{cave.url}}">{{cave.official_name}}</a> - identifier:{{cave}}</h1>
 {% endautoescape %}
 <!-- the only thing passed into this template is the object identifier for a cave.
 All the processing to extract the survex subdriectories and survex files is done in this template -->
@@ -27,6 +27,13 @@ All the processing to extract the survex subdriectories and survex files is done
     <a href="/handbook/survey/caveentry.html">this part of the survey handbook</a>.
   {% endfor %}
 </p>
+<p>If you can see a filename here: [&nbsp;<a href="/survexfile/{{cave.survex_file}}">{{cave.survex_file}}</a> &nbsp;] which does <em>not</em> match any in the list below <em>including the directories beginning with </em><strong>caves-162X/</strong></em> 
+    not just the filename</em>, then (if logged on) you can
+<strong>click here <var><a href="/cave/{{cave.slug}}/edit/">/cave{{cave.slug}}/edit</a></var></strong>
+to go to a form to correct the online data.
+    Instructions for filling in this form are in this part 
+    <a href="/handbook/survey/caveentry.html"> of the survey handbook</a>.
+</p>
 {% for survexdirectory in cave.survexdirectory_set.all %}
     <h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
     <table>