3d CaveView regeneates .3d file in cache

core/views/caves.py

@@ -10,7 +10,7 @@ from PIL import Image, ImageDraw, ImageFont
 from django import forms
 from django.conf import settings
 from django.urls import reverse
-from django.http import HttpResponse, HttpResponseRedirect
+from django.http import HttpResponse, HttpResponseRedirect, HttpResponseNotFound
 from django.shortcuts import get_object_or_404, render
 from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 
@@ -120,10 +120,10 @@ def getnotablecaves():
             cave = Cave.objects.get(kataster_number=kataster_number)
             notablecaves.append(cave)
         except:
-            print(" ! FAILED to get only one cave per kataster_number OR invalid number for: "+kataster_number)
+            #print(" ! FAILED to get only one cave per kataster_number OR invalid number for: "+kataster_number)
             caves = Cave.objects.all().filter(kataster_number=kataster_number)
             for c in caves:
-                print(c.kataster_number, c.slug())
+                #print(c.kataster_number, c.slug())
                 if c.slug() != None:
                     notablecaves.append(c)
     return notablecaves
@@ -140,6 +140,9 @@ def cave3d(request, cave_id=''):
     '''This is used to create a download url in templates/cave.html if anyone wants to download the .3d file 
     The caller template tries kataster first, then unofficial_number if that kataster number does not exist
     but only if Cave.survex_file is non-empty
+    
+    But the template file cave.html has its own ideas about the name of the file and thus the href. Ouch.
+    /cave/3d/<cave_id>
     '''
     try:
         cave = getCave(cave_id) 
@@ -157,49 +160,114 @@ def cave3d(request, cave_id=''):
 
 def file3d(request, cave, cave_id):
     '''Produces a .3d file directly for download.
-    survex_file should be in format 'caves-1623/264/264.svx' but it might be mis-entered as simply '2012-ns-10.svx'
+    survex_file should be in valid path format 'caves-1623/264/264.svx' but it might be mis-entered as simply '2012-ns-10.svx'
-    Here we only use the stem of the last part anyway.
+
+    Also the cave.survex_file may well not match the cave description path: 
+    e.g. it might be to the whole system 'smk-system.svx' instead of just for the specific cave.
     
-    TO DO properly decide whether we want to use the stem of the .svx file or the cave slug . This assumes they are the same...
+    - If the expected .3d file corresponding to cave.survex_file is present, return it.
+    - If the cave.survex_file exists, generate the 3d file, cache it and return it
+    - Use the cave_id to guess what the 3d file might be and, if in the cache, return it
+    - Use the cave_id to guess what the .svx file might be and generate the .3d file and return it
+    - (Use the incomplete cave.survex_file and a guess at the missing directories to guess the real .svx file location ?)   
     '''
-    survexfilename = Path(settings.SURVEX_DATA, cave.survex_file)
+    def runcavern(survexpath):
-    threedfilename = Path(settings.THREEDCACHEDIR, cave_id +'.3d') # assumes cave_id is stem of survex_file. oops.
+        #print(" -  Regenerating cavern .log and .3d for '{}'".format(survexpath))
+        if not survexpath.is_file():
+            #print(" -  - Regeneration ABORT\n -  - from '{}'".format(survexpath))
+            
+        try:
+            completed_process = subprocess.run([settings.CAVERN, "--log", "--output={}".format(settings.THREEDCACHEDIR), "{}".format(survexpath)])
+        except OSError as ex:
+            # propagate this to caller. 
+            raise OSError(completed_process.stdout) from ex
+
+        op3d = (Path(settings.THREEDCACHEDIR) / Path(survexpath).name).with_suffix('.3d')
+        op3dlog = Path(op3d.with_suffix('.log'))
+
+        if not op3d.is_file():
+            print(" -  - Regeneration FAILED\n -  - from '{}'\n -  - to   '{}'".format(survexpath, op3d))
+            print(" -  - Regeneration stdout: ", completed_process.stdout)
+            print(" -  - Regeneration cavern log output: ", op3dlog.read_text())
+          
+            
+    def return3d(threedpath):
+        if threedpath.is_file():
+            response = HttpResponse(content=open(threedpath, 'rb'), content_type='application/3d')
+            response['Content-Disposition'] = 'attachment; filename={}'.format(threedpath.name)
+            return response
+        else:
+            message = '<h1>Path provided  does not correspond to any actual 3d file.</h1><p>path: "{}"'.format(threedpath)
+            #print(message)
+            return HttpResponseNotFound(message) 
+    
+    survexname = Path(cave.survex_file).name # removes directories
+    survexpath = Path(settings.SURVEX_DATA, cave.survex_file)
+    threedname = Path(survexname).with_suffix('.3d') # removes .svx, replaces with .3d
+    threedpath = Path(settings.THREEDCACHEDIR, threedname) 
     threedcachedir = Path(settings.THREEDCACHEDIR)
     
-    if not threedfilename.is_file() or os.path.getmtime(survexfilename) > os.path.getmtime(threedfilename):
+    # These if statements need refactoring more cleanly
-        try:
+    if cave.survex_file:
-            op = subprocess.check_output([settings.CAVERN, "--log", "--output={}".format(threedcachedir), "{}".format(survexfilename)])
+        #print(" -  cave.survex_file '{}'".format(cave.survex_file))
-        except OSError as ex:
+        if threedpath.is_file():
-            # propagate this to caller
+            #print(" -  threedpath '{}'".format(threedpath))
-            raise OSError(op) from ex
+            # possible error here as several .svx files of same names in different directories will overwrite in /3d/
-    
+            if survexpath.is_file():
-    if threedfilename.is_file():
+                if os.path.getmtime(survexpath) > os.path.getmtime(threedpath):
-        response = HttpResponse(content=open(threedfilename, 'rb'), content_type='application/3d')
+                    runcavern(survexpath)
-        response['Content-Disposition'] = 'attachment; filename={}.3d'.format(cave_id)
+            return return3d(threedpath)
-        # response['X-Sendfile'] = "%s.3d" % cave_id
+        else:
-        # It's usually a good idea to set the 'Content-Length' header too.
+            #print(" -  - survexpath '{}'".format(survexpath))
-        # You can also set any other required headers: Cache-Control, etc.
+            if survexpath.is_file():
-        return response
+                #print(" -  -  - survexpath '{}'".format(survexpath))
+                runcavern(survexpath)
+                return return3d(threedpath)
+
+    # Get here if cave.survex_file was set but did not correspond to a valid svx file
+    if survexpath.is_file():
+        # a file, but invalid format
+        message='<h1>File is not valid .svx format.</h1><p>Could not generate 3d file from "{}"'.format(survexpath)
     else:
-        return None 
+        # we could try to guess that 'caves-1623/' is missing,... nah.
+        message = '<h1>Path provided  does not correspond to any actual file.</h1><p>path: "{}"'.format(survexpath)
+
+    return HttpResponseNotFound(message) 
+    
+   
+def rendercave(request, cave, slug, cave_id=''):
+    '''Gets the data and files ready and then triggers Django to render the template.
+    The resulting html contains urls which are dispatched independently, e.g. the 'download' link
+    '''
+    #print(" ! rendercave:'{}' slug:'{}' cave_id:'{}'".format(cave, slug, cave_id))
+
+    if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
+        return render(request, 'nonpublic.html', {'instance': cave, 'cavepage': True, 'cave_id': cave_id})
+    else:
+        svxstem = Path(cave.survex_file).parent / Path(cave.survex_file).stem
+        svx3d = Path(cave.survex_file).stem
+        # NOTE the template itself loads the 3d file using javascript before it loads anything else.
+        # Django cannot see what this javascript is doing, so we need to ensure that the 3d file exists first.
+        # run this just for the side-effect of generating the 3d file? Nope, crashes. 
+        # TO DO - restructure this so that the regeneration is callable form here.
+        #discard_response = cave3d(request, cave_id=cave_id)
+        return render(request,'cave.html', {'settings': settings, 'cave': cave, 'cavepage': True, 
+            'cave_id': cave_id, 'svxstem': svxstem, 'svx3d':svx3d})
 
 def cavepage(request, karea, subpath):
     '''Displays a cave description page
+    accessed by kataster area number specifically
     
     There are A LOT OF URLS to e.g. /1623/161/l/rl89a.htm which are IMAGES and html files
     in cave descriptions. These need to be handled HERE
     '''
     path = karea + subpath
-    print(" ! cavepage:'{}' kataster area:'{}' rest of path:'{}'".format(path, karea, subpath))
+    #print(" ! cavepage:'{}' kataster area:'{}' rest of path:'{}'".format(path, karea, subpath))
     
     try:
         cave = Cave.objects.get(url = path) # ideally this will be unique
-        slug = cave.slug()
+        return rendercave(request, cave, cave.slug())
-        print(" - cavepage:'{}' cave:'{}' cave-slug:'{}'".format(path, str(cave), slug))
+
-        if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
-            return render(request,'nonpublic.html', {'instance': cave, 'cave_editable': slug})
-        else:
-            return render(request,'cave.html', {'cave': cave, 'cave_editable': slug})
     except Cave.DoesNotExist:
         # probably a link to text or an image e.g. 1623/161/l/rl89a.htm i.e. an expoweb page
         return expo.expopage(request, path)
@@ -209,14 +277,10 @@ def cavepage(request, karea, subpath):
     except:
         return render(request, 'pagenotfound.html', {'path': path})
 
-def caveSlug(request, slug):
-    cave = Cave.objects.get(caveslug__slug = slug)
-    if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
-        return render(request,'nonpublic.html', {'instance': cave, 'cave_editable': slug})
-    else:
-        return render(request,'cave.html', {'cave': cave, 'cave_editable': slug})
-
 def cave(request, cave_id='', offical_name=''):
+    '''Displays a cave description page
+    accesssed by a fairly random id which might be anything
+    '''
     try:
         cave=getCave(cave_id) 
     except MultipleObjectsReturned:
@@ -227,10 +291,7 @@ def cave(request, cave_id='', offical_name=''):
     except:
         return render(request, 'svxcavesingle404.html', {'settings': settings })
 
-    if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
+    return rendercave(request, cave, cave.slug(), cave_id=cave_id)
-        return render(request, 'nonpublic.html', {'instance': cave, 'cavepage': True, 'cave_id': cave_id})
-    else:
-        return render(request,'cave.html', {'settings': settings, 'cave': cave, 'cavepage': True, 'cave_id': cave_id})
 
 def caveEntrance(request, slug):
     cave = Cave.objects.get(caveslug__slug = slug)

templates/cave.html

@@ -3,7 +3,10 @@
 {% block extraheaders %}
 {% if cave.survex_file %}
 <style>
+# This is presumably very similar caveview.css but why is it copied here ?
+# Because it is NOT the same as the distrubuted CaveView code.
 
+# Needs to be separated out into JSLIB local. (PMS 3/4/2021)
 
 div.cv-panel {
 	position: absolute;
@@ -420,16 +423,17 @@ div#scene {
 		// the configuration object specifies the location of CaveView, surveys and terrain files
 		CV.UI.init( 'scene', {
 			home: '/javascript/CaveView/',
-			surveyDirectory: '/cave/3d/',
+			surveyDirectory: '/expowebcache/3d/',
-		terrainDirectory: '/loser/surface/terrain/'
+            terrainDirectory: '/loser/surface/terrain/'
 		} );
 
 		// load a single survey to display
-		CV.UI.loadCave( '{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{ cave.unofficial_number }}{% endif %}.3d' );
+        // Note the special code in views.caves.py to do this. The appropriate .svx/.3d file may not be simply the cave name +.3d
+		CV.UI.loadCave('{{svx3d}}.3d');
 	}
         window.onload = onLoad;
 </script>
-{% endif %}
+{% endif %} <!-- all the above only loads if cave.survex_file is not empty-->
 {% endblock %}
 
 
@@ -460,6 +464,52 @@ div#scene {
 {% block related %}
 {% endblock %}{% endblock %}
 
+
+
+<div id="Description">
+
+<p>{% if cave.explorers %}
+    <h2>Explorers</h2>
+    {{ cave.explorers|safe }}
+{% endif %}
+{% if cave.underground_description %}
+    <h2>Underground Description</h2>
+    {{ cave.underground_description|safe }}
+{% endif %}
+{% if cave.equipment %}
+    <h2>Equipment</h2>
+    {{ cave.equipment|safe }}
+{% endif %}
+{% if cave.references %}
+    <h2>References</h2>
+    {{ cave.references|safe }}
+{% endif %}
+{% if cave.survey %}
+    <h2>Survey</h2>
+    {{ cave.survey|safe }}
+{% endif %}
+{% if cave.kataster_status %}
+    <h2>Kataster_status</h2>
+    {{ cave.kataster_status|safe }}
+{% endif %}
+{% if cave.underground_centre_line %}
+    <h2>Underground Centre Line</h2>
+    {{ cave.underground_centre_line|safe }}
+{% endif %}
+{% if cave.survex_file %}
+    <h2>Survex File</h2>
+    <a href="{% url "survexcavessingle" cave.kataster_number %}">All survex files</a> &nbsp;&nbsp;&nbsp;
+    <a href="{% if cave.kataster_number %}{% url "cave3d" cave.kataster_number %}{% else %}{% url "cave3d" cave.unofficial_number %}{% endif %}">3d file download</a>&nbsp;&nbsp;&nbsp;
+    <a href="{% url "svx" svxstem %}">This survex file</a> &nbsp;&nbsp;&nbsp;
+    
+    <div id='scene'></div>
+{% endif %}
+{% if cave.notes %}
+    <h2>Notes</h2>
+    {{ cave.notes|safe }}
+{% endif %}</p>
+
+</div>
 <div id="entrances">
 		<p>{% if cave.entrances %}
     <h2>Entrances</h2>
@@ -531,47 +581,4 @@ div#scene {
 {% endif %}</p>
 <a href="{% url "newentrance" cave.slug %}">New Entrance</a>
 </div>
-
-<div id="Description">
-
-<p>{% if cave.explorers %}
-    <h2>Explorers</h2>
-    {{ cave.explorers|safe }}
-{% endif %}
-{% if cave.underground_description %}
-    <h2>Underground Description</h2>
-    {{ cave.underground_description|safe }}
-{% endif %}
-{% if cave.equipment %}
-    <h2>Equipment</h2>
-    {{ cave.equipment|safe }}
-{% endif %}
-{% if cave.references %}
-    <h2>References</h2>
-    {{ cave.references|safe }}
-{% endif %}
-{% if cave.survey %}
-    <h2>Survey</h2>
-    {{ cave.survey|safe }}
-{% endif %}
-{% if cave.kataster_status %}
-    <h2>Kataster_status</h2>
-    {{ cave.kataster_status|safe }}
-{% endif %}
-{% if cave.underground_centre_line %}
-    <h2>Underground Centre Line</h2>
-    {{ cave.underground_centre_line|safe }}
-{% endif %}
-{% if cave.survex_file %}
-    <h2>Survex File</h2>
-    {{ cave.survex_file|safe }} <a href="{% if cave.kataster_number %}{% url "cave3d" cave.kataster_number %}{% else %}{% url "cave3d" cave.unofficial_number %}{% endif %}">3d file</a>
-    <div id='scene'></div>
-{% endif %}
-{% if cave.notes %}
-    <h2>Notes</h2>
-    {{ cave.notes|safe }}
-{% endif %}</p>
-
-</div>
-
 {% endblock content %}

urls.py

@@ -82,17 +82,16 @@ trogglepatterns = [
     url(r'^getLogBookEntries/(?P<expeditionslug>.*)', logbooks.get_logbook_entries, name = "get_logbook_entries"), #works
 
     url(r'^cave/new/$', caves.edit_cave, name="newcave"),
+    url(r'^cave/3d/(?P<cave_id>[^/]+)$', caves.cave3d, name="cave3d"),
     url(r'^cave/(?P<cave_id>[^/]+)/?$', caves.cave, name="cave"),
     url(r'^cave/(?P<cave_id>[^/]+)/?(?P<ent_letter>[^/])$', ent), # view_caves.ent
     url(r'^cave/(?P<slug>[^/]+)/edit/$', caves.edit_cave, name="edit_cave"),
     url(r'^cave/(?P<cave_id>[^/]+)/(?P<year>\d\d\d\d)-(?P<qm_id>\d*)(?P<grade>[ABCDX]?)?$', caves.qm, name="qm"),
 
-    url(r'^caveslug/([^/]+)/?$', caves.caveSlug, name="caveSlug"),
     url(r'^cave/entrance/([^/]+)/?$', caves.caveEntrance),
     url(r'^cave/description/([^/]+)/?$', caves.caveDescription),
     url(r'^cave/qms/([^/]+)/?$', caves.caveQMs), # blank page
     url(r'^cave/logbook/([^/]+)/?$', caves.caveLogbook),
-    url(r'^cave/3d/(?P<cave_id>[^/]+).3d$', caves.cave3d, name="cave3d"),
 
     url(r'^entrance/(?P<caveslug>[^/]+)/(?P<slug>[^/]+)/edit/', caves.editEntrance, name = "editentrance"),
     url(r'^entrance/new/(?P<caveslug>[^/]+)/', caves.editEntrance, name = "newentrance"),
@@ -139,6 +138,7 @@ trogglepatterns = [
     url(r'^site_media/(?P<subpath>.*)$',  mediapage, {'doc_root': settings.MEDIA_ROOT},  name="mediapage"), # MEDIA_ROOT: CSS and JS 
     url(r'^static/(?P<subpath>.*)$',      mediapage, {'doc_root': settings.MEDIA_ROOT},  name="mediapage"), # STATIC is in MEDIA now!
     url(r'^javascript/(?P<subpath>.*)$',  mediapage, {'doc_root': settings.JSLIB_ROOT},  name="mediapage"), # JSLIB_URL 
+    url(r'^expowebcache/3d/(?P<subpath>.*)$',  mediapage, {'doc_root': settings.THREEDCACHEDIR},  name="mediapage"),  
 
     # This next is for shorthand references such as /1623/264
     url(r'^(?P<karea>\d\d\d\d)(?P<subpath>.*)$',     cavepage,     name="cavepage"), # Cave description