fixed serving expofiles from test server

core/view_surveys.py

@@ -1,113 +1,56 @@
 import os, stat
 import re
-import urllib.request, urllib.parse, urllib.error
+from pathlib import Path
+from urllib.parse import urljoin, unquote as urlunquote
+from urllib.request import urlopen
 
 from django.conf import settings
-from django.shortcuts import render_to_response
+from django.shortcuts import render
 from django.http import HttpResponse, Http404
 
 from troggle.core.models_survex import ScansFolder, SingleScan, SurvexBlock, TunnelFile
 from troggle.flatpages import views as flatviews
 import parsers.surveys
 
-
+'''Some of these views serve files as binary blobs, and simply set the mime type based on the file extension,
-
+as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked 
-def fa_readFile(*path):
+by looking inside the file before being served.
-    try:
+'''
-        f = open(os.path.join(settings.FILES, *path))
-    except:
-        f = urllib.request.urlopen(settings.FILES+"download/")
-    return f.read()
-    
-def upload(request, path):
-    pass
-
-def download(request, path):
-    #try:
-        
-        return HttpResponse(fa_readFile(path), content_type=flatviews.getmimetype(path))
-    #except:
-    #    raise Http404
-
-def UniqueFile(fname):
-    while True:
-        if not os.path.exists(fname):
-            break
-        mname = re.match("(.*?)(?:-(\d+))?\.(png|jpg|jpeg)$(?i)", fname)
-        if mname:
-            fname = "%s-%d.%s" % (mname.group(1), int(mname.group(2) or "0") + 1, mname.group(3))
-    return fname
-
-
-# join it all up and then split them off for the directories that don't exist
-# anyway, this mkdir doesn't work
-def SaveImageInDir(name, imgdir, project, fdata, bbinary):
-    print(("hihihihi", fdata, settings.SURVEYS))
-    fimgdir = os.path.join(settings.SURVEYS, imgdir)
-    if not os.path.isdir(fimgdir):
-        print("*** Making directory", fimgdir)
-        os.path.mkdir(fimgdir)
-    fprojdir = os.path.join(fimgdir, project)
-    if not os.path.isdir(fprojdir):
-        print("*** Making directory", fprojdir)
-        os.path.mkdir(fprojdir)
-        print("hhh")
-        
-    fname = os.path.join(fprojdir, name)
-    print(fname, "fff")
-    fname = UniqueFile(fname)
-    
-    p2, p1 = os.path.split(fname)
-    p3, p2 = os.path.split(p2)
-    p4, p3 = os.path.split(p3)
-    res = os.path.join(p3, p2, p1)
-    
-    print("saving file", fname)
-    fout = open(fname, (bbinary and "wb" or "w"))
-    fout.write(fdata.read())
-    fout.close()
-    res = os.path.join(imgdir, name)
-    return res.replace("\\", "/")
 
 def surveyscansfolder(request, path):
     #print [ s.walletname  for s in ScansFolder.objects.all() ]
-    scansfolder = ScansFolder.objects.get(walletname=urllib.parse.unquote(path))
+    scansfolder = ScansFolder.objects.get(walletname=urlunquote(path))
-    return render_to_response('scansfolder.html', { 'scansfolder':scansfolder, 'settings': settings })
+    return render(request, 'scansfolder.html', { 'scansfolder':scansfolder, 'settings': settings })
 
 def surveyscansingle(request, path, file):
-    scansfolder = ScansFolder.objects.get(walletname=urllib.parse.unquote(path))
+    '''sends a single binary file to the user,
+    '''
+    scansfolder = ScansFolder.objects.get(walletname=urlunquote(path))
     singlescan = SingleScan.objects.get(scansfolder=scansfolder, name=file)
-    print("SSS {} {} :{}:".format(path, file, flatviews.getmimetype(file)))
+    # print(" - surveyscansingle {}:{}:{}:".format(path, file, flatviews.getmimetype(file)))
-    return HttpResponse(content=open(singlescan.ffile,"rb"), content_type=flatviews.getmimetype(file))
+    return HttpResponse(content=open(singlescan.ffile,"rb"), content_type=flatviews.getmimetype(file)) # any type of image
-    #return render_to_response('scansfolder.html', { 'scansfolder':scansfolder, 'settings': settings })
 
-def expofilessingle(request, filepath):
-    # defaults to content_type="text/pain" needs fixing for PDFs
-    fn=urllib.parse.unquote(filepath)
-    return HttpResponse(content=open(settings.EXPOFILES+fn,"rb"))
-
-def cssfilessingle(request, filepath):
-    fn=urllib.parse.unquote(filepath)
-    return HttpResponse(content=open(settings.MEDIA_ROOT+fn,"r"),content_type="text/css")
 
 def surveyscansfolders(request):
     manyscansfolders = ScansFolder.objects.all()
-    return render_to_response('manyscansfolders.html', { 'manyscansfolders':manyscansfolders, 'settings': settings })
+    return render(request, 'manyscansfolders.html', { 'manyscansfolders':manyscansfolders, 'settings': settings })
     
     
 def tunneldata(request):
     tunnelfiles = TunnelFile.objects.all()
-    return render_to_response('tunnelfiles.html', { 'tunnelfiles':tunnelfiles, 'settings': settings })
+    return render(request, 'tunnelfiles.html', { 'tunnelfiles':tunnelfiles, 'settings': settings })
     
 
-def tunnelfile(request, path):
+def tunnelfilesingle(request, path):
-    tunnelfile = TunnelFile.objects.get(tunnelpath=urllib.parse.unquote(path))
+    '''sends a single binary file to the user, We should have a renderer that syntax-colours this Tunnel xml 
-    tfile = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
+    '''
-    return HttpResponse(content=open(tfile), content_type="text/plain")
+    tunnelfile = TunnelFile.objects.get(tunnelpath=urlunquote(path))
+    tfile = Path(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
+    return HttpResponse(content=open(tfile), content_type="text/xhtml") # for display not download
     
 def tunnelfileupload(request, path):
-    tunnelfile = TunnelFile.objects.get(tunnelpath=urllib.parse.unquote(path))
+    tunnelfile = TunnelFile.objects.get(tunnelpath=urlunquote(path))
-    tfile = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
+    tfile = Path(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
     
     project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"]
     print((project, user, tunnelversion))

flatpages/views.py

@@ -1,6 +1,8 @@
 import os
 import re
 from pathlib import Path
+from urllib.parse import urljoin, unquote as urlunquote
+from urllib.request import urlopen
 
 from django.shortcuts import render, redirect
 from django.http import HttpResponse, HttpResponseRedirect, Http404
@@ -19,16 +21,44 @@ import troggle.settings as settings
 def expofiles_redirect(request, path):
     '''This is used only when running as a test system without a local copy of /expofiles/
     '''
-    return redirect('http://expo.survex.com/expofiles/' + path)
+    return redirect(urljoin('http://expo.survex.com/expofiles/', path))
+
+def expofilessingle(request, filepath):
+    '''sends a single binary file to the user,
+    '''
+    fn=urlunquote(filepath)
+    fn = Path(settings.EXPOFILES,filepath)
+    if fn.is_dir():
+        return expofilesdir(request, Path(fn), Path(filepath))
+    print(" - expofilessingle {}:{}:{}:".format(filepath, fn, getmimetype(fn)))
+    return HttpResponse(content=open(fn, "rb"),content_type=getmimetype(filepath)) # any file
+
+def expofilesdir(request, dirpath, filepath):
+    '''does a directory display. If there is an index.html file we should display that.
+    - dirpath is a Path() and it does not have /expofiles/ in it
+    '''
+    print(" - expofilesdir {}".format(dirpath))
+    urlpath = 'expofiles' / Path(filepath)
+    fileitems = []
+    diritems = []
+    for f in dirpath.iterdir():
+        if f.is_dir():
+             diritems.append((urlpath / f.parts[-1], str(f.parts[-1])))
+        else:
+            # if f.parts[-1].lower() == 'index.htm' or f.parts[-1].lower() == 'index.html': # css cwd problem
+                # return HttpResponse(content=open(f, "rb"),content_type=getmimetype(filepath)) # any file
+                # return expofilessingle(request, str(Path(filepath /  f.parts[-1])))             
+            fileitems.append((Path(urlpath) / f.parts[-1], str(f.parts[-1]), getmimetype(f)))
+    return render(request, 'dirdisplay.html', { 'filepath': urlpath, 'fileitems':fileitems, 'diritems': diritems,'settings': settings })
 
 def flatpage(request, path):
-    #print(" - FLATPAGES delivering the file: {} as MIME type: {}".format(path,getmimetype(path)))
+    '''Either renders an HTML page from expoweb with all the menus,
-    # try:
+    or serves an unadorned binary file with mime type'''
-        # r = Redirect.objects.get(originalURL = path)
+    print(" - FLATPAGES delivering the file: {} as MIME type: {}".format(path,getmimetype(path)),flush=True)
-        # #print(" - FLATPAGES REDIRECT the file: {} as: {}".format(path,r))
+
-        # return HttpResponseRedirect(r.newURL) # Redirect after POST
+    if path.startswith("noinfo") and settings.PUBLIC_SITE and not request.user.is_authenticated():
-    # except Redirect.DoesNotExist:
+        print((" - FLATPAGES redirect to logon: flat path noinfo", path))
-        # pass
+        return HttpResponseRedirect(urljoin(reverse("auth_login"),'?next={}'.format(request.path)))
 
     try:
         r = Cave.objects.get(url = path)
@@ -37,6 +67,7 @@ def flatpage(request, path):
         pass
     except:
         #print(" ! FAILED to get only one cave per slug for: "+path)
+        # we should do this proper;ly, not this hack that returns the first cave that matches
         caves = Cave.objects.all().filter(url = path)
         for c in caves:
             print(path, c.slug())
@@ -44,20 +75,9 @@ def flatpage(request, path):
                 return troggle.core.views_caves.caveSlug(request, c.slug())
         pass
 
-    # try:
-        # r = EntranceRedirect.objects.get(originalURL = path)
-        # return troggle.core.views_caves.entranceSlug(request, r.entrance.slug())
-    # except EntranceRedirect.DoesNotExist:
-        # pass
-
-
-    if path.startswith("noinfo") and settings.PUBLIC_SITE and not request.user.is_authenticated():
-        print(("flat path noinfo", path))
-        return HttpResponseRedirect(reverse("auth_login") + '?next=%s' % request.path)
-
     expowebpath = Path(settings.EXPOWEB)
     if path.endswith("/") or path == "":
-        #print(" - FLATPAGES the file: {} ENDSWITH ...".format(path))
+        print(" - FLATPAGES the file: {} ENDSWITH ...".format(path))
 
         try:
             o = open(os.path.normpath(expowebpath / path / "index.html"), "rb")
@@ -69,28 +89,32 @@ def flatpage(request, path):
             except IOError:
                 return render(request, 'pagenotfound.html', {'path': path})
     else:
-        try:
+        print(" - FLATPAGES the file: '{}'  ...".format(path))
-            #print(" - FLATPAGES the file: '{}'  ...".format(path))
-            #print(" - FLATPAGES MEDIA_ROOT: '{}'  ...".format(settings.MEDIA_ROOT))
         if path.startswith('site_media'):
-                #print(" - MEDIA_ROOT: {}  ...".format(settings.MEDIA_ROOT))
+            print(" - MEDIA_ROOT: {}  ...{}".format(settings.MEDIA_ROOT, path))
             path = path.replace("site_media", settings.MEDIA_ROOT)
             filetobeopened = os.path.normpath(path)
         elif path.startswith("static"):
-                #print(" - STATIC_ROOT: {}  ...".format(settings.MEDIA_ROOT))
+            print(" - STATIC_ROOT: {}  ...{}".format(settings.MEDIA_ROOT, path))
             path = path.replace("static", settings.MEDIA_ROOT)
             filetobeopened = os.path.normpath(path)
         else:
-                #print(" - NO    _ROOT: {}  ...".format(expowebpath))
+            print(" - NO    _ROOT: {}  ...".format(expowebpath))
             filetobeopened = os.path.normpath(expowebpath / path)
         
-            #print(" - FLATPAGES full path : {}  ...".format(filetobeopened))
+        print(" - FLATPAGES full path : {}  ...".format(filetobeopened))
+        try:
             o = open(filetobeopened, "rb")
-            #print(" - FLATPAGES full path no error: {}  ...".format(filetobeopened))
+            print(" - FLATPAGES full path no error: {}  ...".format(filetobeopened))
         except IOError:
-            #print(" - FLATPAGES ERROR: {}  ...".format(filetobeopened))
+            print(" - FLATPAGES ERROR: {}  ...".format(filetobeopened))
+            #o.close() 
             return render(request, 'pagenotfound.html', {'path': path})
+ 
+ 
     if path.endswith(".htm") or path.endswith(".html"):
+        # add the menus etc.
+        with open(os.path.normpath(expowebpath / path), "rb") as o:   
             html = o.read()
         
         m = re.search(rb'(.*)<\s*head([^>]*)>(.*)<\s*/head\s*>(.*)<\s*body([^>]*)>(.*)<\s*/body\s*>(.*)', html, re.DOTALL + re.IGNORECASE)
@@ -123,21 +147,24 @@ def flatpage(request, path):
         return render(request, 'flatpage.html', {'editable': editable, 'path': path, 'title': title, 
                     'body': body, 'homepage': (path == "index.htm"), 'has_menu': has_menu})
     else:
-        #print(" - FLATPAGES delivering the file: {} as MIME type: {}".format(path,getmimetype(path)))
+        print(" - FLATPAGES delivering the file: {} as MIME type: {}".format(path,getmimetype(path)))
-        return HttpResponse(o.read(), content_type=getmimetype(path))
+        return HttpResponse(content=open(filetobeopened, "rb"), content_type=getmimetype(path))
+        #return HttpResponse(content=open(singlescan.ffile,"rb"), content_type=getmimetype(path))
 
 def getmimetype(path):
+    path = str(path)
     if path.lower().endswith(".css"): return "text/css"
     if path.lower().endswith(".txt"): return "text/css"
     if path.lower().endswith(".js"): return "application/javascript"
     if path.lower().endswith(".json"): return "application/javascript"
+    if path.lower().endswith(".ico"): return "image/vnd.microsoft.icon"
     if path.lower().endswith(".png"): return "image/png"
     if path.lower().endswith(".tif"): return "image/tif"
     if path.lower().endswith(".gif"): return "image/gif"
     if path.lower().endswith(".jpeg"): return "image/jpeg"
     if path.lower().endswith(".jpg"): return "image/jpeg"
     if path.lower().endswith("svg"): return "image/svg+xml"
-    if path.lower().endswith("xml"): return "image/xml"
+    if path.lower().endswith("xml"): return "application/xml" # we use "text/xhtml" for tunnel files
     if path.lower().endswith(".pdf"): return "application/pdf"
     if path.lower().endswith(".ps"): return "application/postscript"
     if path.lower().endswith(".svx"): return "application/x-survex-svx"
@@ -146,6 +173,11 @@ def getmimetype(path):
     if path.lower().endswith(".err"): return "application/x-survex-err"
     if path.lower().endswith(".odt"): return "application/vnd.oasis.opendocument.text"
     if path.lower().endswith(".ods"): return "application/vnd.oasis.opendocument.spreadsheet"
+    if path.lower().endswith(".docx"): return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    if path.lower().endswith(".xslx"): return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+    if path.lower().endswith(".gz"): return "application/x-7z-compressed"
+    if path.lower().endswith(".7z"): return "application/x-7z-compressed"
+    if path.lower().endswith(".zip"): return "application/zip"
     return ""
 
 @login_required_if_public

templates/dirdisplay.html

@@ -0,0 +1,28 @@
+{% extends "base.html" %}
+
+{% block title %}Cambridge Expeditions to Austria{% endblock %}
+
+{% block content %}
+
+<div id="col1">
+<h3>{{filepath}}/</h3>
+
+{% for f in fileitems %}
+<a href="/{{ f.0 }}">{{ f.1 }}</a><br />
+{% endfor %}
+
+<p>
+<h3>{{filepath}}/</h3>
+
+{% for d in diritems %}
+<a href="/{{ d.0 }}">/{{ d.1 }}/<br />
+{% empty %}
+<p>&lt;No subdirectories&gt;
+{% endfor %}
+
+{% endblock content %}
+
+{% block margins %}
+<img  src="{{ settings.MEDIA_URL }}eieshole.jpg">
+<img  src="{{ settings.MEDIA_URL }}goesser.jpg">
+{% endblock margins %}

urls.py

@@ -33,16 +33,16 @@ The API urls return TSV or JSON and are new in July 2020.
 
 if settings.EXPOFILESREMOTE:
     expofilesurls = [
-        url(r'^(?P<path>.*)$',    flatviews.expofiles_redirect, name="expofiles_redirect"), # to expo.survex.com/expofiles
+        url(r'^/*(?P<path>.*)$',    flatviews.expofiles_redirect, name="expofiles_redirect"), # to expo.survex.com/expofiles
     ]
 else:
     expofilesurls = [
-        url(r'^(?P<filepath>.*)$',view_surveys.expofilessingle, name="single"), # local copy of EXPOFILES
+        url(r'^/*(?P<filepath>.*)$',flatviews.expofilessingle, name="single"), # local copy of EXPOFILES
     ]
    
  
 actualurlpatterns = [
-    url(r'^expofiles/', include(expofilesurls)), 
+    url(r'^expofiles', include(expofilesurls)), 
     
     url(r'^troggle$',    views_other.frontpage,      name="frontpage"), # control panel. Shows recent actions.
     url(r'^caves$',      views_caves.caveindex,      name="caveindex"),
@@ -110,7 +110,6 @@ actualurlpatterns = [
     url(r'^cave/(?P<cave_id>[^/]+)/qm\.csv/?$', views_other.downloadQMs, name="downloadqms"),
     url(r'^downloadqms$', views_other.downloadQMs),
 
-
 #   url(r'^map/', .........), # Intercepted by Apache. Yields OpenStreetMap. Redirects to expoweb/map
 
     url(r'^survexfile/(?P<survex_file>.*?)\.svx$', views_survex.svx,        name="svx"),
@@ -121,22 +120,20 @@ actualurlpatterns = [
     url(r'^survexfile/caves/$',                       views_survex.survexcaveslist,  name="survexcaveslist"),
     url(r'^survexfile/(?P<survex_cave>.*)$',          views_survex.survexcavesingle, name="survexcavessingle"),
 
-#   url(r'^survey_files/download/(?P<path>.*)$',      view_surveys.download), # needs rewriting
-
     url(r'^survey_scans/$',                           view_surveys.surveyscansfolders, name="surveyscansfolders"), 
     url(r'^survey_scans/(?P<path>[^/]+)/$',           view_surveys.surveyscansfolder,  name="surveyscansfolder"),
     url(r'^survey_scans/(?P<path>[^/]+)/(?P<file>[^/]+)$', 
                                                       view_surveys.surveyscansingle,   name="surveyscansingle"), 
 
     url(r'^tunneldata/$',                             view_surveys.tunneldata,         name="tunneldata"), 
-    url(r'^tunneldataraw/(?P<path>.+?\.xml)$',        view_surveys.tunnelfile,         name="tunnelfile"), 
+    url(r'^tunneldataraw/(?P<path>.+?\.xml)$',        view_surveys.tunnelfilesingle,   name="tunnelfile"), 
 #   url(r'^tunneldatainfo/(?P<path>.+?\.xml)$',       view_surveys.tunnelfileinfo,     name="tunnelfileinfo"), 
     url(r'^tunneldataraw/(?P<path>.+?\.xml)/upload$', view_surveys.tunnelfileupload,   name="tunnelfileupload"), 
 
     url(r'^prospecting/(?P<name>[^.]+).png$', prospecting_image, name="prospecting_image"),
     
   
-#   url(r'^javascript/(?P<filepath>.*)$', view_surveys.cssfilessingle,  name="single"), # JSLIB_URL - Apache: Alias /javascript /usr/share/javascript
+#   url(r'^javascript/(?P<filepath>.*)$', view_surveys.expofilessingle,  name="single"), # JSLIB_URL - Apache: Alias /javascript /usr/share/javascript
 
 #   static views not working, removed as a plugin. Use apache instead to serve these:
 #   url(r'^photos/(?P<path>.*)$', staticviews.serve,
@@ -144,17 +141,17 @@ actualurlpatterns = [
 #   url(r'^gallery/(?P<path>.*)$', staticviews.serve,
 #            {'document_root': settings.PHOTOS_ROOT, 'show_indexes':True}),
  
-#   url(r'^site_media/(?P<filepath>.*)$', view_surveys.cssfilessingle,  name="single"), # MEDIA_ROOT: CSS and JS 
+#   url(r'^site_media/(?P<filepath>.*)$', view_surveys.expofilessingle,  name="single"), # MEDIA_ROOT: CSS and JS 
     url(r'^(site_media/.*)$',   flatviews.flatpage, name="flatpage"), # MEDIA_ROOT: CSS and JS 
 
-#   url(r'^static/(?P<filepath>.*)$',     view_surveys.cssfilessingle,  name="single"), # MEDIA_ROOT: CSS and JS
+#   url(r'^static/(?P<filepath>.*)$',     view_surveys.expofilessingle,  name="single"), # MEDIA_ROOT: CSS and JS
     url(r'^(static/.*)$',       flatviews.flatpage, name="flatpage"), # STATIC: CSS and JS 
 
     url(r'^(.*)_edit$',         flatviews.editflatpage, name="editflatpage"),
-    url(r'^(.*)$',              flatviews.flatpage,     name="flatpage"), # files assumed relative to EXPOWEB
+    url(r'^(.*)$',              flatviews.flatpage,     name="flatpage"), # CATCHALL assumed relative to EXPOWEB, some expofiles getting here..
 ]
 
-#Allow prefix to all urls
+#Allow DIR_ROOT prefix to all urls
 urlpatterns = [
     url('^%s' % settings.DIR_ROOT, include(actualurlpatterns))
 ]