More security, middleware upgrade, dj-reg.2.5

core/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2020-06-20 17:43
+# Generated by Django 1.11.29 on 2020-06-20 21:27
 from __future__ import unicode_literals
 
 from django.conf import settings

core/view_surveys.py

@@ -1,13 +1,16 @@
-from django.conf import settings
-#from . import fileAbstraction
-from django.shortcuts import render_to_response
-from django.http import HttpResponse, Http404
 import os, stat
 import re
-from troggle.core.models_survex import SurvexScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile
-import parsers.surveys
 import urllib.request, urllib.parse, urllib.error
 
+from django.conf import settings
+from django.shortcuts import render_to_response
+from django.http import HttpResponse, Http404
+
+from troggle.core.models_survex import SurvexScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile
+from troggle.flatpages import views as flatviews
+import parsers.surveys
+#from . import fileAbstraction
+
 def fa_readFile(*path):
     try:
         f = open(os.path.join(settings.FILES, *path))
@@ -15,22 +18,13 @@ def fa_readFile(*path):
         f = urllib.request.urlopen(settings.FILES+"download/")
     return f.read()
     
-def getMimeType(extension):
-    try:
-        return {"txt": "text/plain",
-                "html": "text/html",
-                }[extension]
-    except:
-        print("unknown file type")
-        return "text/plain"
-
 def upload(request, path):
     pass
 
 def download(request, path):
     #try:
         
-        return HttpResponse(fa_readFile(path), content_type=getMimeType(path.split(".")[-1]))
+        return HttpResponse(fa_readFile(path), content_type=flatviews.getmimetype(path))
     #except:
     #    raise Http404
 
@@ -82,7 +76,8 @@ def surveyscansfolder(request, path):
 def surveyscansingle(request, path, file):
     survexscansfolder = SurvexScansFolder.objects.get(walletname=urllib.parse.unquote(path))
     survexscansingle = SurvexScanSingle.objects.get(survexscansfolder=survexscansfolder, name=file)
-    return HttpResponse(content=open(survexscansingle.ffile,"rb"), content_type=getMimeType(path.split(".")[-1]))
+    print("SSS {} {} :{}:".format(path, file, flatviews.getmimetype(file)))
+    return HttpResponse(content=open(survexscansingle.ffile,"rb"), content_type=flatviews.getmimetype(file))
     #return render_to_response('survexscansfolder.html', { 'survexscansfolder':survexscansfolder, 'settings': settings })
 
 def expofilessingle(request, filepath):
@@ -92,7 +87,7 @@ def expofilessingle(request, filepath):
 
 def cssfilessingle(request, filepath):
     fn=urllib.parse.unquote(filepath)
-    return HttpResponse(content=open(settings.MEDIA_ROOT+fn,"rb"),content_type="text/css")
+    return HttpResponse(content=open(settings.MEDIA_ROOT+fn,"r"),content_type="text/css")
 
 def surveyscansfolders(request):
     survexscansfolders = SurvexScansFolder.objects.all()

flatpages/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2020-06-20 17:43
+# Generated by Django 1.11.29 on 2020-06-20 21:27
 from __future__ import unicode_literals
 
 from django.db import migrations, models

flatpages/views.py

@@ -14,8 +14,10 @@ import troggle.core.views_caves
 import troggle.settings as settings
 
 def flatpage(request, path):
+    #print(" - FLATPAGES delivering the file: {} as MIME type: {}".format(path,getmimetype(path)))
     try:
         r = Redirect.objects.get(originalURL = path)
+        #print(" - FLATPAGES REDIRECT the file: {} as: {}".format(path,r))
         return HttpResponseRedirect(r.newURL) # Redirect after POST
     except Redirect.DoesNotExist:
         pass
@@ -26,7 +28,7 @@ def flatpage(request, path):
     except Cave.DoesNotExist:
         pass
     except:
-        print(" ! FAILED to get only one cave per slug for: "+path)
+        #print(" ! FAILED to get only one cave per slug for: "+path)
         caves = Cave.objects.all().filter(url = path)
         for c in caves:
             print(path, c.slug())
@@ -46,6 +48,8 @@ def flatpage(request, path):
         return HttpResponseRedirect(reverse("auth_login") + '?next=%s' % request.path)
 
     if path.endswith("/") or path == "":
+        #print(" - FLATPAGES the file: {} ENDSWITH ...".format(path))
+
         try:
             o = open(os.path.normpath(settings.EXPOWEB + path + "index.html"), "rb")
             path = path + "index.html"
@@ -57,9 +61,22 @@ def flatpage(request, path):
                 return render(request, 'pagenotfound.html', {'path': path})
     else:
         try:
+            #print(" - FLATPAGES the file: {}  ...".format(path))
+            if path.startswith("site_media"):
+                #print(" - MEDIA_ROOT: {}  ...".format(settings.MEDIA_ROOT))
+                path = path.replace("site_media", settings.MEDIA_ROOT)
+                filetobeopened = os.path.normpath(path)
+            elif path.startswith("static"):
+                #print(" - STATIC_ROOT: {}  ...".format(settings.MEDIA_ROOT))
+                path = path.replace("static", settings.MEDIA_ROOT)
+                filetobeopened = os.path.normpath(path)
+            else:
                 filetobeopened = os.path.normpath(settings.EXPOWEB + path)
+            #print(" - FLATPAGES full path : {}  ...".format(filetobeopened))
             o = open(filetobeopened, "rb")
+            #print(" - FLATPAGES full path no error: {}  ...".format(filetobeopened))
         except IOError:
+            #print(" - FLATPAGES ERROR: {}  ...".format(filetobeopened))
             return render(request, 'pagenotfound.html', {'path': path})
     if path.endswith(".htm") or path.endswith(".html"):
         html = o.read()
@@ -94,9 +111,12 @@ def flatpage(request, path):
         return render(request, 'flatpage.html', {'editable': editable, 'path': path, 'title': title, 
                     'body': body, 'homepage': (path == "index.htm"), 'has_menu': has_menu})
     else:
+        #print(" - FLATPAGES delivering the file: {} as MIME type: {}".format(path,getmimetype(path)))
         return HttpResponse(o.read(), content_type=getmimetype(path))
 
 def getmimetype(path):
+    if path.lower().endswith(".css"): return "text/css"
+    if path.lower().endswith(".js"): return "application/javascript"
     if path.lower().endswith(".png"): return "image/png"
     if path.lower().endswith(".tif"): return "image/tif"
     if path.lower().endswith(".gif"): return "image/gif"

lines-of-python.txt

@@ -20,26 +20,26 @@
 41 ./localsettingspotatohut.py
 41 ./middleware.py
 44 ./dump.py
-47 ./reset-django.py
 48 ./core/templatetags/survex_markup.py
 49 ./parsers/subcaves.py
+52 ./reset-django.py
 63 ./logbooksdump.py
 69 ./core/TESTS/tests.py
-70 ./urls.py
+69 ./urls.py
 73 ./localsettings.py
 73 ./localsettingsWSL.py
-81 ./settings.py
+86 ./settings.py
 92 ./core/views_statistics.py
 97 ./core/forms.py
 98 ./core/admin.py
 99 ./parsers/QMs.py
 102 ./parsers/people.py
-103 ./core/view_surveys.py
+104 ./core/view_surveys.py
 124 ./core/templatetags/wiki_markup.py
 135 ./utils.py
 150 ./parsers/surveys.py
-156 ./flatpages/views.py
 160 ./core/models_survex.py
+162 ./flatpages/views.py
 164 ./modelviz.py
 167 ./core/models.py
 175 ./core/views_other.py
@@ -52,4 +52,4 @@
 431 ./parsers/survex.py
 450 ./core/models_caves.py
 515 ./parsers/logbooks.py
-5950
+5966

lines-of-templates.txt

@@ -6,6 +6,10 @@
 7 ./templates/admin/base_site.html
 7 ./templates/pagenotfound.html
 8 ./templates/nonpublic.html
+10 ./templates/profiles/create_profile.html
+10 ./templates/profiles/edit_profile.html
+10 ./templates/profiles/profile_detail.html
+10 ./templates/registration/registration_complete.html
 12 ./templates/cave_logbook.html
 13 ./templates/cave_description.html
 13 ./templates/core/expedition_list.html
@@ -22,6 +26,9 @@
 17 ./templates/editcave2.html
 19 ./templates/cavemillenial.html
 19 ./templates/registration/activate.html
+20 ./templates/cave_qms.html
+20 ./templates/cavebase.html
+20 ./templates/plainbase.html
 21 ./templates/logbook2005style.html
 22 ./templates/tasks.html
 23 ./templates/statistics.html
@@ -47,6 +54,7 @@
 62 ./templates/svxfile.html
 64 ./templates/svxcavesingle.html
 69 ./templates/logbookentry.html
+70 ./templates/cave_entrances.html
 71 ./templates/expedition.html
 75 ./templates/base.html
 75 ./templates/entrance.html
@@ -55,4 +63,4 @@
 102 ./templates/prospecting.html
 115 ./templates/controlPanel.html
 439 ./templates/cave.html
-2265
+2435

localsettingsWSL.py

@@ -34,7 +34,7 @@ DATABASES = {
     'default': { 
         'ENGINE': 'django.db.backends.sqlite3', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
         'NAME' : 'troggle.sqlite',
- #       'NAME' : ':memory:',
+#        'NAME' : ':memory:',
         'USER' : 'expo',        # Not used with sqlite3.
         'PASSWORD' : 'sekrit',  # Not used with sqlite3.
         'HOST' : '',            # Set to empty string for localhost. Not used with sqlite3.
@@ -56,7 +56,6 @@ TEMPLATES = [
         'DIRS': [
             PYTHON_PATH + "templates"
         ],
-#        'APP_DIRS': True,
         'OPTIONS': {
             'debug': 'DEBUG',
             'context_processors': [

pre-run.sh

@@ -10,10 +10,14 @@ echo ""
 find . -name \*.html -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v ":0$" | awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-templates.txt
 
 find . -name \*.py -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v ":0$" | grep -v "/migrations/" |grep -v "troggle-inspectdb.py"| awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-python.txt
-echo `tail -1 lines-of-python.txt`   non-comment lines of python.
+
 
 # This deletes the database so must run after generating troggle-inspectdb.py
 python reset-django.py
 echo After cleanup deletion, remake all migrations.
 python manage.py makemigrations >/dev/null
 python manage.py test
+python  manage.py check -v 3 --deploy 2>security-warnings.txt >/dev/null
+python  manage.py check -v 3 --deploy
+echo ""
+echo `tail -1 lines-of-python.txt`   non-comment lines of python.

requirements.txt

@@ -1,6 +1,6 @@
 confusable-homoglyphs==3.2.0
 Django==1.11.29
-django-registration==2.4
+django-registration==2.5
 Pillow==7.1.2
 pytz==2020.1
 six==1.15.0

reset-django.py

@@ -49,14 +49,20 @@ def delete_migrations():
 def delete_sqlite3():
     global base_dir
     
-    db_file = os.path.join(base_dir, "troggle.sqlite3")
+    db_file = os.path.join(base_dir, "troggle.sqlite")
-    
+    #print("troggle.sqlite: {}".format(db_file))
     if os.path.exists(db_file):
+        try:
             os.remove(db_file)
+            print("\n>>> troggle.sqlite: {} DELETED\n".format(db_file))
+        except:
+            print("troggle.sqlite: {} NOT deleted".format(db_file))
 
 
 def main():
     global folders
+    print("base directory used: {}".format(base_dir))
+
 
     try: 
         get_directory_list()

security-warnings.txt

@@ -0,0 +1,9 @@
+System check identified some issues:
+
+WARNINGS:
+?: (security.W004) You have not set a value for the SECURE_HSTS_SECONDS setting. If your entire site is served only over SSL, you may want to consider setting a value and enabling HTTP Strict Transport Security. Be sure to read the documentation first; enabling HSTS carelessly can cause serious, irreversible problems.
+?: (security.W008) Your SECURE_SSL_REDIRECT setting is not set to True. Unless your site should be available over both SSL and non-SSL connections, you may want to either set this setting True or configure a load balancer or reverse-proxy server to redirect all connections to HTTPS.
+?: (security.W018) You should not have DEBUG set to True in deployment.
+?: (security.W019) You have 'django.middleware.clickjacking.XFrameOptionsMiddleware' in your MIDDLEWARE_CLASSES, but X_FRAME_OPTIONS is not set to 'DENY'. The default is 'SAMEORIGIN', but unless there is a good reason for your site to serve other parts of itself in a frame, you should change it to 'DENY'.
+
+System check identified 4 issues (0 silenced).

settings.py

@@ -95,6 +95,11 @@ SMART_APPEND_SLASH = True
 SECRET_KEY = "not-the-real-secret-key-a#vaeozn0---^fj!355qki*vj2"
 
 LOGIN_REDIRECT_URL = '/'
+SECURE_CONTENT_TYPE_NOSNIFF = True
+SECURE_BROWSER_XSS_FILTER = True
+#SESSION_COOKIE_SECURE = True # if enabled, cannot login to Django control panel
+CSRF_COOKIE_SECURE = True
+X_FRAME_OPTIONS = 'SAMEORIGIN'  # change to "DENY" after we eliminate all the iframes in use.
 
 INSTALLED_APPS = (
     'django.contrib.admin',
@@ -102,7 +107,7 @@ INSTALLED_APPS = (
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.messages',
-#    'django.contrib.staticfiles',
+#    'django.contrib.staticfiles', # apparently not working. Using workarounds with flatpages
     'registration',
     'troggle.profiles',
     'troggle.core',
@@ -110,6 +115,7 @@ INSTALLED_APPS = (
 )
 
 MIDDLEWARE_CLASSES = (
+    'django.middleware.security.SecurityMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',

urls.py

@@ -113,8 +113,6 @@ actualurlpatterns = [
     url(r'^prospecting/(?P<name>[^.]+).png$', prospecting_image, name="prospecting_image"),
     
     url(r'^expofiles/(?P<filepath>.*)$',  view_surveys.expofilessingle, name="single"), # EXPOFILES
-    url(r'^static/(?P<filepath>.*)$',     view_surveys.cssfilessingle,  name="single"), # MEDIA_ROOT: CSS and JS
-    url(r'^site_media/(?P<filepath>.*)$', view_surveys.cssfilessingle,  name="single"), # MEDIA_ROOT: CSS and JS 
 #   url(r'^javascript/(?P<filepath>.*)$', view_surveys.cssfilessingle,  name="single"), # JSLIB_URL - unused
 
 #   static views not working, removed as a plugin. Use apache instead to serve these:
@@ -123,8 +121,14 @@ actualurlpatterns = [
 #   url(r'^gallery/(?P<path>.*)$', staticviews.serve,
 #            {'document_root': settings.PHOTOS_ROOT, 'show_indexes':True}),
  
+#   url(r'^site_media/(?P<filepath>.*)$', view_surveys.cssfilessingle,  name="single"), # MEDIA_ROOT: CSS and JS 
+    url(r'^(site_media/.*)$',   flatviews.flatpage, name="flatpage"), # MEDIA_ROOT: CSS and JS 
+
+#   url(r'^static/(?P<filepath>.*)$',     view_surveys.cssfilessingle,  name="single"), # MEDIA_ROOT: CSS and JS
+    url(r'^(static/.*)$',       flatviews.flatpage, name="flatpage"), # STATIC: CSS and JS 
+
     url(r'^(.*)_edit$',         flatviews.editflatpage, name="editflatpage"),
-    url(r'^(.*)$',      flatviews.flatpage,     name="flatpage"),
+    url(r'^(.*)$',              flatviews.flatpage,     name="flatpage"), # files assumed relative to EXPOWEB
 ]
 
 #Allow prefix to all urls