import os import re from pathlib import Path from sys import getfilesystemencoding as sys_getfilesystemencoding from urllib.parse import unquote as urlunquote from urllib.parse import urljoin import django.forms as forms from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import redirect, render from django.urls import reverse from django.views.decorators.csrf import ensure_csrf_cookie import troggle.core.views.caves import troggle.settings as settings from troggle.core.models.caves import Cave from troggle.core.utils import WriteAndCommitError, write_and_commit, current_expo from troggle.core.views.editor_helpers import HTMLarea from troggle.core.views.uploads import edittxtpage from .auth import login_required_if_public """Formerly a separate package called 'flatpages' written by Martin Green 2011. This was NOT django.contrib.flatpages which stores HTML in the database, so the name was changed to expopages. Then it was incorporated into troggle directly, rather than being an unnecessary external package. This is a succession of hacks and needs to be redisgned and refactored. """ default_head = """<head> <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> <title>CUCC Expedition - index</title> <link rel="stylesheet" type="text/css" href="../css/main2.css" /> <link rel="stylesheet" type="text/css" href="../../css/main2.css" /> <link rel="stylesheet" type="text/css" href="../../../css/main2.css" /> </head> <body> <h1>Expo</h1> <h2 id="tophead">CUCC Expedition</h2> <ul id="links"> <li><a href="/index.htm">Home</a></li> <li><a href="/infodx.htm">Main Index</a></li> <li><a href="/handbook/index.htm">Handbook</a></li> <li><a href="/handbook/computing/onlinesystems.html">Online systems</a></li> <li><a href="/pubs.htm">Reports</a></li> <li><a href="/areas.htm">Areas</a></li> <li><a href="/caves">Caves</a></li> <li><a href="/expedition/2019">Troggle</a></li> <li><form name=P method=get action="/search" target="_top"> <input id="omega-autofocus" type=search name=P size=8 autofocus> <input type=submit value="Search"></li> </ul>""" # this gets overwritten by templates/menu.html by django for most normal pages def expofiles_redirect(request, filepath): """This is used only when running as a test system without a local copy of /expofiles/ when settings.EXPOFILESREMOTE is True """ return redirect(urljoin("http://expo.survex.com/expofiles/", filepath)) def spider(request, _): # urls ending in "_edit_edit" return render(request, "pagenotfound.html", {"path": path}, status=404) # return redirect("/?#") # so that suffixes applied by spider are no longer part of the url def map(request): """Serves unadorned the expoweb/map/slippy/map.html file""" fn = Path(settings.EXPOWEB, "map", "slippy", "map.html") return HttpResponse(content=open(fn, "r"), content_type="text/html") def mapfile(request, path): """Serves unadorned file: everything in the /map/... folder tree""" fn = Path(settings.EXPOWEB, "map", path) print(f"MAP cuttout. \n{path=}\n{fn=} mime:{getmimetype(fn)}") return HttpResponse(content=open(fn, "r"), content_type=getmimetype(fn)) def expofilessingle(request, filepath): """sends a single binary file to the user, if not found, show the parent directory If the path actually is a directory, then show that. """ # print(f' - expofilessingle {filepath}') if filepath == "" or filepath == "/": return expofilesdir(request, settings.EXPOFILES, "") fn = urlunquote(filepath) fn = Path(settings.EXPOFILES, filepath) if fn.is_dir(): return expofilesdir(request, Path(fn), Path(filepath)) if fn.is_file(): return HttpResponse(content=open(fn, "rb"), content_type=getmimetype(filepath)) # any file else: # not a file, so show parent directory - DANGER need to check this is limited to below expofiles if Path(fn).parent == Path(settings.EXPOFILES).parent: return expofilesdir(request, Path(settings.EXPOFILES), Path(filepath).parent) else: return expofilesdir(request, Path(fn).parent, Path(filepath).parent) def expofilesdir(request, dirpath, filepath): """does a directory display. If there is an index.html file we should display that. - dirpath is a full Path() resolved including local machine /expofiles/ - filepath is a Path() and it does not have /expofiles/ in it """ # print(f' - expofilesdir {dirpath} settings.EXPOFILESREMOTE: {settings.EXPOFILESREMOTE}') if filepath: urlpath = "expofiles" / Path(filepath) else: urlpath = Path("expofiles") try: for f in dirpath.iterdir(): pass except FileNotFoundError: # print(f' - expofilesdir error {dirpath}') return expofilesdir(request, dirpath.parent, filepath.parent) fileitems = [] diritems = [] for f in dirpath.iterdir(): if f.is_dir(): diritems.append((urlpath / f.parts[-1], str(f.parts[-1]))) else: # if f.parts[-1].lower() == 'index.htm' or f.parts[-1].lower() == 'index.html': # css cwd problem # return HttpResponse(content=open(f, "rb"),content_type=getmimetype(filepath)) # any file # return expofilessingle(request, str(Path(filepath / f.parts[-1]))) fileitems.append((Path(urlpath) / f.parts[-1], str(f.parts[-1]), getmimetype(f))) return render( request, "dirdisplay.html", {"filepath": urlpath, "fileitems": fileitems, "diritems": diritems, "settings": settings}, ) def pubspage(request): """This is a single-page function, specifically to cope with the updated "current year" which is hard-coded into the HTML of the page. This is a placeholder, we need to do something like what is done in indexpage() below... """ expowebpath = Path(settings.EXPOWEB) print("--redirecting from pubspage()") return expowebpage(request, expowebpath, "pubs.htm") def indexpage(request): """This is a single-page function, specifically to cope with the updated "current year" which is hard-coded into the HTML of the page. This edits the most recent year on the fly.. This now works. """ print("--redirecting from indexpage()") subpath = Path(settings.EXPOWEB) / "index00.htm" with open(subpath, "r") as o: html = o.read() html = html.replace('2xxx', f"{current_expo()}") content_type = "text/html" return HttpResponse(html, content_type=content_type) def expowebpage(request, expowebpath, path): """Adds menus and serves an HTML page""" if path == "" or path == "index.htm": return indexpage(request) if not os.path.isfile(expowebpath / path): # Should not get here if the path has suffix "_edit" print(f" - 404 error in expowebpage() {path}") return render(request, "pagenotfound.html", {"path": path}, status=404) # print(f' - {sys_getfilesystemencoding()=}') if sys_getfilesystemencoding() != "utf-8": return HttpResponse( default_head + "<h3>UTF-8 Parsing Failure:<br>Default file encoding on this Troggle installation is not UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please Please reconfigure Debian/Apache/Django to fix this, i.e. contact Wookey. </body" ) # This next bit can be drastically simplified now that we know that the system encoding actually is utf-8 try: with open(expowebpath / path, "r", encoding="utf-8") as o: html = o.read() except: # exception raised on debian with python 3.9.2 but not on WSL Ubuntu with python 3.9.5 # because debian was assuming default text encoding was 'ascii'. Now specified explicitly so should be OK try: with open(expowebpath / path, "rb") as o: html = str(o.read()).replace("<h1>", "<h1>BAD NON-UTF-8 characters here - ") html = html.replace("\\n", "\n") html = html.replace("\\r", "") html = html.replace("\\t", "\t") html = html.replace("\\'", "'") except: return HttpResponse( default_head + "<h3>UTF-8 Parsing Failure:<br>Page could not be parsed using UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please edit this <var>:expoweb:</var> page to replace dubious umlauts and £ symbols with correct HTML entities e.g. <em>&pound;;</em>. </body" ) m = re.search( r"(.*)<\s*head([^>]*)>(.*)<\s*/head\s*>(.*)<\s*body([^>]*)>(.*)<\s*/body\s*>(.*)", html, re.DOTALL + re.IGNORECASE, ) if m: preheader, headerattrs, head, postheader, bodyattrs, body, postbody = m.groups() else: return HttpResponse( default_head + html + "<h3>HTML Parsing failure:<br>Page could not be parsed into header and body:<br>failure detected in expowebpage in views.expo.py</h3> Please edit this <var>:expoweb:</var> page to be in the expected full HTML format </body" ) m = re.search(r"<title>(.*)</title>", head, re.DOTALL + re.IGNORECASE) if m: (title,) = m.groups() else: title = "" m = re.search(r"noedit", head, re.DOTALL + re.IGNORECASE) if m: editable = False # print(f"NOEDIT set") else: editable = os.access(expowebpath / path, os.W_OK) # are file permissions writeable? # print(f"EDITABLE ? {editable}\n{head}") has_menu = False menumatch = re.match(r'(.*)<ul id="links">', body, re.DOTALL + re.IGNORECASE) if menumatch: has_menu = False # Determine which caves this page relates to m = re.search(r"(162\d\/[^\/]+)[\/\.]", path, re.DOTALL + re.IGNORECASE) if m: (path_start,) = m.groups() parent_caves = Cave.objects.filter(url__startswith=path_start) else: parent_caves = None # Determine if this page relates to a particular year m = re.search(r"years\/(\d\d\d\d)\/.*", path, re.DOTALL + re.IGNORECASE) if m: (year,) = m.groups() else: year = None # Determine if this page is part of the handbook handbook = path.startswith("handbook") return render( request, "expopage.html", { "editable": editable, "path": path, "title": title, "body": body, "homepage": (path == "index.htm"), "has_menu": has_menu, "year": year, "handbook": handbook, "parent_caves": parent_caves, }, ) def mediapage(request, subpath=None, doc_root=None): """This is for special prefix paths /photos/ /site_media/, /static/ etc. as defined in urls.py . If given a directory, gives a failure page. """ #print("mediapath", subpath) if doc_root is not None: filetobeopened = Path(doc_root, subpath) if filetobeopened.is_dir(): return render(request, "nodirlist.html", {"path": subpath}) try: return HttpResponse(content=open(filetobeopened, "rb"), content_type=getmimetype(subpath)) except IOError: return render(request, "pagenotfound.html", {"path": subpath}, status=404) else: return render(request, "pagenotfound.html", {"path": subpath}, status=404) def expopage(request, path): """Either renders an HTML page from expoweb with all the menus, or serves an unadorned binary file with mime type """ # print(" - EXPOPAGES delivering the file: '{}':{} as MIME type: {}".format(request.path, path,getmimetype(path)),flush=True) if path.startswith("noinfo") and settings.PUBLIC_SITE and not request.user.is_authenticated: return HttpResponseRedirect(urljoin(reverse("expologin"), f"?next={request.path}")) if path.startswith("admin/"): # don't even attempt to handle these sorts of mistakes return HttpResponseRedirect("/admin/") expowebpath = Path(settings.EXPOWEB) if path == "": return expowebpage(request, expowebpath, "index.htm") if path.endswith(".htm") or path.endswith(".html"): return expowebpage(request, expowebpath, path) if Path(expowebpath / path).is_dir(): for p in ["index.html", "index.htm"]: if (expowebpath / path / p).is_file(): # This needs to reset the path to the new subdirectory return HttpResponseRedirect("/" + str(Path(path) / p)) return render(request, "pagenotfound.html", {"path": Path(path) / "index.html"}, status=404) if path.endswith("/"): # we already know it is not a directory. # the final / may have been appended by middleware if there was no page without it # do not redirect to a file path without the slash as we may get in a loop. Let the user fix it: return render(request, "dirnotfound.html", {"path": path, "subpath": path[0:-1]}) # So it must be a file in /expoweb/ but not .htm or .html probably an image, maybe a txt file filetobeopened = expowebpath / path # print(f' - {sys_getfilesystemencoding()=}') if sys_getfilesystemencoding() != "utf-8": return HttpResponse( default_head + "<h3>UTF-8 Parsing Failure:<br>Default file encoding on this Troggle installation is not UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please Please reconfigure Debian/Apache/Django to fix this, i.e. contact Wookey. </body" ) try: content = open(filetobeopened, "rb") content_type = getmimetype(path) return HttpResponse(content=content, content_type=content_type) except IOError: return render(request, "pagenotfound.html", {"path": path}, status=404) def getmimetype(path): """Our own version rather than relying on what is provided by the python library. Note that when Apache or nginx is used to deliver /expofiles/ it will use it's own idea of mimetypes and not these. This sets the Content Type in the HTTP header, e.g. Content-Type: text/html; charset=utf-8 """ path = str(path) if path.lower().endswith("gpx"): return "application/octet-stream" if path.lower().endswith("kml"): return "application/octet-stream" if path.lower().endswith(".css"): return "text/css" if path.lower().endswith(".txt"): return "text/css" if path.lower().endswith(".js"): return "application/javascript" if path.lower().endswith(".json"): return "application/javascript" if path.lower().endswith(".ico"): return "image/vnd.microsoft.icon" if path.lower().endswith(".png"): return "image/png" if path.lower().endswith(".tif"): return "image/tif" if path.lower().endswith(".gif"): return "image/gif" if path.lower().endswith(".jpeg"): return "image/jpeg" if path.lower().endswith(".jpg"): return "image/jpeg" if path.lower().endswith("svg"): return "image/svg+xml" if path.lower().endswith("xml"): return "application/xml" # we use "text/xml" for tunnel files if path.lower().endswith(".pdf"): return "application/pdf" if path.lower().endswith(".ps"): return "application/postscript" if path.lower().endswith(".svx"): return "application/x-survex-svx" if path.lower().endswith(".3d"): return "application/x-survex-3d" if path.lower().endswith(".pos"): return "application/x-survex-pos" if path.lower().endswith(".err"): return "application/x-survex-err" if path.lower().endswith(".odt"): return "application/vnd.oasis.opendocument.text" if path.lower().endswith(".ods"): return "application/vnd.oasis.opendocument.spreadsheet" if path.lower().endswith(".docx"): return "application/vnd.openxmlformats-officedocument.wordprocessingml.document" if path.lower().endswith(".xslx"): return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" if path.lower().endswith(".gz"): return "application/x-7z-compressed" if path.lower().endswith(".7z"): return "application/x-7z-compressed" if path.lower().endswith(".zip"): return "application/zip" return "" @login_required_if_public @ensure_csrf_cookie def editexpopage(request, path): """Manages the 'Edit this Page' capability for expo handbook and other html pages. Relies on HTML5 or javascript to provide the in-browser editing environment. """ try: # if a cave not a webpage at all. r = Cave.objects.get(url=path) return troggle.core.views.caves.editCave(request, r.cave.slug) except Cave.DoesNotExist: pass print(f" - {sys_getfilesystemencoding()=}") if sys_getfilesystemencoding() != "utf-8": return HttpResponse( default_head + "<h3>UTF-8 Parsing Failure:<br>Default file encoding on this Troggle installation is not UTF-8:<br>failure detected in expowebpage in views.expo.py</h3> Please Please reconfigure Debian/Apache/Django to fix this, i.e. contact Wookey. </body" ) # detect if this is a text file for editing filepath = Path(settings.EXPOWEB) / path if filepath.suffix in [".txt", ".sty", ".tex"] or filepath.name == "Makefile": print("Editing TEXT file", filepath) return edittxtpage(request, path, filepath) try: filepath = Path(settings.EXPOWEB) / path o = open(filepath, "r", encoding="utf8") html = o.read() autogeneratedmatch = re.search( r"\<\!--\s*(.*?(Do not edit|It is auto-generated|NOEDIT).*?)\s*--\>", html, re.DOTALL + re.IGNORECASE ) if autogeneratedmatch: message = "\tThis page is either auto-generated, and so cannot be edited,\n\t\tor it is too complex to allow users to edit it safely. \n\n\t\tA safer means of updating this page is on the programmers' to-do list." return render(request, "errors/notice.html", {"message": message}) m = re.search(r"(.*)<head([^>]*)>(.*)</head>(.*)<body([^>]*)>(.*)</body>(.*)", html, re.DOTALL + re.IGNORECASE) if m: filefound = True preheader, headerargs, head, postheader, bodyargs, body, postbody = m.groups() # linksmatch = re.match(r'(.*)(<ul\s+id="links">.*)', body, re.DOTALL + re.IGNORECASE) # if linksmatch: # body, links = linksmatch.groups() else: return HttpResponse( default_head + html + "<h3>HTML Parsing failure:<br>Page could not be parsed into header and body:<br>failure detected in expowebpage in views.expo.py</h3> Please edit this <var>:expoweb:</var> page to be in the expected full HTML format .</body>" ) except IOError: print("### File not found ### ", filepath) filefound = False if request.method == "POST": # If the form has been submitted... pageform = ExpoPageForm(request.POST) # A form bound to the POST data if pageform.is_valid(): # Form valid therefore write file # print("### \n", str(pageform)[0:300]) # print("### \n csrfmiddlewaretoken: ",request.POST['csrfmiddlewaretoken']) if filefound: headmatch = re.match(r"(.*)<title>.*</title>(.*)", head, re.DOTALL + re.IGNORECASE) if headmatch: head = ( headmatch.group(1) + "<title>" + pageform.cleaned_data["title"] + "</title>" + headmatch.group(2) ) else: head = "<title>" + pageform.cleaned_data["title"] + "</title>" else: head = "<title>" + pageform.cleaned_data["title"] + "</title>" preheader = "<html>" headerargs = "" postheader = "" bodyargs = "" postbody = "</html>\n" body = pageform.cleaned_data["html"] body = body.replace("\r", "") result = f"{preheader}<head{headerargs}>{head}</head>{postheader}<body{bodyargs}>\n{body}</body>{postbody}" if not filefound or result != html: # Check if content changed at all try: change_message = pageform.cleaned_data["change_message"] write_and_commit([(filepath, result, "utf-8")], f"{change_message} - online edit of {path}") except WriteAndCommitError as e: return render(request, "errors/generic.html", {"message": e.message}) return HttpResponseRedirect(reverse("expopage", args=[path])) # Redirect after POST else: if filefound: m = re.search(r"<title>(.*)</title>", head, re.DOTALL + re.IGNORECASE) if m: (title,) = m.groups() else: title = "" pageform = ExpoPageForm(initial={"html": body, "title": title}) else: pageform = ExpoPageForm() return render( request, "editexpopage.html", { "path": path, "form": pageform, }, ) class ExpoPageForm(forms.Form): """The form used by the editexpopage function""" title = forms.CharField( widget=forms.TextInput(attrs={"size": "60", "placeholder": "Enter title (displayed in tab)"}) ) html = forms.CharField( widget=HTMLarea( attrs={"height": "80%", "rows": 20, "placeholder": "Enter page content (using HTML)"}, preview=True ), required=False, ) change_message = forms.CharField( widget=forms.Textarea( attrs={"cols": 80, "rows": 3, "placeholder": "Describe the change made (for version control records)"} ) )