import subprocess from datetime import datetime, timedelta, timezone from pathlib import Path from django import forms from django.core.files.storage import FileSystemStorage from django.http import HttpResponseRedirect from django.shortcuts import redirect, render import settings from troggle.core.models.caves import GetCaveLookup from troggle.core.models.survex import DrawingFile from troggle.core.models.troggle import DataIssue, Expedition, PersonExpedition from troggle.core.utils import ( COOKIE_MAX_AGE, alphabet_suffix, current_expo, get_editor, is_identified_user, git_add, git_commit, git_string, sanitize_name, unique_slug, write_and_commit, ) from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* from .auth import login_required_if_public """File upload 'views' Note that there are other file upload forms in views/wallet_edit.py and views/logbook_edit.py and that core/forms.py contains Django class-based forms for caves and entrances. """ todo = """ - Ideally we should validate uploaded file as being a valid file type, not a dubious script or hack Validate image files using a magic recogniser in walletedit() https://pypi.org/project/reportlab/ or https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file - Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says tunnel is only mostly correct XML, and it does fail at least one XML parser. - parse the uploaded drawing file for links to wallets and scan files as done in parsers/drawings.py - Enable folder creation in dwguploads or as a separate form - Enable file rename on expofiles, not just for /surveyscans/ (aka wallets) - Make file rename utility less ugly. """ class DrawingsFilesForm(forms.Form): # not a model-form, just a form-form uploadfiles = forms.FileField() identified_login = forms.BooleanField(required=False,widget=forms.CheckboxInput(attrs={"onclick":"return false"})) # makes it readonly who_are_you = forms.CharField( widget=forms.TextInput( attrs={"size": 100, "placeholder": "You are editing this page, who are you ? e.g. 'Becka' or 'Animal '", "style": "vertical-align: text-top;"} ) ) class WalletFilesForm(forms.Form): # not a model-form, just a form-form """Used only for uploading to expofiles/surveyscans// which is not a git repo so we do not need an "editor" to assign blame to """ uploadfiles = forms.FileField() class PhotographerForm(forms.Form): # not a model-form, just a form-form photographer = forms.CharField(strip=True) class GPXuploadForm(forms.Form): # not a model-form, just a form-form prospector = forms.CharField(strip=True) class FilesRenameForm(forms.Form): # not a model-form, just a form-form """Used only for renaming photos in /expofiles/photos/ which is not a git repo """ uploadfiles = forms.FileField() renameto = forms.CharField(strip=True, required=False) class ExpofileRenameForm(forms.Form): # not a model-form, just a form-form renameto = forms.CharField(strip=True, required=False) @login_required_if_public def expofilerename(request, filepath): """Rename any single file in /expofiles/ - eventually. Currently this just does files within wallets i.e. in /surveyscans/ and it returns control to the original wallet edit page """ def is_rotatable(path): """If file is a JPG but has no filename extension, then it must be renamed before it can be rotated. """ print(f"{path}: '{Path(path).suffix.lower()}'") if Path(path).suffix.lower() in [".png", ".jpg", ".jpeg"]: return True else: return False def rotate_image(): wallet = str(Path(filepath).parent).lstrip("surveyscans/") cwd = settings.SCANS_ROOT / wallet print(f"ROTATE \n{cwd=} \n{filename=}") mogrify = settings.MOGRIFY rot = subprocess.run( [mogrify, "-rotate", "90", filename], cwd=cwd, capture_output=True, text=True ) msgdata = ( rot.stderr + "\n" + rot.stdout + "\nreturn code: " + str(rot.returncode) ) message = f'! - ROTATE - Success: rotated this file {filename}.' + msgdata print(message) # DataIssue.objects.create(parser="mogrify", message=message) if rot.returncode != 0: msgdata = ( "Ask a nerd to fix this.\n\n" + rot.stderr + "\n\n" + rot.stdout + "\n\nreturn code: " + str(rot.returncode) ) message = ( f"! - ROTATE - CANNOT blurk for this file {filename}. \n" + msgdata ) print(message) DataIssue.objects.create(parser="mogrify", message=message) return simple_get() def simple_get(): form = ExpofileRenameForm() return render( request, "renameform.html", { "form": form, "filepath": filepath, "filename": filename, "filesize": filesize, "files": files, "walletpath": walletpath, "wallet": wallet, "notpics": notpics, "rotatable": rotatable, }, ) if filepath: # using EXPOFILES not SCANS_ROOT in case we want to extend this to other parts of the system actualpath = Path(settings.EXPOFILES) / Path(filepath) else: message = f'\n File to rename not specified "{filepath}"' print(message) return render(request, "errors/generic.html", {"message": message}) if not actualpath.is_file(): message = f'\n File not found when attempting rename "{filepath}"' print(message) return render(request, "errors/generic.html", {"message": message}) else: filename = Path(filepath).name walletpath = Path(filepath).parent wallet = Path(walletpath).name folder = actualpath.parent filesize = f"{actualpath.stat().st_size:,}" rotatable= is_rotatable(filename) if not actualpath.is_relative_to(Path(settings.SCANS_ROOT)): message = f'\n Can only do rename within wallets (expofiles/surveyscans/) currently, sorry. "{actualpath}" ' print(message) return render(request, "errors/generic.html", {"message": message}) files = [] dirs = [] notpics =[] dirpath = actualpath.parent print(f'! - FORM rename expofile - start \n{filepath=} \n{dirpath=} \n{walletpath=}') if dirpath.is_dir(): try: for f in dirpath.iterdir(): if f.is_dir(): for d in f.iterdir(): dirs.append(f"{f.name}/{d.name}") if f.is_file(): if is_rotatable(f.name): # should allow all images here which can be thumsized, not just rotatables. e.g. PDF files.append(f.name) else: notpics.append(f.name) except FileNotFoundError: files.append( "(Error. There should be at least one filename visible here. Try refresh.)" ) if request.method == "GET": return simple_get() elif request.method == "POST": form = ExpofileRenameForm(request.POST) if not form.is_valid(): message = f'Invalid form response for file renaming "{request.POST}"' print(message) return render(request, "errors/generic.html", {"message": message}) if "rotate" in request.POST: return rotate_image() if "rename" in request.POST: if "renametoname" not in request.POST: print("renametoname not in request.POST") # blank filename passed it, so just treat as another GET return simple_get() renameto = sanitize_name(request.POST["renametoname"]) if (folder / renameto).is_file() or (folder / renameto).is_dir(): rename_bad = renameto message = f'\n Cannot rename to an existing file or folder. "{filename}" -> "{(folder / renameto)}"' print(message) return render( request, "renameform.html", { "form": form, "filepath": filepath, "filename": filename, "filesize": filesize, "files": files, "walletpath": walletpath, "wallet": wallet, "notpics": notpics, "rename_bad": rename_bad, }, ) actualpath.rename((folder / renameto)) message = f'\n RENAMED "{filename}" -> "{(folder / renameto)}"' print(message) walletid = actualpath.relative_to(Path(settings.SCANS_ROOT)).parent.stem.replace("#",":") print(walletid) return redirect(f'/survey_scans/{walletid}/') else: # not GET or POST print("UNRECOGNIZED action") return simple_get() @login_required_if_public def photoupload(request, folder=None): """Upload photo image files into /expofiles/photos/// This does NOT use a Django model linked to a Django form. Just a simple Django form. You will find the Django documentation on forms very confusing, This is simpler. When uploading from a phone, it is useful to be able to rename the file to something meaningful as this is difficult to do on a phone. Previously we had assumed files would be renamed to something useful before starting the upload. Unfortunately this only works when uploading one file at at time , inevitable once you think about it. Pending generic file renaming capability more generally. """ year = current_expo() # year = settings.PHOTOS_YEAR filesaved = False actual_saved = [] context = {"year": year, "placeholder": "AnathemaDevice"} yearpath = Path(settings.PHOTOS_ROOT, year) if folder == str(year) or folder == str(year) + "/": folder = None if folder is None: folder = "" # improve this later dirpath = Path(settings.PHOTOS_ROOT, year) urlfile = f"/expofiles/photos/{year}" urldir = f"/photoupload/{year}" else: # it will contain the year as well as the photographer dirpath = Path(settings.PHOTOS_ROOT, folder) if dirpath.is_dir(): urlfile = f"/expofiles/photos/{folder}" urldir = Path("/photoupload") / folder else: folder = "" # improve this later dirpath = Path(settings.PHOTOS_ROOT, year) urlfile = f"/expofiles/photos/{year}" urldir = f"/photoupload/{year}" form = FilesRenameForm() formd = PhotographerForm() if request.method == "POST": if "photographer" in request.POST: # then we are creating a new folder formd = PhotographerForm(request.POST) if formd.is_valid(): newphotographer = sanitize_name(request.POST["photographer"]) try: (yearpath / newphotographer).mkdir(parents=True, exist_ok=True) except: message = f'\n !! Permissions failure ?! 0 attempting to mkdir "{(yearpath / newphotographer)}"' print(message) return render(request, "errors/generic.html", {"message": message}) else: # then we are renaming the file ? form = FilesRenameForm(request.POST, request.FILES) if form.is_valid(): f = request.FILES["uploadfiles"] multiple = request.FILES.getlist("uploadfiles") # NO CHECK that the files being uploaded are image files fs = FileSystemStorage(dirpath) renameto = sanitize_name(request.POST["renameto"]) actual_saved = [] if multiple: if len(multiple) == 1: if renameto != "": try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(renameto, content=f) except: print( f'\n !! Permissions failure ?! 1 attempting to save "{f.name}" in "{dirpath}" {renameto=}' ) if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) filesaved = True else: # multiple is the uploaded content try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(f.name, content=f) except: print( f'\n !! Permissions failure ?! 2 attempting to save "{f.name}" in "{dirpath}" {renameto=}' ) if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) filesaved = True else: # multiple is a list of content for f in multiple: try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(f.name, content=f) except: print( f'\n !! Permissions failure ?! 3 attempting to save "{f.name}" in "{dirpath}" {renameto=}' ) if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) filesaved = True files = [] dirs = [] try: for f in dirpath.iterdir(): if f.is_dir(): dirs.append(f.name) if f.is_file(): files.append(f.name) except FileNotFoundError: files.append("(no folder yet - would be created)") if len(files) > 0: files = sorted(files) if dirs: dirs = sorted(dirs) return render( request, "photouploadform.html", { "form": form, **context, "urlfile": urlfile, "urldir": urldir, "folder": folder, "files": files, "dirs": dirs, "filesaved": filesaved, "actual_saved": actual_saved, }, ) @login_required_if_public def gpxupload(request, folder=None): """Copy of photo upload folder is the "path" """ def gpxvalid(name): if Path(name).suffix.lower() in [".xml", ".gpx"]: return True # dangerous, we should check the actual file binary signature return False print(f"gpxupload() {folder=}") year = current_expo() filesaved = False actual_saved = [] context = {"year": year, "placeholder": "AnathemaDevice"} yearpath = Path(settings.EXPOFILES) / "gpslogs" / year if folder == str(year) or folder == str(year) + "/": folder = None if folder is None: folder = "" # improve this later dirpath = yearpath urlfile = f"/expofiles/gpslogs/{year}" urldir = f"/gpxupload/{year}" else: # it will contain the year as well as the prospector dirpath = Path(settings.EXPOFILES) / "gpslogs" / folder if dirpath.is_dir(): urlfile = f"/expofiles/gpslogs/{folder}" urldir = Path("/gpxupload") / folder else: folder = "" # improve this later dirpath = yearpath urlfile = f"/expofiles/gpslogs/{year}" urldir = f"/gpxupload/{year}" print(f"gpxupload() {folder=} {dirpath=} {urlfile=} {urldir=}") form = FilesRenameForm() formd = GPXuploadForm() print(f"gpxupload() {form=} {formd=} ") if request.method == "POST": print(f"gpxupload() method=POST") for i in request.POST: print(" ",i) if "prospector" in request.POST: print(f"gpxupload() {request.POST=}\n {request.POST['prospector']=}") formd = GPXuploadForm(request.POST) if formd.is_valid(): newprospector = sanitize_name(request.POST["prospector"]) print(f"gpxupload() {newprospector=}") try: (yearpath / newprospector).mkdir(parents=True, exist_ok=True) except: message = f'\n !! Permissions failure ?! 0 attempting to mkdir "{(yearpath / newprospector)}"' print(message) raise return render(request, "errors/generic.html", {"message": message}) else: print(f"gpxupload() no prospector field") print(f"gpxupload() {request.FILES=}") for i in request.FILES: print(" ",i) form = FilesRenameForm(request.POST, request.FILES) print(f"gpxupload() is the FilesRenameForm valid? {form=}") for i in form: print(" ",i) if not form.is_valid(): print(f"gpxupload() Form is not valid {form=}") else: print(f"gpxupload() about to look at request.FILES") f = request.FILES["uploadfiles"] multiple = request.FILES.getlist("uploadfiles") # NO CHECK that the files being uploaded are image files fs = FileSystemStorage(dirpath) renameto = sanitize_name(request.POST["renameto"]) actual_saved = [] if multiple: if len(multiple) == 1: if renameto != "": try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(renameto, content=f) except: print( f'\n !! Permissions failure ?! 1 attempting to save "{f.name}" in "{dirpath}" {renameto=}' ) if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) filesaved = True else: # multiple is the uploaded content try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(f.name, content=f) except: print( f'\n !! Permissions failure ?! 2 attempting to save "{f.name}" in "{dirpath}" {renameto=}' ) if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) filesaved = True else: # multiple is a list of content for f in multiple: if gpxvalid(f.name): try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(f.name, content=f) except: print( f'\n !! Permissions failure ?! 3 attempting to save "{f.name}" in "{dirpath}" {renameto=}' ) if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) filesaved = True else: print(f"gpxupload(): not a GPX file {f.name=}") print(f"gpxupload() drop through") files = [] dirs = [] try: for f in dirpath.iterdir(): if f.is_dir(): dirs.append(f.name) if f.is_file(): files.append(f.name) except FileNotFoundError: files.append("(no folder yet - would be created)") except Exception as e: print(f"gpxupload() EXCEPTION\n {e}") if len(files) > 0: files = sorted(files) if dirs: dirs = sorted(dirs) print(f"gpxupload() about to render..") return render( request, "gpxuploadform.html", { "form": form, **context, "urlfile": urlfile, "urldir": urldir, "folder": folder, "files": files, "dirs": dirs, "filesaved": filesaved, "actual_saved": actual_saved, }, ) @login_required_if_public def dwgupload(request, folder=None, gitdisable="no"): """Upload DRAWING files (tunnel or therion) into the upload folder in :drawings AND registers it into the :drawings: git repo. This does NOT use a Django model linked to a Django form. Just a simple Django form. You will find the Django documentation on forms very confusing, This is simpler. We could validate the uploaded files as being a valid files using an XML parser, not a dubious script or hack, but this won't work on Tunnel files as Tunnel does not produce exactly valid xml (!) We use get_or_create instead of simply creating a new object in case someone uploads the same file several times in one session, and expects them to be overwritten in the database. (Although the actual file will be duplicated in the filesystem with different random name ending, and this will need to be cleaned-up manually by a nerd later.) """ def dwgvalid(name): if name in [ ".gitignore", ]: return False if Path(name).suffix.lower() in [".xml", ".th", ".th2", "", ".svg", ".txt"]: return True # dangerous, we should check the actual file binary signature return False def dwgvaliddisp(name): """OK to display, even if we are not going to allow a new one to be uploaded""" if name in [ ".gitignore", ]: return False if Path(name).suffix.lower() in [ ".xml", ".th", ".th2", "", ".svg", ".txt", ".jpg", ".jpeg", ".png", ".pdf", ".top", ".topo", ]: return True # dangerous, we should check the actual file binary signature return False filesaved = False actual_saved = [] refused = [] doesnotexist = "" # print(f'! - FORM dwgupload - start "{folder}" - gitdisable "{gitdisable}"') if folder is None: folder = "" # improve this later dirpath = Path(settings.DRAWINGS_DATA) urlfile = "/dwgdataraw" urldir = "/dwgupload" else: dirpath = Path(settings.DRAWINGS_DATA, folder) urlfile = Path("/dwgdataraw/") / folder urldir = Path("/dwgupload/") / folder identified_login = is_identified_user(request.user) editor = get_editor(request) form = DrawingsFilesForm() if request.method == "POST": form = DrawingsFilesForm(request.POST, request.FILES) if form.is_valid(): # print(f'! - FORM dwgupload - POST valid: "{request.FILES["uploadfiles"]}" ') editor = form.cleaned_data["who_are_you"] editor = git_string(editor) f = request.FILES["uploadfiles"] multiple = request.FILES.getlist("uploadfiles") savepath = Path(settings.DRAWINGS_DATA, folder) fs = FileSystemStorage(savepath) actual_saved = [] refused = [] # GIT see also core/views/expo.py editexpopage() # GIT see also core/models/cave.py writetrogglefile() if gitdisable != "yes": # set in url 'dwguploadnogit/' git = settings.GIT else: git = "echo" # print(f'git DISABLED {f.name}') # For saving, and then comitting, multiple files, we should be using write_and_commit() # # try: # write_and_commit([(filepath, newtext, "utf-8")], f"Online edit of {path}", editor) # except WriteAndCommitError as e: # return render(request, "errors/generic.html", {"message": e.message}) if multiple: for f in multiple: # print(f'! - FORM dwgupload - file {f} in {multiple=}') if dwgvalid(f.name): try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file! saved_filename = fs.save(f.name, content=f) except: print( f'! - FORM dwgupload - \n!! Permissions failure ?! on attempting to save file "{f.name}" in "{savepath}". Attempting to continue..' ) if "saved_filename" in locals(): filepath = dirpath / saved_filename if filepath.is_file(): actual_saved.append(saved_filename) if gitdisable != "yes": commands = git_add(filepath, dirpath) dwgfile, created = DrawingFile.objects.get_or_create( dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size ) dwgfile.save() else: message = f"! - FORM dwgupload - NOT A FILE {Path(dirpath, saved_filename)=}. " print(message) else: message = f"! - FORM dwgupload - Save failure for {f.name}. Changes NOT saved." print(message) return render(request, "errors/generic.html", {"message": message}) if saved_filename != f.name: # message = f'! - FORM dwgupload - Save RENAME {f.name} renamed as {saved_filename}. This is OK.' # print(message) pass else: refused.append(f.name) # print(f'REFUSED {f.name}') if actual_saved: filesaved = True if len(actual_saved) > 1: dots = f"{len(actual_saved)} files" else: dots = f"{actual_saved[0]}" commit_msg = f"Drawings upload - {dots}" if gitdisable != "yes": git_commit(dirpath, commit_msg, editor, commands) else: # maybe all were refused by the suffix test in dwgvalid() message = f"! - FORM dwgupload - Nothing actually saved. All were refused. {actual_saved=}" print(message) # GET request starts here, also drops through from POST files = [] dirs = [] # print(f'! - FORM dwgupload - start {folder=} \n"{dirpath=}" \n"{dirpath.parent=}" \n"{dirpath.exists()=}"') try: for f in dirpath.iterdir(): if f.is_dir(): if f.name not in [".git"]: dirs.append(f.name) continue if f.is_file(): if dwgvaliddisp(f.name): files.append(f.name) continue except FileNotFoundError: doesnotexist = True if files: files = sorted(files) if dirs: dirs = sorted(dirs) if identified_login: # disable editing the git id string as we get it from the logged-on user data form.fields["who_are_you"].widget.attrs["readonly"]="readonly" response = render( request, "dwguploadform.html", # a bit more primitive than many forms in troggle, everything is very explicit and doesn't use widgets { "form": form, "identified_login": identified_login, "doesnotexist": doesnotexist, "urlfile": urlfile, "urldir": urldir, "folder": folder, "files": files, "dirs": dirs, "filesaved": filesaved, "actual_saved": actual_saved, "refused": refused, "who_are_you": editor, }, ) response.set_cookie('editor_id', editor, max_age=COOKIE_MAX_AGE) # cookie expires after COOKIE_MAX_AGE seconds return response