import subprocess from pathlib import Path from django import forms from django.core.files.storage import FileSystemStorage from django.shortcuts import render import settings from troggle.core.models.survex import DrawingFile # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* # from troggle import settings from .auth import login_required_if_public # from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt """File upload 'views' """ todo = """ - Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import - Need to validate uploaded file as being a valid image file, not a dubious script or hack - Write equivalent GPX upload form system, similar to walletedit() but in expofiles/gpslogs/ Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack - Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says tunnel is only mostly correct XML - Validate image files using a magic recogniser in walletedit() https://pypi.org/project/reportlab/ or https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file - Enable folder creation in dwguploads or as a separate form """ class FilesForm(forms.Form): # not a model-form, just a form-form uploadfiles = forms.FileField() class FilesRenameForm(forms.Form): # not a model-form, just a form-form uploadfiles = forms.FileField() renameto = forms.CharField(strip=True, required=False) class TextForm(forms.Form): # not a model-form, just a form-form photographer = forms.CharField(strip=True) @login_required_if_public def photoupload(request, folder=None): """Upload photo image files into /expofiles/photos/// This does NOT use a Django model linked to a Django form. Just a simple Django form. You will find the Django documentation on forms very confusing, This is simpler. When uploading from a phone, it is useful to be able to rename the file to something meaningful as this is difficult to do on a phone. Previously we had assumed files would be renamed to something useful before starting the upload. Unfortunately this only works when uploading one file at at time , inevitable once you think about it. """ year = settings.PHOTOS_YEAR filesaved = False actual_saved = [] context = {"year": year, "placeholder": "AnathemaDevice"} yearpath = Path(settings.PHOTOS_ROOT, year) if folder == str(year) or folder == str(year) + "/": folder = None if folder is None: folder = "" # improve this later dirpath = Path(settings.PHOTOS_ROOT, year) urlfile = f"/expofiles/photos/{year}" urldir = f"/photoupload/{year}" else: # it will contain the year as well as the photographer dirpath = Path(settings.PHOTOS_ROOT, folder) if dirpath.is_dir(): urlfile = f"/expofiles/photos/{folder}" urldir = Path("/photoupload") / folder else: folder = "" # improve this later dirpath = Path(settings.PHOTOS_ROOT, year) urlfile = f"/expofiles/photos/{year}" urldir = f"/photoupload/{year}" form = FilesRenameForm() formd = TextForm() if request.method == "POST": if "photographer" in request.POST: formd = TextForm(request.POST) if formd.is_valid(): newphotographer = request.POST["photographer"] try: (yearpath / newphotographer).mkdir(exist_ok=True) except: message = f'\n !! Permissions failure ?! 0 attempting to mkdir "{(yearpath / newphotographer)}"' print(message) return render(request, "errors/generic.html", {"message": message}) else: form = FilesRenameForm(request.POST, request.FILES) if form.is_valid(): f = request.FILES["uploadfiles"] multiple = request.FILES.getlist("uploadfiles") # NO CHECK that the files being uploaded are image files fs = FileSystemStorage(dirpath) renameto = request.POST["renameto"] actual_saved = [] if multiple: if len(multiple) == 1: if renameto != "": try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(renameto, content=f) except: print( f'\n !! Permissions failure ?! 1 attempting to save "{f.name}" in "{dirpath}" {renameto=}' ) if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) filesaved = True else: # multiple is the uploaded content try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(f.name, content=f) except: print( f'\n !! Permissions failure ?! 2 attempting to save "{f.name}" in "{dirpath}" {renameto=}' ) if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) filesaved = True else: # multiple is a list of content for f in multiple: try: # crashes in Django os.chmod call if on WSL, but does save file! saved_filename = fs.save(f.name, content=f) except: print( f'\n !! Permissions failure ?! 3 attempting to save "{f.name}" in "{dirpath}" {renameto=}' ) if "saved_filename" in locals(): if saved_filename.is_file(): actual_saved.append(saved_filename) filesaved = True files = [] dirs = [] try: for f in dirpath.iterdir(): if f.is_dir(): dirs.append(f.name) if f.is_file(): files.append(f.name) except FileNotFoundError: files.append("(no folder yet - would be created)") if len(files) > 0: files = sorted(files) if dirs: dirs = sorted(dirs) return render( request, "photouploadform.html", { "form": form, **context, "urlfile": urlfile, "urldir": urldir, "folder": folder, "files": files, "dirs": dirs, "filesaved": filesaved, "actual_saved": actual_saved, }, ) @login_required_if_public def dwgupload(request, folder=None, gitdisable="no"): """Upload DRAWING files (tunnel or therion) into the upload folder in :drawings AND registers it into the :drawings: git repo. This does NOT use a Django model linked to a Django form. Just a simple Django form. You will find the Django documentation on forms very confusing, This is simpler. We could validate the uploaded files as being a valid files using an XML parser, not a dubious script or hack We use get_or_create instead of simply creating a new object in case someone uploads the same file several times in one session, and expects them to be overwritten in the database. Although the actual file will be duplicated in the filesystem with different random name ending. """ def dwgvalid(name): if name in [ ".gitignore", ]: return False if Path(name).suffix.lower() in [".xml", ".th", ".th2", "", ".svg", ".txt"]: return True # dangerous, we should check the actual file binary signature return False def dwgvaliddisp(name): """OK to display, even if we are not going to allow a new one to be uploaded""" if name in [ ".gitignore", ]: return False if Path(name).suffix.lower() in [ ".xml", ".th", ".th2", "", ".svg", ".txt", ".jpg", ".jpeg", ".png", ".pdf", ".top", ".topo", ]: return True # dangerous, we should check the actual file binary signature return False filesaved = False actual_saved = [] refused = [] doesnotexist = "" # print(f'! - FORM dwgupload - start "{folder}" - gitdisable "{gitdisable}"') if folder is None: folder = "" # improve this later dirpath = Path(settings.DRAWINGS_DATA) urlfile = "/dwgdataraw" urldir = "/dwgupload" else: dirpath = Path(settings.DRAWINGS_DATA, folder) urlfile = Path("/dwgdataraw/") / folder urldir = Path("/dwgupload/") / folder form = FilesForm() if request.method == "POST": form = FilesForm(request.POST, request.FILES) if form.is_valid(): # print(f'! - FORM dwgupload - POST valid: "{request.FILES["uploadfiles"]}" ') f = request.FILES["uploadfiles"] multiple = request.FILES.getlist("uploadfiles") savepath = Path(settings.DRAWINGS_DATA, folder) fs = FileSystemStorage(savepath) actual_saved = [] refused = [] # GIT see also core/views/expo.py editexpopage() # GIT see also core/models/cave.py writetrogglefile() if gitdisable != "yes": # set in url 'dwguploadnogit/' git = settings.GIT else: git = "echo" # print(f'git DISABLED {f.name}') if multiple: for f in multiple: # print(f'! - FORM dwgupload - file {f} in {multiple=}') if dwgvalid(f.name): try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file! saved_filename = fs.save(f.name, content=f) except: print( f'! - FORM dwgupload - \n!! Permissions failure ?! on attempting to save file "{f.name}" in "{savepath}". Attempting to continue..' ) if "saved_filename" in locals(): if Path(dirpath, saved_filename).is_file(): actual_saved.append(saved_filename) if gitdisable != "yes": dr_add = subprocess.run( [git, "add", saved_filename], cwd=dirpath, capture_output=True, text=True ) msgdata = ( dr_add.stderr + "\n" + dr_add.stdout + "\nreturn code: " + str(dr_add.returncode) ) # message = f'! - FORM dwgupload - Success: git ADD on server for this file {saved_filename}.' + msgdata # print(message) if dr_add.returncode != 0: msgdata = ( "Ask a nerd to fix this.\n\n" + dr_add.stderr + "\n\n" + dr_add.stdout + "\n\nreturn code: " + str(dr_add.returncode) ) message = ( f"! - FORM dwgupload - CANNOT git ADD on server for this file {saved_filename}. Edits saved but not added to git.\n" + msgdata ) print(message) return render(request, "errors/generic.html", {"message": message}) dwgfile, created = DrawingFile.objects.get_or_create( dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size ) dwgfile.save() else: message = f"! - FORM dwgupload - NOT A FILE {Path(dirpath, saved_filename)=}. " print(message) else: message = f"! - FORM dwgupload - Save failure for {f.name}. Changes NOT saved." print(message) return render(request, "errors/generic.html", {"message": message}) if saved_filename != f.name: # message = f'! - FORM dwgupload - Save RENAME {f.name} renamed as {saved_filename}. This is OK.' # print(message) pass else: refused.append(f.name) # print(f'REFUSED {f.name}') if actual_saved: filesaved = True if len(actual_saved) > 1: dots = "..." else: dots = "" if gitdisable != "yes": dr_commit = subprocess.run( [git, "commit", "-m", f"Drawings upload - {actual_saved[0]}{dots}"], cwd=dirpath, capture_output=True, text=True, ) # message = f'! - FORM dwgupload - For uploading {actual_saved[0]}{dots}. Edits saved, added to git, and COMMITTED.\n' + msgdata # print(message) # This produces return code = 1 if it commits OK if dr_commit.returncode != 0: msgdata = ( "Ask a nerd to fix this.\n\n" + dr_commit.stderr + "\n" + dr_commit.stdout + "\nreturn code: " + str(dr_commit.returncode) ) message = ( f"! - FORM dwgupload -Error code with git on server for this {actual_saved[0]}{dots}. Edits saved, added to git, but NOT committed.\n" + msgdata ) print(message) return render(request, "errors/generic.html", {"message": message}) else: print(f' git disabled "{git=}"') else: # maybe all were refused by the suffix test in dwgvalid() message = f"! - FORM dwgupload - Nothing actually saved. All were refused. {actual_saved=}" print(message) files = [] dirs = [] # print(f'! - FORM dwgupload - start {folder=} \n"{dirpath=}" \n"{dirpath.parent=}" \n"{dirpath.exists()=}"') try: for f in dirpath.iterdir(): if f.is_dir(): if f.name not in [".git"]: dirs.append(f.name) continue if f.is_file(): if dwgvaliddisp(f.name): files.append(f.name) continue except FileNotFoundError: doesnotexist = True if files: files = sorted(files) if dirs: dirs = sorted(dirs) return render( request, "dwguploadform.html", { "form": form, "doesnotexist": doesnotexist, "urlfile": urlfile, "urldir": urldir, "folder": folder, "files": files, "dirs": dirs, "filesaved": filesaved, "actual_saved": actual_saved, "refused": refused, }, )