expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-22 23:31:52 +00:00
Code Issues 19 Projects 3 Activity
93397a774f
troggle/core/views/uploads.py

632 lines
26 KiB
Python
Raw Blame History

import subprocess
import hashlib
from pathlib import Path
from django import forms
from django.core.files.storage import FileSystemStorage
from django.shortcuts import render, redirect
import settings
from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry
from troggle.core.models.survex import DrawingFile
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from .auth import login_required_if_public
"""File upload 'views'
Note that there are other file upload forms in views/wallet_edit.py
and that core/forms.py contains Django class-based forms for caves and entrances.
"""
todo = """
- Ideally we should validate uploaded file as being a valid file type, not a dubious script or hack
Validate image files using a magic recogniser in walletedit()
https://pypi.org/project/reportlab/ or
https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file
- Write equivalent GPX upload form system, similar to walletedit() but in expofiles/gpslogs/
Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
- Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says
tunnel is only mostly correct XML, and it does fail at least one XML parser.
- parse the uploaded drawing file for links to wallets and scan files as done
in parsers/drawings.py
- Enable folder creation in dwguploads or as a separate form
- Enable file rename on expofiles, not just for /surveyscans/ (aka wallets)
- Make file rename utility less ugly.
"""
sha = hashlib.new('sha256')
class FilesForm(forms.Form): # not a model-form, just a form-form
uploadfiles = forms.FileField()
class FilesRenameForm(forms.Form): # not a model-form, just a form-form
uploadfiles = forms.FileField()
renameto = forms.CharField(strip=True, required=False)
class TextForm(forms.Form): # not a model-form, just a form-form
photographer = forms.CharField(strip=True)
class ExpofileRenameForm(forms.Form): # not a model-form, just a form-form
renameto = forms.CharField(strip=True, required=False)
class LogbookEditForm(forms.Form): # not a model-form, just a form-form
author = forms.CharField(strip=True, required=False)
@login_required_if_public
def logbookedit(request, year=None, slug=None):
"""Edit a logbook entry
This is daft: we have the parsed identity of the person and we render it to text as 'fullname', to be re-parsed on re-importing.
And there is no guarantee that this will be the same thing, esp. as aliases are used in the initial data input.
So we are losing all the cute aliases that have been used over the years by this export/re-import process. Bother.
But they have already been lost in the Great Format Conversion of 2022-23 when everything was chnaged to use the same HTML parser.
Which is a shame.
Fix is to add "alias_used" as a field in class PersonLogEntry, so that we can preserve
all those cute names. But it's rather a large manual effort (with some scripting) to recover the aliases from the original logbook
html files which are now only in the git history. Bother. Very sorry.
"""
def clean_tu(tu):
if tu =="":
return 0
try:
tu = float(tu)/1 # check numeric
except:
return 0
return tu
def unique_id(text, n):
"""This gives each logbook entry a unique id based on the date+content, so the order of entries on a particular day
does not matter. This is a change (August 2023) from previous process.
Otherwise we could get 2023-07-20a and 2023-07-20b swapped on exporting and re-importing logbooks
because the database does not record precedence.
2 hex digits would seem adequate for each expo day, but we might get a collision.
The hash is based on the content after substitution of <p> so should be stable. Which means these ids
can be used elsewhere in the troggle system as permanent slugs.
"""
sha.update(text.encode('utf-8'))
return sha.hexdigest()[0:n]
if not year:
year = 2023
if request.method == "POST":
form = LogbookEditForm(request.POST)
if not form.is_valid():
message = f'Invalid form response for logbook entry creating "{request.POST}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
# validation all to be done yet..
date = request.POST["date"].strip() # TODO check valid and this year
author = request.POST["author"].strip() # TODO check against personexpedition
others = request.POST["others"].strip() # TODO check each against personexpedition
place = request.POST["place"].strip().replace('-','=') # no hyphens !
title = request.POST["title"].strip()
entry = request.POST["text"].strip()
entry = entry.replace('\r','') # remove HTML-standard CR inserted
entry = entry.replace('\n\n','\n<br />\n<br />\n') # replace 2 \n with <br><br>
entry = entry.replace('<p','<br />\n<br') # replace <p> tag, even if it has attributes, with <br><br>
entry = entry.replace('<br>','<br />') # clean up previous hack
tu = request.POST["tu"].strip()
tu = clean_tu(tu)
uniq = unique_id(entry,2)
print(uniq)
# OK this could be done by rendering a template, but for such a small bit of HTML, it is easier to have
# it all in one place: here
output = f'''
<hr />
<div class="tripdate" id="{date}-{uniq}">{date}</div>
<div class="trippeople"><u>{author}</u>, {others}</div>
<div class="triptitle">{place} - {title}</div>
{entry}
<div class="timeug">T/U {tu} hrs</div>'''
return render(
request,
"logbookform.html",
{
"form": form,
"year": year,
"date": date,
"author": author,
"others": others,
"place": place,
"title": title,
"tu": tu,
"entry": entry,
"output": output,
},
)
# GET here
else:
form = LogbookEditForm()
if slug:
lbes = LogbookEntry.objects.filter(slug=slug)
if lbes:
if len(lbes) > 1:
return render(request, "object_list.html", {"object_list": lbe}) # ie a bug
else:
lbe = lbes[0]
print(f"{lbe}")
tu = clean_tu(lbe.time_underground)
people = []
for p in lbe.personlogentry_set.filter(logbook_entry=lbe):
if p.is_logbook_entry_author:
author = p.personexpedition.person.fullname
else:
people.append(p.personexpedition.person.fullname)
others =', '.join(people)
lenothers = min(70,max(20, len(others)))
print(f"{lenothers}")
text = lbe.text
rows = max(5,len(text)/50)
return render(
request,
"logbookform.html",
{
"form": form,
"year": year,
"date": lbe.date.isoformat(),
"author": author,
"others": others,
"lenothers": lenothers,
"place": lbe.place,
"title": lbe.title.replace(f"{lbe.place} - ",""),
"tu": tu,
"entry": text,
"textrows": rows,
#"output": output,
},
)
else:
return render(
request,
"logbookform.html",
{
"form": form,
"year": year,
},
)
@login_required_if_public
def expofilerename(request, filepath):
"""Rename any single file in /expofiles/ - eventually.
Currently this just does files within wallets i.e. in /surveyscans/
and it returns control to the original wallet edit page
"""
if filepath:
actualpath = Path(settings.EXPOFILES) / Path(filepath)
else:
message = f'\n File to rename not specified "{filepath}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
if not actualpath.is_file():
message = f'\n File not found when attempting rename "{filepath}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
filename = Path(filepath).name
folder = actualpath.parent
filesize = f"{actualpath.stat().st_size:,}"
if not actualpath.is_relative_to(Path(settings.SCANS_ROOT)):
message = f'\n Can only do rename within wallets (expofiles/surveyscans/) currently, sorry. "{actualpath}" '
print(message)
return render(request, "errors/generic.html", {"message": message})
if request.method == "POST":
form = ExpofileRenameForm(request.POST)
if not form.is_valid():
message = f'Invalid form response for file renaming "{request.POST}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
renameto = request.POST["renameto"]
if (folder / renameto).is_file() or (folder / renameto).is_dir():
rename_bad = renameto
message = f'\n Cannot rename to an existing file or folder. "{filename}" -> "{(folder / renameto)}"'
print(message)
return render(
request,
"renameform.html",
{
"form": form,
"filepath": filepath,
"filename": filename,
"filesize": filesize,
"rename_bad": rename_bad,
},
)
else:
actualpath.rename((folder / renameto))
message = f'\n RENAMED "{filename}" -> "{(folder / renameto)}"'
print(message)
walletid = actualpath.relative_to(Path(settings.SCANS_ROOT)).parent.stem.replace("#",":")
print(walletid)
return redirect(f'/survey_scans/{walletid}/')
else:
form = ExpofileRenameForm()
return render(
request,
"renameform.html",
{
"form": form,
"filepath": filepath,
"filename": filename,
"filesize": filesize,
},
)
@login_required_if_public
def photoupload(request, folder=None):
"""Upload photo image files into /expofiles/photos/<year>/<photographer>/
This does NOT use a Django model linked to a Django form. Just a simple Django form.
You will find the Django documentation on forms very confusing, This is simpler.
When uploading from a phone, it is useful to be able to rename the file to something
meaningful as this is difficult to do on a phone. Previously we had assumed files would
be renamed to something useful before starting the upload.
Unfortunately this only works when uploading one file at at time ,
inevitable once you think about it.
Pending generic file renaming capability more generally.
"""
year = settings.PHOTOS_YEAR
filesaved = False
actual_saved = []
context = {"year": year, "placeholder": "AnathemaDevice"}
yearpath = Path(settings.PHOTOS_ROOT, year)
if folder == str(year) or folder == str(year) + "/":
folder = None
if folder is None:
folder = "" # improve this later
dirpath = Path(settings.PHOTOS_ROOT, year)
urlfile = f"/expofiles/photos/{year}"
urldir = f"/photoupload/{year}"
else: # it will contain the year as well as the photographer
dirpath = Path(settings.PHOTOS_ROOT, folder)
if dirpath.is_dir():
urlfile = f"/expofiles/photos/{folder}"
urldir = Path("/photoupload") / folder
else:
folder = "" # improve this later
dirpath = Path(settings.PHOTOS_ROOT, year)
urlfile = f"/expofiles/photos/{year}"
urldir = f"/photoupload/{year}"
form = FilesRenameForm()
formd = TextForm()
if request.method == "POST":
if "photographer" in request.POST:
formd = TextForm(request.POST)
if formd.is_valid():
newphotographer = request.POST["photographer"]
try:
(yearpath / newphotographer).mkdir(exist_ok=True)
except:
message = f'\n !! Permissions failure ?! 0 attempting to mkdir "{(yearpath / newphotographer)}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
form = FilesRenameForm(request.POST, request.FILES)
if form.is_valid():
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist("uploadfiles")
# NO CHECK that the files being uploaded are image files
fs = FileSystemStorage(dirpath)
renameto = request.POST["renameto"]
actual_saved = []
if multiple:
if len(multiple) == 1:
if renameto != "":
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(renameto, content=f)
except:
print(
f'\n !! Permissions failure ?! 1 attempting to save "{f.name}" in "{dirpath}" {renameto=}'
)
if "saved_filename" in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
filesaved = True
else: # multiple is the uploaded content
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(
f'\n !! Permissions failure ?! 2 attempting to save "{f.name}" in "{dirpath}" {renameto=}'
)
if "saved_filename" in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
filesaved = True
else: # multiple is a list of content
for f in multiple:
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(
f'\n !! Permissions failure ?! 3 attempting to save "{f.name}" in "{dirpath}" {renameto=}'
)
if "saved_filename" in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
filesaved = True
files = []
dirs = []
try:
for f in dirpath.iterdir():
if f.is_dir():
dirs.append(f.name)
if f.is_file():
files.append(f.name)
except FileNotFoundError:
files.append("(no folder yet - would be created)")
if len(files) > 0:
files = sorted(files)
if dirs:
dirs = sorted(dirs)
return render(
request,
"photouploadform.html",
{
"form": form,
**context,
"urlfile": urlfile,
"urldir": urldir,
"folder": folder,
"files": files,
"dirs": dirs,
"filesaved": filesaved,
"actual_saved": actual_saved,
},
)
@login_required_if_public
def dwgupload(request, folder=None, gitdisable="no"):
"""Upload DRAWING files (tunnel or therion) into the upload folder in :drawings
AND registers it into the :drawings: git repo.
This does NOT use a Django model linked to a Django form. Just a simple Django form.
You will find the Django documentation on forms very confusing, This is simpler.
We could validate the uploaded files as being a valid files using an XML parser, not a dubious script or hack,
but this won't work on Tunnel files as Tunnel does not produce exactly valid xml
We use get_or_create instead of simply creating a new object in case someone uploads the same file
several times in one session, and expects them to be overwritten in the database. Although
the actual file will be duplicated in the filesystem with different random name ending.
"""
def dwgvalid(name):
if name in [
".gitignore",
]:
return False
if Path(name).suffix.lower() in [".xml", ".th", ".th2", "", ".svg", ".txt"]:
return True # dangerous, we should check the actual file binary signature
return False
def dwgvaliddisp(name):
"""OK to display, even if we are not going to allow a new one to be uploaded"""
if name in [
".gitignore",
]:
return False
if Path(name).suffix.lower() in [
".xml",
".th",
".th2",
"",
".svg",
".txt",
".jpg",
".jpeg",
".png",
".pdf",
".top",
".topo",
]:
return True # dangerous, we should check the actual file binary signature
return False
filesaved = False
actual_saved = []
refused = []
doesnotexist = ""
# print(f'! - FORM dwgupload - start "{folder}" - gitdisable "{gitdisable}"')
if folder is None:
folder = "" # improve this later
dirpath = Path(settings.DRAWINGS_DATA)
urlfile = "/dwgdataraw"
urldir = "/dwgupload"
else:
dirpath = Path(settings.DRAWINGS_DATA, folder)
urlfile = Path("/dwgdataraw/") / folder
urldir = Path("/dwgupload/") / folder
form = FilesForm()
if request.method == "POST":
form = FilesForm(request.POST, request.FILES)
if form.is_valid():
# print(f'! - FORM dwgupload - POST valid: "{request.FILES["uploadfiles"]}" ')
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist("uploadfiles")
savepath = Path(settings.DRAWINGS_DATA, folder)
fs = FileSystemStorage(savepath)
actual_saved = []
refused = []
# GIT see also core/views/expo.py editexpopage()
# GIT see also core/models/cave.py writetrogglefile()
if gitdisable != "yes": # set in url 'dwguploadnogit/'
git = settings.GIT
else:
git = "echo"
# print(f'git DISABLED {f.name}')
if multiple:
for f in multiple:
# print(f'! - FORM dwgupload - file {f} in {multiple=}')
if dwgvalid(f.name):
try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(
f'! - FORM dwgupload - \n!! Permissions failure ?! on attempting to save file "{f.name}" in "{savepath}". Attempting to continue..'
)
if "saved_filename" in locals():
if Path(dirpath, saved_filename).is_file():
actual_saved.append(saved_filename)
if gitdisable != "yes":
dr_add = subprocess.run(
[git, "add", saved_filename], cwd=dirpath, capture_output=True, text=True
)
msgdata = (
dr_add.stderr
+ "\n"
+ dr_add.stdout
+ "\nreturn code: "
+ str(dr_add.returncode)
)
# message = f'! - FORM dwgupload - Success: git ADD on server for this file {saved_filename}.' + msgdata
# print(message)
if dr_add.returncode != 0:
msgdata = (
"Ask a nerd to fix this.\n\n"
+ dr_add.stderr
+ "\n\n"
+ dr_add.stdout
+ "\n\nreturn code: "
+ str(dr_add.returncode)
)
message = (
f"! - FORM dwgupload - CANNOT git ADD on server for this file {saved_filename}. Edits saved but not added to git.\n"
+ msgdata
)
print(message)
return render(request, "errors/generic.html", {"message": message})
dwgfile, created = DrawingFile.objects.get_or_create(
dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size
)
dwgfile.save()
else:
message = f"! - FORM dwgupload - NOT A FILE {Path(dirpath, saved_filename)=}. "
print(message)
else:
message = f"! - FORM dwgupload - Save failure for {f.name}. Changes NOT saved."
print(message)
return render(request, "errors/generic.html", {"message": message})
if saved_filename != f.name:
# message = f'! - FORM dwgupload - Save RENAME {f.name} renamed as {saved_filename}. This is OK.'
# print(message)
pass
else:
refused.append(f.name)
# print(f'REFUSED {f.name}')
if actual_saved:
filesaved = True
if len(actual_saved) > 1:
dots = "..."
else:
dots = ""
if gitdisable != "yes":
dr_commit = subprocess.run(
[git, "commit", "-m", f"Drawings upload - {actual_saved[0]}{dots}"],
cwd=dirpath,
capture_output=True,
text=True,
)
# message = f'! - FORM dwgupload - For uploading {actual_saved[0]}{dots}. Edits saved, added to git, and COMMITTED.\n' + msgdata
# print(message)
# This produces return code = 1 if it commits OK
if dr_commit.returncode != 0:
msgdata = (
"Ask a nerd to fix this.\n\n"
+ dr_commit.stderr
+ "\n"
+ dr_commit.stdout
+ "\nreturn code: "
+ str(dr_commit.returncode)
)
message = (
f"! - FORM dwgupload -Error code with git on server for this {actual_saved[0]}{dots}. Edits saved, added to git, but NOT committed.\n"
+ msgdata
)
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
print(f' git disabled "{git=}"')
else: # maybe all were refused by the suffix test in dwgvalid()
message = f"! - FORM dwgupload - Nothing actually saved. All were refused. {actual_saved=}"
print(message)
files = []
dirs = []
# print(f'! - FORM dwgupload - start {folder=} \n"{dirpath=}" \n"{dirpath.parent=}" \n"{dirpath.exists()=}"')
try:
for f in dirpath.iterdir():
if f.is_dir():
if f.name not in [".git"]:
dirs.append(f.name)
continue
if f.is_file():
if dwgvaliddisp(f.name):
files.append(f.name)
continue
except FileNotFoundError:
doesnotexist = True
if files:
files = sorted(files)
if dirs:
dirs = sorted(dirs)
return render(
request,
"dwguploadform.html",
{
"form": form,
"doesnotexist": doesnotexist,
"urlfile": urlfile,
"urldir": urldir,
"folder": folder,
"files": files,
"dirs": dirs,
"filesaved": filesaved,
"actual_saved": actual_saved,
"refused": refused,
},
)
Reference in New Issue View Git Blame Copy Permalink