import subprocess
from datetime import datetime, timezone
from pathlib import Path
from django import forms
from django.core.files.storage import FileSystemStorage
from django.http import HttpResponseRedirect
from django.shortcuts import render, redirect
import settings
from troggle.core.models.caves import GetCaveLookup
from troggle.core.models.logbooks import LogbookEntry, writelogbook, PersonLogEntry
from troggle.core.models.survex import DrawingFile
from troggle.core.models.troggle import DataIssue, Expedition, PersonExpedition
from troggle.core.utils import alphabet_suffix, current_expo, sanitize_name, unique_slug, write_and_commit
from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from .auth import login_required_if_public
"""File upload 'views'
Note that there are other file upload forms in views/wallet_edit.py
and that core/forms.py contains Django class-based forms for caves and entrances.
"""
todo = """
- Ideally we should validate uploaded file as being a valid file type, not a dubious script or hack
Validate image files using a magic recogniser in walletedit()
https://pypi.org/project/reportlab/ or
https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file
- Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says
tunnel is only mostly correct XML, and it does fail at least one XML parser.
- parse the uploaded drawing file for links to wallets and scan files as done
in parsers/drawings.py
- Enable folder creation in dwguploads or as a separate form
- Enable file rename on expofiles, not just for /surveyscans/ (aka wallets)
- Make file rename utility less ugly.
"""
def create_new_lbe_slug(date):
onthisdate = LogbookEntry.objects.filter(date=date)
n = len(onthisdate)
print(f" Already entries on this date: {n}\n {onthisdate}")
suffix = alphabet_suffix(n+1)
tid = f"{date}{suffix}"
if len(tid) <=4 :
print(f"BAD ERROR {tid=}")
tid = f"{date}_{LogbookEntry.objects.count()}_{n}" # fudged number
print(f"{tid=}")
return tid
def store_edited_entry_into_database(date, place, title, text, others, author, tu, slug):
"""saves a single logbook entry and related personlogentry items
need to select out *guest and foreign friends from others
Rather similar to similarly named function in parsers/logbooks but circular reference prevents us using it directly,
and they need refactoring anyway.
"""
year = slug[0:4]
expedition = Expedition.objects.get(year=year)
cave = GetCaveLookup().get(place.lower())
# print(f"store_edited_entry_into_database(): {place=} {cave=}")
if LogbookEntry.objects.filter(slug=slug).exists():
# oops.
message = f" ! - DUPLICATE SLUG for logbook entry {date} - {slug}"
DataIssue.objects.create(parser="logbooks", message=message)
slug = slug + "_" + unique_slug(text,2)
otherAttribs = {
"place": place,
"text": text,
"expedition": expedition,
"time_underground": tu,
"cave": cave,
"title": f"{place} - {title}",
# "other_people": others
}
coUniqueAttribs = {"slug": slug, "date": date }
lbo = LogbookEntry.objects.create(**otherAttribs, **coUniqueAttribs)
pt_list = []
# These entities have to be PersonExpedition objects
team = others.split(",")
team.append(author)
odds = []
for name in team:
name = name.strip()
if len(name) > 0:
if name[0] == "*": # a name prefix of "*" is special, just a string.
odds.append(name)
print(f" - adding * special name '{name}'")
else:
try:
personyear = GetPersonExpeditionNameLookup(expedition).get(name.lower())
if not personyear:
odds.append(name)
print(f" - adding unrecognised expoer '{name}'")
if known_foreigner(name):
message = f" ! - Known foreigner: '{name}' in entry {slug=}"
print(message)
else:
message = f" ! - No name match for: '{name}' in entry {slug=}"
print(message)
DataIssue.objects.create(parser="logbooks", message=message)
else:
coUniqueAttribs = {"personexpedition": personyear, "nickname_used": name, "logbook_entry": lbo} # lbo is primary key
otherAttribs = {"time_underground": tu, "is_logbook_entry_author": (name==author)}
pt_list.append(PersonLogEntry(**otherAttribs, **coUniqueAttribs))
except:
# This should not happen. We do not raise exceptions in that function
message = f" ! - EXCEPTION: '{name}' in entry {slug=}"
print(message)
DataIssue.objects.create(parser="logbooks", message=message)
raise
PersonLogEntry.objects.bulk_create(pt_list)
lbo.other_people = ", ".join(odds)
print(f" - Saving other_people '{lbo.other_people}'")
lbo.save()
class FilesForm(forms.Form): # not a model-form, just a form-form
uploadfiles = forms.FileField()
class FilesRenameForm(forms.Form): # not a model-form, just a form-form
uploadfiles = forms.FileField()
renameto = forms.CharField(strip=True, required=False)
class TextForm(forms.Form): # not a model-form, just a form-form
photographer = forms.CharField(strip=True)
class TextProspectorForm(forms.Form): # not a model-form, just a form-form
prospector = forms.CharField(strip=True)
class ExpofileRenameForm(forms.Form): # not a model-form, just a form-form
renameto = forms.CharField(strip=True, required=False)
class ExpotextfileForm(forms.Form): # not a model-form, just a form-form
text = forms.CharField(strip=True, required=False)
class LogbookEditForm(forms.Form): # not a model-form, just a form-form
author = forms.CharField(strip=True, required=False)
@login_required_if_public
def edittxtpage(request, path, filepath):
"""Editing a .txt file on expoweb/
"""
def simple_get(viewtext):
form = ExpotextfileForm()
return render(
request,
"textfileform.html",
{
"form": form,
"path": path,
"message": message,
"filepath": filepath,
"text": viewtext,
},
)
message=""
if not filepath.is_file():
print(f"Not a file: {filepath}")
errpage = f"" + default_head + f"File not found '{filepath}'
failure detected in expowebpage() in views.expo.py
"
return HttpResponse(errpage)
try:
with open(filepath, "r") as f:
originaltext = f.read()
except IOError:
message = f'Cannot open {filepath} for text file reading even though it is a file.'
print(message)
return render(request, "errors/generic.html", {"message": message})
if request.method == "GET":
return simple_get(originaltext)
elif request.method == "POST":
form = ExpotextfileForm(request.POST)
if not form.is_valid():
message = f'Invalid form response for text file editing "{request.POST}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
# for i in request.POST:
# print(":: ",i, " => ", request.POST[i])
newtext = request.POST["text"]
print("POST")
if "Cancel" in request.POST:
print("cancel")
return simple_get(originaltext)
if "Save" in request.POST:
print("submitted for saving..")
if newtext != originaltext: # Check if content has changed at all
print("text changed.. saving and committing")
try:
write_and_commit([(filepath, newtext, "utf-8")], f"Online edit of {path}")
except WriteAndCommitError as e:
return render(request, "errors/generic.html", {"message": e.message})
print("re-reading from file..")
try:
with open(filepath) as f:
rereadtext = f.read()
except:
print("### File reading failure, but it exists.. ### ", filepath)
return render(request, "errors/generic.html", {"message": e.message})
savepath = "/" + path
print(f"redirect {savepath}")
return redirect(savepath) # Redirect after POST
else:
# no changes
pass
return simple_get(originaltext)
else:
# mistake not POST or GET
message="Something went wrong"
print(message)
return simple_get(originaltext)
@login_required_if_public
def logbookedit(request, year=None, slug=None):
"""Edit a logbook entry
This 'validates' the author as being on expo in the current year, but only indicates this by
putting the text of the form prompt in red (same as for an invalid date, which is arguably more important).
No check is done on the other people on the trip as this is picked up anyway by parsing on import
and we don't really care at this point.
If the author name is mispelled, noticed, and changed, then two logbook entries are created
with sequential slugs ...b ...c etc. This is because we are doing validation on GET not on POST
and we are not rewriting the URL when a slug gets set. Hmm.
Normal use of this form is producing duplicate logbook entries.. why ?!
"""
def validate_year(year):
try:
expo = Expedition.objects.get(year=year)
except:
year = current_expo() # creates new Expedition object if needed
return year
def new_entry_form():
return render(
request,
"logbookform.html",
{
"form": form,
"year": year,
},
)
def clean_tu(tu):
if tu =="":
return 0
try:
tu = float(tu)/1 # check numeric
except:
return 0
return tu
if not year:
if not slug: # not in the URL, we have not read teh POST response yet.. wich might have a slug in it
year = current_expo()
else:
year = slug[0:4]
try:
year = str(int(year)) # but maybe slug was hand-edited to be a future year..
year = validate_year(year) # so fix that
except:
year = current_expo()
author = ""
if request.method == "POST":
prev_slug = "" # None value pending overwrite from submitted form
form = LogbookEditForm(request.POST)
if not form.is_valid():
message = f'Invalid form response for logbook entry creating "{request.POST}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
# if there is no slug then this is probably a completely new lbe and we need to enter it into the db
# otherwise it is an update
# validation all to be done yet..
date = request.POST["date"].strip()
author = request.POST["author"].strip() # TODO check against personexpedition on submit
others = request.POST["others"].strip() # TODO check each against personexpedition on submit
place = request.POST["place"].strip().replace(' - ',' = ') # no hyphens !
title = request.POST["title"].strip()
entry = request.POST["text"].strip()
if "prev_slug" in request.POST:
prev_slug = request.POST["prev_slug"].strip() # if we are re-editing the same entry again
entry = entry.replace('\r','') # remove HTML-standard CR inserted from form.
entry = entry.replace('\n\n','\n\n') # replace 2 \n with
tu = request.POST["tu"].strip()
tu = clean_tu(tu)
try:
odate = datetime.strptime(date.replace(".", "-"), "%Y-%m-%d").date()
print(f"{odate.year=}")
if str(odate.year) == year:
dateflag = False
else:
print(f"Trying to change the year ! No!! {odate.year=}")
odate = datetime.strptime(f"{year}-01-01", "%Y-%m-%d").date()
dateflag = True
except:
odate = datetime.strptime(f"{year}-01-01", "%Y-%m-%d").date()
print(f"! Invalid date string {date}, setting to {odate}")
dateflag = True
date = odate.isoformat()
year = validate_year(year)
expo = Expedition.objects.get(year=year)
personyear = GetPersonExpeditionNameLookup(expo).get(author.lower())
if personyear:
authorflag = False
else:
authorflag = True
print(f"! Unrecognised author: {author}")
# if somehow we get a slug set to just '2024', not eg '2020-08-10b'
# because the URL of the page is /logbookedit/2022 for a new entry
# This is a hack, why can we not reproduce this bug on the dev system?
if slug not in locals():
slug = ""
if len(slug) < 11:
slug = ""
if len(prev_slug) < 11:
prev_slug = ""
if not prev_slug and not slug:
# Creating a new logbook entry with all the gubbins
slug = create_new_lbe_slug(date)
if prev_slug and not slug:
# if this was a previous post, then prev_slug will have been set on the form
# we are editing a previous thing, so we don't create a new lbe
slug = prev_slug
# OK we could patch the object in place, but if the people on the trip have changed this
# would get very messy. So we delete it, and thus all the dependent objects,
# and recreate it and all its links. It might not exist though.
print(f"- Deleting the LogBookEntry {slug}")
LogbookEntry.objects.filter(slug=slug).delete() # works even if it does not exist
print(f"- Creating the LogBookEntry {slug}")
year = slug[0:4]
try:
expedition = Expedition.objects.get(year=year)
except Expedition.DoesNotExist:
message = f'''! - This expo "{year}" not created yet
It needs to be created before you can save a logbook entry.
See /handbook/computing/newyear.html
WHAT TO DO NOW:
1. Press the Back button on your proswer to return to the screen where you typed up the entry,
2. Copy the text of what you wrote into a new text file,
3. Direct a nerd to fix this. It should take only a couple of minutes.'''
print(message)
return render(request, "errors/generic.html", {"message": message})
store_edited_entry_into_database(date, place, title, entry, others, author, tu, slug)
print(f"- Rewriting the entire {year} logbook to disc ")
filename= "logbook.html"
try:
print(f" - Logbook for {year} to be exported and written out.")
writelogbook(year, filename) # uses a template, not the code fragment below which is just a visible hint to logged on user
except:
message = f'! - Logbook saving failed - \n!! Permissions failure ?! on attempting to save file "logbook.html"'
print(message)
return render(request, "errors/generic.html", {"message": message})
# Code fragment illustration - not actually what gets saved to database
output = f'''
{date}
{author}, {others}
{place} - {title}
{entry}
T/U {tu} hrs
'''
# Successful POST
# So save to database and then write out whole new logbook.html file
# We do author validation on the form as displayed by GET, not at the moment of POST.
# If we had JS validation then we could be more timely.
git = settings.GIT
dirpath = Path(settings.EXPOWEB) / "years" / str(year)
lbe_add = subprocess.run(
[git, "add", filename], cwd=dirpath, capture_output=True, text=True
)
msgdata = (
lbe_add.stderr
+ "\n"
+ lbe_add.stdout
+ "\nreturn code: "
+ str(lbe_add.returncode)
)
message = f'! - FORM Logbook Edit {slug} - Success: git ADD on server for this file {filename}.' + msgdata
print(message)
if lbe_add.returncode != 0:
msgdata = (
"Ask a nerd to fix this.\n\n"
+ lbe_add.stderr
+ "\n\n"
+ lbe_add.stdout
+ "\n\nreturn code: "
+ str(lbe_add.returncode)
)
message = (
f"! - FORM Logbook Edit - CANNOT git ADD on server for this file {filename}. {slug} edits saved but not added to git.\n"
+ msgdata
)
print(message)
return render(request, "errors/generic.html", {"message": message})
lbe_commit = subprocess.run(
[git, "commit", "-m", f"Logbook edited {slug}"],
cwd=dirpath,
capture_output=True,
text=True,
)
message = f'! - FORM Logbook Edit - {filename}. {slug} edits saved, added to git, and COMMITTED.\n' + msgdata
print(message)
#This produces return code = 1 if it commits OK
if lbe_commit.returncode != 0:
msgdata = (
"Ask a nerd to fix this.\n\n"
+ lbe_commit.stderr
+ "\n"
+ lbe_commit.stdout
+ "\nreturn code: "
+ str(lbe_commit.returncode)
)
message = (
f"! - FORM Logbook Edit -Error code '{lbe_commit.returncode}' with git on server for {filename}. {slug} edits saved, added to git, but NOT committed.\n"
+ msgdata
)
print(message)
if not (lbe_commit.returncode ==1 and settings.DEVSERVER):
# rc=1 is OK on the development server
return render(request, "errors/generic.html", {"message": message})
# This does not change the URL in the browser, so despite a new slug being created,
# the next time this code is run it thinks a new slug needs to be created. So we should
# actually redirect to a new URL (an edit not a create) not simply return a render object.
# logbookedit/2022-08-21a
# HOWEVER by doing a redirect rather than returning a rendered page, we lose all the
# error settings e.g dateflag and authroflag so the user gets no feedback about bad data entered.
return HttpResponseRedirect(f"/logbookedit/{slug}?dateflag={dateflag}&authorflag={authorflag}")
# return render(
# request,
# "logbookform.html",
# {
# "form": form,
# "year": year,
# "date": date, "dateflag": dateflag,
# "author": author, "authorflag": authorflag,
# "others": others,
# "place": place,
# "title": title,
# "tu": tu,
# "entry": entry,
# "output": output,
# "slug": slug,
# },
# )
# GET here
else:
form = LogbookEditForm()
year = validate_year(year)
if request.GET.get('dateflag', 'False') == "True":
dateflag = True
else:
dateflag = False
if request.GET.get('authorflag', 'False') == "True":
authorflag = True
else:
authorflag = False
if not slug: # no slug or bad slug for an lbe which does not exist
return new_entry_form()
else:
lbes = LogbookEntry.objects.filter(slug=slug)
if not lbes:
return new_entry_form()
else:
if len(lbes) > 1:
return render(request, "object_list.html", {"object_list": lbes}) # ie a bug
else:
lbe = lbes[0]
print(f"{lbe}")
tu = clean_tu(lbe.time_underground)
people = []
for p in lbe.personlogentry_set.filter(logbook_entry=lbe): # p is a PersonLogEntry object
if p.is_logbook_entry_author:
# author = p.personexpedition.person.fullname
author = p.nickname_used
else:
# people.append(p.personexpedition.person.fullname)
people.append(p.nickname_used)
others =', '.join(people)
if lbe.other_people:
others = others + ", " + lbe.other_people
lenothers = min(70,max(20, len(others)))
text = lbe.text
rows = max(5,len(text)/50)
return render(
request,
"logbookform.html",
{
"form": form,
"year": year,
"date": lbe.date.isoformat(), "dateflag": dateflag,
"author": author, "authorflag": authorflag,
"others": others,
"lenothers": lenothers,
"place": lbe.place,
"title": lbe.title.replace(f"{lbe.place} - ",""),
"tu": tu,
"entry": text,
"textrows": rows,
"slug": slug,
},
)
@login_required_if_public
def expofilerename(request, filepath):
"""Rename any single file in /expofiles/ - eventually.
Currently this just does files within wallets i.e. in /surveyscans/
and it returns control to the original wallet edit page
"""
def is_rotatable(path):
"""If file is a JPG but has no filename extension, then it must be renamed
before it can be rotated.
"""
print(f"{path}: '{Path(path).suffix.lower()}'")
if Path(path).suffix.lower() in [".png", ".jpg", ".jpeg"]:
return True
else:
return False
def rotate_image():
wallet = str(Path(filepath).parent).lstrip("surveyscans/")
cwd = settings.SCANS_ROOT / wallet
print(f"ROTATE \n{cwd=} \n{filename=}")
mogrify = settings.MOGRIFY
rot = subprocess.run(
[mogrify, "-rotate", "90", filename], cwd=cwd, capture_output=True, text=True
)
msgdata = (
rot.stderr
+ "\n"
+ rot.stdout
+ "\nreturn code: "
+ str(rot.returncode)
)
message = f'! - ROTATE - Success: rotated this file {filename}.' + msgdata
print(message)
# DataIssue.objects.create(parser="mogrify", message=message)
if rot.returncode != 0:
msgdata = (
"Ask a nerd to fix this.\n\n"
+ rot.stderr
+ "\n\n"
+ rot.stdout
+ "\n\nreturn code: "
+ str(rot.returncode)
)
message = (
f"! - ROTATE - CANNOT blurk for this file {filename}. \n"
+ msgdata
)
print(message)
DataIssue.objects.create(parser="mogrify", message=message)
return simple_get()
def simple_get():
form = ExpofileRenameForm()
return render(
request,
"renameform.html",
{
"form": form,
"filepath": filepath,
"filename": filename,
"filesize": filesize,
"files": files,
"walletpath": walletpath,
"wallet": wallet,
"notpics": notpics,
"rotatable": rotatable,
},
)
if filepath:
# using EXPOFILES not SCANS_ROOT in case we want to extend this to other parts of the system
actualpath = Path(settings.EXPOFILES) / Path(filepath)
else:
message = f'\n File to rename not specified "{filepath}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
if not actualpath.is_file():
message = f'\n File not found when attempting rename "{filepath}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
filename = Path(filepath).name
walletpath = Path(filepath).parent
wallet = Path(walletpath).name
folder = actualpath.parent
filesize = f"{actualpath.stat().st_size:,}"
rotatable= is_rotatable(filename)
if not actualpath.is_relative_to(Path(settings.SCANS_ROOT)):
message = f'\n Can only do rename within wallets (expofiles/surveyscans/) currently, sorry. "{actualpath}" '
print(message)
return render(request, "errors/generic.html", {"message": message})
files = []
dirs = []
notpics =[]
dirpath = actualpath.parent
print(f'! - FORM rename expofile - start \n{filepath=} \n{dirpath=} \n{walletpath=}')
if dirpath.is_dir():
try:
for f in dirpath.iterdir():
if f.is_dir():
for d in f.iterdir():
dirs.append(f"{f.name}/{d.name}")
if f.is_file():
if is_rotatable(f.name): # should allow all images here which can be thumsized, not just rotatables. e.g. PDF
files.append(f.name)
else:
notpics.append(f.name)
except FileNotFoundError:
files.append(
"(Error. There should be at least one filename visible here. Try refresh.)"
)
if request.method == "GET":
return simple_get()
elif request.method == "POST":
form = ExpofileRenameForm(request.POST)
if not form.is_valid():
message = f'Invalid form response for file renaming "{request.POST}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
if "rotate" in request.POST:
return rotate_image()
if "rename" in request.POST:
if "renametoname" not in request.POST:
print("renametoname not in request.POST")
# blank filename passed it, so just treat as another GET
return simple_get()
renameto = sanitize_name(request.POST["renametoname"])
if (folder / renameto).is_file() or (folder / renameto).is_dir():
rename_bad = renameto
message = f'\n Cannot rename to an existing file or folder. "{filename}" -> "{(folder / renameto)}"'
print(message)
return render(
request,
"renameform.html",
{
"form": form,
"filepath": filepath,
"filename": filename,
"filesize": filesize,
"files": files,
"walletpath": walletpath,
"wallet": wallet,
"notpics": notpics,
"rename_bad": rename_bad,
},
)
actualpath.rename((folder / renameto))
message = f'\n RENAMED "{filename}" -> "{(folder / renameto)}"'
print(message)
walletid = actualpath.relative_to(Path(settings.SCANS_ROOT)).parent.stem.replace("#",":")
print(walletid)
return redirect(f'/survey_scans/{walletid}/')
else: # not GET or POST
print("UNRECOGNIZED action")
return simple_get()
@login_required_if_public
def photoupload(request, folder=None):
"""Upload photo image files into /expofiles/photos///
This does NOT use a Django model linked to a Django form. Just a simple Django form.
You will find the Django documentation on forms very confusing, This is simpler.
When uploading from a phone, it is useful to be able to rename the file to something
meaningful as this is difficult to do on a phone. Previously we had assumed files would
be renamed to something useful before starting the upload.
Unfortunately this only works when uploading one file at at time ,
inevitable once you think about it.
Pending generic file renaming capability more generally.
"""
year = current_expo()
# year = settings.PHOTOS_YEAR
filesaved = False
actual_saved = []
context = {"year": year, "placeholder": "AnathemaDevice"}
yearpath = Path(settings.PHOTOS_ROOT, year)
if folder == str(year) or folder == str(year) + "/":
folder = None
if folder is None:
folder = "" # improve this later
dirpath = Path(settings.PHOTOS_ROOT, year)
urlfile = f"/expofiles/photos/{year}"
urldir = f"/photoupload/{year}"
else: # it will contain the year as well as the photographer
dirpath = Path(settings.PHOTOS_ROOT, folder)
if dirpath.is_dir():
urlfile = f"/expofiles/photos/{folder}"
urldir = Path("/photoupload") / folder
else:
folder = "" # improve this later
dirpath = Path(settings.PHOTOS_ROOT, year)
urlfile = f"/expofiles/photos/{year}"
urldir = f"/photoupload/{year}"
form = FilesRenameForm()
formd = TextForm()
if request.method == "POST":
if "photographer" in request.POST:
# then we are creating a new folder
formd = TextForm(request.POST)
if formd.is_valid():
newphotographer = sanitize_name(request.POST["photographer"])
try:
(yearpath / newphotographer).mkdir(exist_ok=True)
except:
message = f'\n !! Permissions failure ?! 0 attempting to mkdir "{(yearpath / newphotographer)}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
# then we are renaming the file ?
form = FilesRenameForm(request.POST, request.FILES)
if form.is_valid():
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist("uploadfiles")
# NO CHECK that the files being uploaded are image files
fs = FileSystemStorage(dirpath)
renameto = sanitize_name(request.POST["renameto"])
actual_saved = []
if multiple:
if len(multiple) == 1:
if renameto != "":
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(renameto, content=f)
except:
print(
f'\n !! Permissions failure ?! 1 attempting to save "{f.name}" in "{dirpath}" {renameto=}'
)
if "saved_filename" in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
filesaved = True
else: # multiple is the uploaded content
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(
f'\n !! Permissions failure ?! 2 attempting to save "{f.name}" in "{dirpath}" {renameto=}'
)
if "saved_filename" in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
filesaved = True
else: # multiple is a list of content
for f in multiple:
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(
f'\n !! Permissions failure ?! 3 attempting to save "{f.name}" in "{dirpath}" {renameto=}'
)
if "saved_filename" in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
filesaved = True
files = []
dirs = []
try:
for f in dirpath.iterdir():
if f.is_dir():
dirs.append(f.name)
if f.is_file():
files.append(f.name)
except FileNotFoundError:
files.append("(no folder yet - would be created)")
if len(files) > 0:
files = sorted(files)
if dirs:
dirs = sorted(dirs)
return render(
request,
"photouploadform.html",
{
"form": form,
**context,
"urlfile": urlfile,
"urldir": urldir,
"folder": folder,
"files": files,
"dirs": dirs,
"filesaved": filesaved,
"actual_saved": actual_saved,
},
)
@login_required_if_public
def gpxupload(request, folder=None):
"""Copy of photo upload
folder is the "path"
"""
def gpxvalid(name):
if Path(name).suffix.lower() in [".xml", ".gpx"]:
return True # dangerous, we should check the actual file binary signature
return False
print(f"gpxupload() {folder=}")
year = current_expo()
filesaved = False
actual_saved = []
context = {"year": year, "placeholder": "AnathemaDevice"}
yearpath = Path(settings.EXPOFILES) / "gpslogs" / year
if folder == str(year) or folder == str(year) + "/":
folder = None
if folder is None:
folder = "" # improve this later
dirpath = yearpath
urlfile = f"/expofiles/gpslogs/{year}"
urldir = f"/gpxupload/{year}"
else: # it will contain the year as well as the prospector
dirpath = Path(settings.EXPOFILES) / "gpslogs" / folder
if dirpath.is_dir():
urlfile = f"/expofiles/gpslogs/{folder}"
urldir = Path("/gpxupload") / folder
else:
folder = "" # improve this later
dirpath = yearpath
urlfile = f"/expofiles/gpslogs/{year}"
urldir = f"/gpxupload/{year}"
print(f"gpxupload() {folder=} {dirpath=} {urlfile=} {urldir=}")
form = FilesRenameForm()
formd = TextProspectorForm()
print(f"gpxupload() {form=} {formd=} ")
if request.method == "POST":
print(f"gpxupload() method=POST")
for i in request.POST:
print(" ",i)
if "prospector" in request.POST:
print(f"gpxupload() {request.POST=}\n {request.POST['prospector']=}")
formd = TextProspectorForm(request.POST)
if formd.is_valid():
newprospector = sanitize_name(request.POST["prospector"])
print(f"gpxupload() {newprospector=}")
try:
(yearpath / newprospector).mkdir(exist_ok=True)
except:
message = f'\n !! Permissions failure ?! 0 attempting to mkdir "{(yearpath / newprospector)}"'
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
print(f"gpxupload() no prospector field")
print(f"gpxupload() {request.FILES=}")
for i in request.FILES:
print(" ",i)
form = FilesRenameForm(request.POST, request.FILES)
print(f"gpxupload() is the FilesRenameForm valid? {form=}")
for i in form:
print(" ",i)
if not form.is_valid():
print(f"gpxupload() Form is not valid {form=}")
else:
print(f"gpxupload() about to look at request.FILES")
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist("uploadfiles")
# NO CHECK that the files being uploaded are image files
fs = FileSystemStorage(dirpath)
renameto = sanitize_name(request.POST["renameto"])
actual_saved = []
if multiple:
if len(multiple) == 1:
if renameto != "":
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(renameto, content=f)
except:
print(
f'\n !! Permissions failure ?! 1 attempting to save "{f.name}" in "{dirpath}" {renameto=}'
)
if "saved_filename" in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
filesaved = True
else: # multiple is the uploaded content
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(
f'\n !! Permissions failure ?! 2 attempting to save "{f.name}" in "{dirpath}" {renameto=}'
)
if "saved_filename" in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
filesaved = True
else: # multiple is a list of content
for f in multiple:
if gpxvalid(f.name):
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(
f'\n !! Permissions failure ?! 3 attempting to save "{f.name}" in "{dirpath}" {renameto=}'
)
if "saved_filename" in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
filesaved = True
else:
print(f"gpxupload(): not a GPX file {f.name=}")
print(f"gpxupload() drop through")
files = []
dirs = []
try:
for f in dirpath.iterdir():
if f.is_dir():
dirs.append(f.name)
if f.is_file():
files.append(f.name)
except FileNotFoundError:
files.append("(no folder yet - would be created)")
except Exception as e:
print(f"gpxupload() EXCEPTION\n {e}")
if len(files) > 0:
files = sorted(files)
if dirs:
dirs = sorted(dirs)
print(f"gpxupload() about to render..")
return render(
request,
"gpxuploadform.html",
{
"form": form,
**context,
"urlfile": urlfile,
"urldir": urldir,
"folder": folder,
"files": files,
"dirs": dirs,
"filesaved": filesaved,
"actual_saved": actual_saved,
},
)
@login_required_if_public
def dwgupload(request, folder=None, gitdisable="no"):
"""Upload DRAWING files (tunnel or therion) into the upload folder in :drawings
AND registers it into the :drawings: git repo.
This does NOT use a Django model linked to a Django form. Just a simple Django form.
You will find the Django documentation on forms very confusing, This is simpler.
We could validate the uploaded files as being a valid files using an XML parser, not a dubious script or hack,
but this won't work on Tunnel files as Tunnel does not produce exactly valid xml
We use get_or_create instead of simply creating a new object in case someone uploads the same file
several times in one session, and expects them to be overwritten in the database. Although
the actual file will be duplicated in the filesystem with different random name ending.
"""
def dwgvalid(name):
if name in [
".gitignore",
]:
return False
if Path(name).suffix.lower() in [".xml", ".th", ".th2", "", ".svg", ".txt"]:
return True # dangerous, we should check the actual file binary signature
return False
def dwgvaliddisp(name):
"""OK to display, even if we are not going to allow a new one to be uploaded"""
if name in [
".gitignore",
]:
return False
if Path(name).suffix.lower() in [
".xml",
".th",
".th2",
"",
".svg",
".txt",
".jpg",
".jpeg",
".png",
".pdf",
".top",
".topo",
]:
return True # dangerous, we should check the actual file binary signature
return False
filesaved = False
actual_saved = []
refused = []
doesnotexist = ""
# print(f'! - FORM dwgupload - start "{folder}" - gitdisable "{gitdisable}"')
if folder is None:
folder = "" # improve this later
dirpath = Path(settings.DRAWINGS_DATA)
urlfile = "/dwgdataraw"
urldir = "/dwgupload"
else:
dirpath = Path(settings.DRAWINGS_DATA, folder)
urlfile = Path("/dwgdataraw/") / folder
urldir = Path("/dwgupload/") / folder
form = FilesForm()
if request.method == "POST":
form = FilesForm(request.POST, request.FILES)
if form.is_valid():
# print(f'! - FORM dwgupload - POST valid: "{request.FILES["uploadfiles"]}" ')
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist("uploadfiles")
savepath = Path(settings.DRAWINGS_DATA, folder)
fs = FileSystemStorage(savepath)
actual_saved = []
refused = []
# GIT see also core/views/expo.py editexpopage()
# GIT see also core/models/cave.py writetrogglefile()
if gitdisable != "yes": # set in url 'dwguploadnogit/'
git = settings.GIT
else:
git = "echo"
# print(f'git DISABLED {f.name}')
if multiple:
for f in multiple:
# print(f'! - FORM dwgupload - file {f} in {multiple=}')
if dwgvalid(f.name):
try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(
f'! - FORM dwgupload - \n!! Permissions failure ?! on attempting to save file "{f.name}" in "{savepath}". Attempting to continue..'
)
if "saved_filename" in locals():
if Path(dirpath, saved_filename).is_file():
actual_saved.append(saved_filename)
if gitdisable != "yes":
dr_add = subprocess.run(
[git, "add", saved_filename], cwd=dirpath, capture_output=True, text=True
)
msgdata = (
dr_add.stderr
+ "\n"
+ dr_add.stdout
+ "\nreturn code: "
+ str(dr_add.returncode)
)
# message = f'! - FORM dwgupload - Success: git ADD on server for this file {saved_filename}.' + msgdata
# print(message)
if dr_add.returncode != 0:
msgdata = (
"Ask a nerd to fix this.\n\n"
+ dr_add.stderr
+ "\n\n"
+ dr_add.stdout
+ "\n\nreturn code: "
+ str(dr_add.returncode)
)
message = (
f"! - FORM dwgupload - CANNOT git ADD on server for this file {saved_filename}. Edits saved but not added to git.\n"
+ msgdata
)
print(message)
return render(request, "errors/generic.html", {"message": message})
dwgfile, created = DrawingFile.objects.get_or_create(
dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size
)
dwgfile.save()
else:
message = f"! - FORM dwgupload - NOT A FILE {Path(dirpath, saved_filename)=}. "
print(message)
else:
message = f"! - FORM dwgupload - Save failure for {f.name}. Changes NOT saved."
print(message)
return render(request, "errors/generic.html", {"message": message})
if saved_filename != f.name:
# message = f'! - FORM dwgupload - Save RENAME {f.name} renamed as {saved_filename}. This is OK.'
# print(message)
pass
else:
refused.append(f.name)
# print(f'REFUSED {f.name}')
if actual_saved:
filesaved = True
if len(actual_saved) > 1:
dots = "..."
else:
dots = ""
if gitdisable != "yes":
dr_commit = subprocess.run(
[git, "commit", "-m", f"Drawings upload - {actual_saved[0]}{dots}"],
cwd=dirpath,
capture_output=True,
text=True,
)
# message = f'! - FORM dwgupload - For uploading {actual_saved[0]}{dots}. Edits saved, added to git, and COMMITTED.\n' + msgdata
# print(message)
# This produces return code = 1 if it commits OK
if dr_commit.returncode != 0:
msgdata = (
"Ask a nerd to fix this.\n\n"
+ dr_commit.stderr
+ "\n"
+ dr_commit.stdout
+ "\nreturn code: "
+ str(dr_commit.returncode)
)
message = (
f"! - FORM dwgupload -Error code with git on server for this {actual_saved[0]}{dots}. Edits saved, added to git, but NOT committed.\n"
+ msgdata
)
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
print(f' git disabled "{git=}"')
else: # maybe all were refused by the suffix test in dwgvalid()
message = f"! - FORM dwgupload - Nothing actually saved. All were refused. {actual_saved=}"
print(message)
files = []
dirs = []
# print(f'! - FORM dwgupload - start {folder=} \n"{dirpath=}" \n"{dirpath.parent=}" \n"{dirpath.exists()=}"')
try:
for f in dirpath.iterdir():
if f.is_dir():
if f.name not in [".git"]:
dirs.append(f.name)
continue
if f.is_file():
if dwgvaliddisp(f.name):
files.append(f.name)
continue
except FileNotFoundError:
doesnotexist = True
if files:
files = sorted(files)
if dirs:
dirs = sorted(dirs)
return render(
request,
"dwguploadform.html",
{
"form": form,
"doesnotexist": doesnotexist,
"urlfile": urlfile,
"urldir": urldir,
"folder": folder,
"files": files,
"dirs": dirs,
"filesaved": filesaved,
"actual_saved": actual_saved,
"refused": refused,
},
)