xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity
c29e240c2b
troggle-unchained/core/views/uploads.py
Philip Sargent c29e240c2b creating new wallet now copies nd commits
2022-07-28 18:36:40 +03:00

642 lines
29 KiB
Python
Raw Blame History

import re, os
import subprocess
import json
import settings
import urllib
import operator
from pathlib import Path
from functools import reduce
from django import forms
from django.conf import settings
from django.urls import reverse
from django.db.models import Q
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.template import Context, loader
from django.core.files.storage import FileSystemStorage, default_storage
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
#from troggle import settings
from troggle.parsers.imports import import_caves, import_people, import_surveyscans
from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml, CopyWalletData
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import DataIssue
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
from troggle.core.models.survex import DrawingFile, Wallet
from troggle.core.views.scans import oldwallet, walletindex
from troggle.core.views.caves import getCave
from .auth import login_required_if_public
#from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt
'''File upload 'views'
'''
todo = '''
- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
- Refactor scanupload() as it contains all the wallets 'complaints' code from the old script wallets.py
- Need to validate uploaded file as being a valid image file, not a dubious script or hack
- Write equivalent GPX upload form system, similar to scanupload() but in expofiles/gpslogs/
Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
- Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says tunnel is only mostly correct XML
- Validate image files using a magic recogniser in scanupload() https://pypi.org/project/reportlab/ or
https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file
- Enable folder creation in dwguploads or as a separate form
'''
class FilesForm(forms.Form): # not a model-form, just a form-form
uploadfiles = forms.FileField()
class TextForm(forms.Form): # not a model-form, just a form-form
photographer = forms.CharField(strip=True)
class WalletForm(forms.Form): # not a model-form, just a form-form
descriptionw = forms.CharField(strip=True, required=False)
people = forms.CharField(strip=True, required=False)
survexnr = forms.CharField(strip=True, required=False)
qmsw = forms.CharField(strip=True, required=False)
date = forms.CharField(strip=True, required=True) # the only required field
websiteupt = forms.CharField(strip=True, required=False)
elevnr = forms.CharField(strip=True, required=False)
cave = forms.CharField(strip=True, required=False)
psg = forms.CharField(strip=True, required=False)
plannr = forms.CharField(strip=True, required=False)
electronic = forms.CharField(strip=True, required=False)
pland = forms.CharField(strip=True, required=False)
elevd = forms.CharField(strip=True, required=False)
url = forms.CharField(strip=True, required=False)
survex = forms.CharField(strip=True, required=False)
xlate = {"url": "description url",
"descriptionw": "description written",
"people": "people",
"date": "date",
"cave": "cave",
"plannr": "plan not required",
"survexnr": "survex not required",
"qmsw": "qms written",
"elevnr": "elev not required",
"websiteupt": "website updated",
"electronic": "electronic survey",
"pland": "plan drawn",
"elevd": "elev drawn",
"psg": "name", # a name for this wallet
"survex": "survex file",
}
def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
'''Taken from old script wallets.py and edited to make more comprehensible
Loads the survex files names and processes all complaints
'''
# Date
if not waldata["date"]:
complaints.append("A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one.")
# People
if waldata["people"]==["NOBODY"] or waldata["people"]==["Unknown"]:
complaints.append("Someody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data.")
survex_complaint = ""
if waldata["survex file"]:
if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
waldata["survex file"] = [waldata["survex file"]]
for svx in waldata["survex file"]:
if svx !="":
svxfiles.append(svx)
if not (Path(settings.SURVEX_DATA) / svx).is_file():
file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo"
complaints.append(file_complaint)
message = f"! {file_complaint}"
print(message)
DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
if waldata["survex not required"] and waldata["survex file"] != "":
survex_complaint = "Survex is stated as not required and yet there is a survex file!"
if not waldata["survex not required"] and waldata["survex file"] == "":
survex_complaint = "A survex file is required, but has not been specified!"
if survex_complaint:
complaints.append(survex_complaint)
# Notes required
if not waldata["electronic survey"]:
notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False)
notes_scanned = reduce(operator.or_, [f.endswith("note") for f in files], notes_scanned)
if not notes_scanned:
complaints.append("The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey.")
# Plan drawing required
plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False)
plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned)
plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"])
if plan_drawing_required:
complaints.append("The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found.")
# Elev drawing required
elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False)
elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned)
elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"])
if elev_drawing_required:
complaints.append("The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found.")
# Electronic Surveys
if not waldata["electronic survey"]:
if elev_drawing_required or plan_drawing_required:
complaints.append("Tunnel or Therion drawing files need drawing. Or if this an electronic survey, please tick the 'Electronic survey' checkbox.")
# Description
if not waldata["description written"]:
complaints.append("The guidebook description needs writing into the survex file. Tick the 'Cave description written' checkbox when this is done.")
# QMs
if not waldata["qms written"]:
complaints.append("The QMs needs writing into the survex file. Tick the 'QMs written' checkbox when this is done.")
# Website
if not waldata["website updated"]:
complaints.append("The cave description website is marked as needing updating using the guidebook description from the survex file. Tick the 'Website updated' checkbox when this is done.")
# FInd the cave, if it exists
if waldata["cave"]:
try:
caveid = waldata["cave"]
caveid = caveid.replace("/","-")
caveobject = getCave(caveid)
print(f'getCave for id "{waldata["cave"]}" {caveobject}')
if not caveobject.url == waldata["description url"]:
complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.')
except Cave.MultipleObjectsReturned:
complaints.append(f'The cave ID \'{waldata["cave"]}\' is AMBIGUOUS. Please fix it.')
caveobject = None
except ObjectDoesNotExist:
complaints.append(f'The cave ID \'{waldata["cave"]}\' is not recognised. Please fix it.')
caveobject = None
else:
complaints.append(f'No cave ID is given. Please give an ID, even if it is just "surface survey" or "scraps found in hut"')
caveobject = None
return complaints, caveobject
# @login_required_if_public
def scanupload(request, path=None):
'''Upload scanned image files into a wallet on /expofiles
Also display and edit the contents.json data in the wallet.
This does NOT use a Django model linked to a Django form. Just a simple Django form.
You will find the Django documentation on forms very confusing, This is simpler.
This subsumes much of the code which was in the old wallets.py script and so this function is very long
indeed and needs refactoring.
'''
filesaved = False
actual_saved = []
checkboxes = ["description written", "survex not required", "qms written", "website updated",
"plan not required", "plan drawn", "elev not required", "elev drawn", "electronic survey" ]
if path:
wallet = urllib.parse.unquote(path)
else:
wallet = "2022#01" # improve this later
year = wallet[:4]
try:
if int(year) <= 1998:
#print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET')
return(oldwallet(request, path))
except:
# if nonumeric wallet name for example
return(oldwallet(request, path))
if str(wallet).lower().endswith('indexpages'):
# print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET')
return(walletindex(request, path))
if not re.match('(19|20)\d\d[:#]\d\d', wallet):
wallet = "2022:01" # improve this later
# print(f'! - FORM scanupload - start {wallet}')
if path:
pass
# print(f'! - FORM scanupload - start wallet:{wallet}: path:{path}:')
if int(year) < 1977:
year = "1977"
if int(year) > 2050:
year = "2050"
nexty = f'{int(year)+1}'
prevy = f'{int(year)-1}'
wnumber = wallet[5:]
next = f'{int(wnumber)+1:02d}'
prev = f'{int(wnumber)-1:02d}'
if int(wnumber) == 0:
prev = f'{int(wnumber):02d}'
wurl = f"/scanupload/{wallet}".replace('#', ':')
wallet = wallet.replace(':','#')
dirpath = Path(settings.SCANS_ROOT, year, wallet)
contents_path = dirpath / contentsjson
walletdata = dirpath / contentsjson
form = FilesForm()
if request.method == 'POST':
if "psg" in request.POST:
formj = WalletForm(request.POST)
# Beware. All fields returned as strings. Must re-type them as lists etc. before using or re-saving
# Also lots of hassle with lists of strings interpreted as a single string
# Unset checkboxes do not return any value, checked ones return "True". So need initialising to False
if formj.is_valid():
#print(f'--- JSON Update form is VALID, saving to {contents_path}')
posted = request.POST.copy()
posted.pop("csrfmiddlewaretoken") # discard this
wd = wallet_blank_json
for f in checkboxes:
wd[f] = False
#print(f'--- wd ${f}$ - {wd[f]}')
for f in posted:
wd[xlate[f]] = posted[f].replace("\'", "\"")
if posted[f] =="True":
wd[xlate[f]] = True
wd["people"] = wd["people"][1:-1].replace("\"", "").split(",")
for i, elem in enumerate(wd["people"]):
wd["people"][i] = elem.strip()
#print(f'--- ${wd["survex file"]}$ - {type(wd["survex file"])}')
if wd["survex file"]: # allow for no survex file at all
if wd["survex file"][0] == '[':
wd["survex file"] = wd["survex file"][1:-1]
wd["survex file"] = wd["survex file"].replace("\"", "").split(",")
for i, elem in enumerate(wd["survex file"]):
wd["survex file"][i] = elem.strip()
#print(f'--- {wd["survex file"]} - {type(wd["survex file"])}')
with open(contents_path, "w") as jfile:
json.dump(wd, jfile, indent = 1)
# print(f'--- FINISHED saving to JSON\n')
# This copies the new data to the drawings repo and commit it
# needs the troggle object wallet, not a string
try:
w, created = Wallet.objects.get_or_create(walletname=wallet)
print(f'wallet string {wallet}, wallet object {w} created new?: {created}')
if created:
w.fpath = Path(settings.SCANS_ROOT, wallet[0:4], wallet)
w.save()
CopyWalletData(w)
except:
print(f'wallet string {wallet}, FAIL TO GET WALLET OBJECT, maybe we need to create it ?')
raise
else:
print(f'--- INVALID JSON Update form submitted')
print(formj.errors)
return render(request,'errors/generic.html', {'message': formj.errors})
else:
form = FilesForm(request.POST,request.FILES)
if form.is_valid():
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist('uploadfiles')
fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary
actual_saved = []
if multiple:
for f in multiple:
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(f'\n !! Permissions failure ?! on attempting to save file {f.name}')
if 'saved_filename' in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
# print(f'! - FORM scanupload multiple {actual_saved}')
filesaved = True
if not contents_path.is_file(): # double-check
with open(contents_path, "w") as json_file:
json.dump(wallet_blank_json, json_file, sort_keys=True, indent = 1)
index_path = dirpath / indexhtml
if not index_path.is_file(): # double-check
thishtml = wallet_blank_html.replace("YEAR", str(year))
thishtml = thishtml.replace("WALLET", str(wallet))
with open(index_path, "w") as html_file:
html_file.write(thishtml )
files = []
dirs = []
# print(f'! - FORM scanupload - start {wallet} {dirpath}')
if dirpath.is_dir():
create = False
try:
for f in dirpath.iterdir():
if f.is_dir():
dirs.append(f.name)
if f.is_file():
if f.name != 'contents.json' and f.name != 'walletindex.html':
files.append(f.name)
except FileNotFoundError:
files.append('(no wallet yet. It would be created if you upload a scan)')
else:
create = True
if len(files) >0 :
files = sorted(files)
if dirs:
dirs = sorted(dirs)
waldata = {}
if contents_path.is_file():
with open(contents_path) as json_file:
try:
waldata = json.load(json_file)
except:
message = f"! {wallet} Failed to load {contents_path} JSON file"
print(message)
DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
raise
cave =""
psg = ""
chkplannr = ""
chkpland = ""
svxfiles = []
checked = {}
if waldata:
if not waldata["people"]:
waldata["people"]=["NOBODY"]
if not type(waldata["people"])==list:
if waldata["people"][0] == '"':
waldata["people"] = waldata["people"][1:-1]
waldata["people"] = list(waldata["people"])
if not waldata["description url"]:
waldata["description url"]=""
if waldata["cave"]:
cave = waldata["cave"] # text string
if waldata["name"]:
psg = waldata["name"]
#Survex and survex complaints
complaints, caveobject = get_complaints([], waldata, svxfiles, files, wallet, wurl)
for f in checkboxes:
if waldata[f]:
checked[f] = "checked"
context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty,
'files': files, 'dirs': dirs, 'waldata': waldata, 'svxfiles': svxfiles,
'checked': checked,
'create': create,
'complaints': complaints,
'caveobject': caveobject,
'people': waldata["people"], 'peoplesize': str(len(str(waldata["people"]))),
'filesaved': filesaved, 'actual_saved': actual_saved }
return render(request, 'walletform.html',
{'form': form, 'wallet': wallet, **context,
'date': waldata["date"],
'url': waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))),
'survex': waldata["survex file"], 'survexsize': str(len(str(waldata["survex file"]))),
'cave': cave, 'psg': psg, 'psgsize': str(max(12,len(str(psg))))})
else: # no wallet here
context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty,
'files': files, 'dirs': dirs, 'waldata': waldata, 'svxfiles': svxfiles,
'checked': checked,
'create': create,
'people': "", 'peoplesize': 12,
'filesaved': filesaved, 'actual_saved': actual_saved }
return render(request, 'walletform.html',
{'form': form, 'wallet': wallet, **context,
'date': "",
'url': "", 'urlsize': 12,
'survex': "", 'survexsize': 12,
'cave': cave, 'psg': psg, 'psgsize': 12})
@login_required_if_public
def photoupload(request, folder=None):
'''Upload photo image files into /expofiles/photos/<year>/<photographer>/
This does NOT use a Django model linked to a Django form. Just a simple Django form.
You will find the Django documentation on forms very confusing, This is simpler.
'''
year = settings.PHOTOS_YEAR
filesaved = False
actual_saved = []
context = {'year': year, 'placeholder': "AnathemaDevice"}
yearpath = Path(settings.PHOTOS_ROOT, year)
if folder == str(year) or folder == str(year) +"/":
folder = None
if folder is None:
folder = "" # improve this later
dirpath = Path(settings.PHOTOS_ROOT, year)
urlfile = f'/expofiles/photos/{year}'
urldir = f'/photoupload/{year}'
else: # it will contain the year as well as the photographer
dirpath = Path(settings.PHOTOS_ROOT, folder)
if dirpath.is_dir():
urlfile = f'/expofiles/photos/{folder}'
urldir = Path('/photoupload') / folder
else:
folder = "" # improve this later
dirpath = Path(settings.PHOTOS_ROOT, year)
urlfile = f'/expofiles/photos/{year}'
urldir = f'/photoupload/{year}'
form = FilesForm()
formd = TextForm()
if request.method == 'POST':
if "photographer" in request.POST:
formd = TextForm(request.POST)
if formd.is_valid():
newphotographer = request.POST["photographer"]
(yearpath / newphotographer).mkdir(exist_ok=True)
else:
form = FilesForm(request.POST,request.FILES)
if form.is_valid():
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist('uploadfiles')
# NO CHECK that the files being uploaded are image files
fs = FileSystemStorage(dirpath)
actual_saved = []
if multiple:
for f in multiple:
try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(f'\n !! Permissions failure ?! on attempting to save file {f.name}')
if 'saved_filename' in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
filesaved = True
files = []
dirs = []
try:
for f in dirpath.iterdir():
if f.is_dir():
dirs.append(f.name)
if f.is_file():
files.append(f.name)
except FileNotFoundError:
files.append('(no folder yet - would be created)')
if len(files) >0 :
files = sorted(files)
if dirs:
dirs = sorted(dirs)
return render(request, 'photouploadform.html',
{'form': form, **context, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved})
@login_required_if_public
def dwgupload(request, folder=None, gitdisable='no'):
'''Upload DRAWING files (tunnel or therion) into the upload folder in :drawings
AND registers it into the :drawings: git repo.
This does NOT use a Django model linked to a Django form. Just a simple Django form.
You will find the Django documentation on forms very confusing, This is simpler.
Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
We use get_or_create instead of simply creating a new object in case someone uploads the same file
several times in one session, and expects them to be overwritten in the database. Although
the actual file will be duplicated in the filesystem with different random name ending.
'''
def dwgvalid(name):
if name in [ '.gitignore', ]:
return False
if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt']:
return True # dangerous, we should check the actual file binary signature
return False
def dwgvaliddisp(name):
'''OK to display, even if we are not going to allow a new one to be uploaded
'''
if name in [ '.gitignore', ]:
return False
if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt', '.jpg', '.jpeg', '.png', '.pdf', '.top', '.topo']:
return True # dangerous, we should check the actual file binary signature
return False
filesaved = False
actual_saved = []
refused = []
doesnotexist = ''
# print(f'! - FORM dwgupload - start "{folder}" - gitdisable "{gitdisable}"')
if folder is None:
folder = "" # improve this later
dirpath = Path(settings.DRAWINGS_DATA)
urlfile = '/dwgdataraw'
urldir = '/dwgupload'
else:
dirpath = Path(settings.DRAWINGS_DATA, folder)
urlfile = Path('/dwgdataraw/') / folder
urldir = Path('/dwgupload/') / folder
form = FilesForm()
if request.method == 'POST':
form = FilesForm(request.POST,request.FILES)
if form.is_valid():
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist('uploadfiles')
fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder))
actual_saved = []
refused = []
# GIT see also core/views/expo.py editexpopage()
# GIT see also core/models/cave.py writetrogglefile()
if gitdisable != 'yes': # set in url 'dwguploadnogit/'
git = settings.GIT
else:
git = 'echo'
if multiple:
for f in multiple:
if dwgvalid(f.name):
try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file!
saved_filename = fs.save(f.name, content=f)
except:
print(f'\n !! Permissions failure ?! on attempting to save file {f.name}')
if 'saved_filename' in locals():
if saved_filename.is_file():
actual_saved.append(saved_filename)
if gitdisable != 'yes':
dr_add = subprocess.run([git, "add", saved_filename], cwd=dirpath, capture_output=True, text=True)
if dr_add.returncode != 0:
msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout + '\n\nreturn code: ' + str(dr_add.returncode)
message = f'CANNOT git on server for this file {saved_filename}. Edits saved but not added to git.\n\n' + msgdata
return render(request,'errors/generic.html', {'message': message})
dwgfile, created = DrawingFile.objects.get_or_create(dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size)
dwgfile.save()
else:
refused.append(f.name)
print(f'REFUSED {f.name}')
if actual_saved: # maybe all were refused by the suffix test in dwgvalid()
filesaved = True
if len(actual_saved) > 1:
dots = "..."
else:
dots = ""
if gitdisable != 'yes':
dr_commit = subprocess.run([git, "commit", "-m", f'Drawings upload - {actual_saved[0]}{dots}'], cwd=dirpath, capture_output=True, text=True)
# This produces return code = 1 if it commits OK
if dr_commit.returncode != 0:
msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout + '\n\nreturn code: ' + str(dr_commit.returncode)
message = f'Error code with git on server for this {actual_saved[0]}{dots}. Edits saved, added to git, but NOT committed.\n\n' + msgdata
return render(request,'errors/generic.html', {'message': message})
files = []
dirs = []
# print(f'! - FORM dwgupload - start {folder} \n"{dirpath}" \n"{dirpath.parent}" \n"{dirpath.exists()}"')
try:
for f in dirpath.iterdir():
if f.is_dir():
if f.name not in ['.git' ]:
dirs.append(f.name)
continue
if f.is_file():
if dwgvaliddisp(f.name):
files.append(f.name)
continue
except FileNotFoundError:
doesnotexist = True
if files:
files = sorted(files)
if dirs:
dirs = sorted(dirs)
return render(request, 'dwguploadform.html',
{'form': form, 'doesnotexist': doesnotexist, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved, 'refused': refused})
Reference in New Issue View Git Blame Copy Permalink