expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-26 17:21:52 +00:00
Code Issues 19 Projects 3 Activity
d3ddcba313
troggle/core/views/uploads.py
Philip Sargent d3ddcba313 bug fixes
2022-03-15 23:00:23 +00:00

376 lines
15 KiB
Python
Raw Blame History

import re, os
import subprocess
import json
import settings
from pathlib import Path
from django import forms
from django.conf import settings
from django.urls import reverse
from django.db.models import Q
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.template import Context, loader
from django.core.files.storage import FileSystemStorage, default_storage
#from troggle import settings
from troggle.parsers.imports import import_caves, import_people, import_surveyscans
from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import DataIssue
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
from troggle.core.models.survex import DrawingFile
from .auth import login_required_if_public
#from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt
'''File upload 'views'
'''
todo = '''
- Write equivalent photo upload form system, similar to scanupload() but in expofiles/photos/
Need to validate it as being a valid image file, not a dubious script or hack
- Write equivalent GPX upload form system, similar to scanupload() but in expofiles/gpslogs/
Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
- Validate Tunnel & Therion files using an XML parser in dwgupload()
- Validate image files using a magic recogniser in scanupload()
- Enable folder creation in dwguploads or as a separate form
- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
'''
class FilesForm(forms.Form): # not a model-form, just a form-form
uploadfiles = forms.FileField()
class TextForm(forms.Form): # not a model-form, just a form-form
photographer = forms.CharField(strip=True)
@login_required_if_public
def scanupload(request, wallet=None):
'''Upload scanned image files into a wallet on /expofiles
This does NOT use a Django model linked to a Django form. Just a simple Django form.
You will find the Django documentation on forms very confusing, This is simpler.
'''
filesaved = False
actual_saved = []
# print(f'! - FORM scanupload - start {wallet}')
if wallet is None:
wallet = "2021#01" # improve this later
if not re.match('(19|20)\d\d:\d\d', wallet):
wallet = "2021:01" # improve this later
year = wallet[:4]
if int(year) < 1977:
year = "1977"
if int(year) > 2050:
year = "2050"
nexty = f'{int(year)+1}'
prevy = f'{int(year)-1}'
wnumber = wallet[5:]
next = f'{int(wnumber)+1:02d}'
prev = f'{int(wnumber)-1:02d}'
if int(wnumber) == 0:
prev = f'{int(wnumber):02d}'
wurl = f"/scanupload/{wallet}"
wallet = wallet.replace(':','#')
dirpath = Path(settings.SURVEY_SCANS, year, wallet)
contents_path = dirpath / contentsjson
walletdata = dirpath / contentsjson
form = FilesForm()
if request.method == 'POST':
form = FilesForm(request.POST,request.FILES)
#print(f'! - FilesForm POSTED')
if form.is_valid():
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist('uploadfiles')
fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary
actual_saved = []
if multiple:
for f in multiple:
actual_saved.append( fs.save(f.name, content=f) )
# print(f'! - FORM scanupload multiple {actual_saved}')
filesaved = True
# Wallet folder created, but index and contents.json need to be created.
if not contents_path.is_file(): # double-check
with open(contents_path, "w") as json_file:
json.dump(wallet_blank_json, json_file, sort_keys=True, indent = 1)
index_path = dirpath / indexhtml
if not index_path.is_file(): # double-check
thishtml = wallet_blank_html.replace("YEAR", str(year))
thishtml = thishtml.replace("WALLET", str(wallet))
with open(index_path, "w") as html_file:
html_file.write(thishtml )
files = []
dirs = []
# print(f'! - FORM scanupload - start {wallet} {dirpath}')
if dirpath.is_dir():
create = False
try:
for f in dirpath.iterdir():
if f.is_dir():
dirs.append(f.name)
if f.is_file():
if f.name != 'contents.json' and f.name != 'walletindex.html':
files.append(f.name)
except FileNotFoundError:
files.append('(no wallet yet. It would be created if you upload a scan)')
else:
create = True
if len(files) >0 :
files = sorted(files)
if dirs:
dirs = sorted(dirs)
waldata = {}
if contents_path.is_file():
with open(contents_path) as json_file:
try:
waldata = json.load(json_file)
except:
message = f"! {wallet} Failed to load {contents_path} JSON file"
print(message)
DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
raise
cave =""
psg = ""
if waldata:
if not waldata["people"]:
waldata["people"]=["NOBODY"]
if waldata["cave"]:
cave = waldata["cave"]
if waldata["name"]:
psg = waldata["name"]
if waldata["survex file"]:
if not isinstance(waldata["survex file"], list):
waldata["survex file"] = [waldata["survex file"]]
for svx in waldata["survex file"]:
print(f'{svx}')
if not (Path(settings.SURVEX_DATA) / svx).is_file():
message = f"! {wallet} Incorrect survex file in wallet data: {svx} not found in LOSER repo"
print(message)
DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty,
'files': files, 'dirs': dirs, 'waldata': waldata, 'create': create,
'filesaved': filesaved, 'actual_saved': actual_saved }
return render(request, 'scanuploadform.html',
{'form': form, 'wallet': wallet, **context, 'cave': cave, 'psg': psg})
@login_required_if_public
def photoupload(request, folder=None):
'''Upload photo image files into /expofiles/photos/<year>/<photographer>/
This does NOT use a Django model linked to a Django form. Just a simple Django form.
You will find the Django documentation on forms very confusing, This is simpler.
'''
year = settings.PHOTOS_YEAR
filesaved = False
actual_saved = []
context = {'year': year, 'placeholder': "AnathemaDevice"}
yearpath = Path(settings.PHOTOS_ROOT, year)
if folder == str(year) or folder == str(year) +"/":
folder = None
if folder is None:
folder = "" # improve this later
dirpath = Path(settings.PHOTOS_ROOT, year)
urlfile = f'/expofiles/photos/{year}'
urldir = f'/photoupload/{year}'
else: # it will contain the year as well as the photographer
dirpath = Path(settings.PHOTOS_ROOT, folder)
if dirpath.is_dir():
urlfile = f'/expofiles/photos/{folder}'
urldir = Path('/photoupload') / folder
else:
folder = "" # improve this later
dirpath = Path(settings.PHOTOS_ROOT, year)
urlfile = f'/expofiles/photos/{year}'
urldir = f'/photoupload/{year}'
form = FilesForm()
formd = TextForm()
if request.method == 'POST':
if "photographer" in request.POST:
formd = TextForm(request.POST)
if formd.is_valid():
newphotographer = request.POST["photographer"]
(yearpath / newphotographer).mkdir(exist_ok=True)
else:
form = FilesForm(request.POST,request.FILES)
if form.is_valid():
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist('uploadfiles')
# NO CHECK that the files being uploaded are image files
fs = FileSystemStorage(dirpath)
actual_saved = []
if multiple:
for f in multiple:
actual_saved.append( fs.save(f.name, content=f) )
filesaved = True
files = []
dirs = []
try:
for f in dirpath.iterdir():
if f.is_dir():
dirs.append(f.name)
if f.is_file():
files.append(f.name)
except FileNotFoundError:
files.append('(no folder yet - would be created)')
if len(files) >0 :
files = sorted(files)
if dirs:
dirs = sorted(dirs)
return render(request, 'photouploadform.html',
{'form': form, **context, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved})
@login_required_if_public
def dwgupload(request, folder=None, gitdisable='no'):
'''Upload DRAWING files (tunnel or therion) into the upload folder in :drawings
AND registers it into the :drawings: git repo.
This does NOT use a Django model linked to a Django form. Just a simple Django form.
You will find the Django documentation on forms very confusing, This is simpler.
Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
We use get_or_create instead of simply creating a new object in case someone uploads the same file
several times in one session, and expects them to be overwritten in the database. Although
the actual file will be duplicated in the filesystem with different random name ending.
'''
def dwgvalid(name):
if name in [ '.gitignore', ]:
return False
if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt']:
return True # dangerous, we should check the actual file binary signature
return False
def dwgvaliddisp(name):
'''OK to display, even if we are not going to allow a new one to be uploaded
'''
if name in [ '.gitignore', ]:
return False
if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt', '.jpg', '.jpeg', '.png', '.pdf', '.top', '.topo']:
return True # dangerous, we should check the actual file binary signature
return False
filesaved = False
actual_saved = []
refused = []
doesnotexist = ''
# print(f'! - FORM dwgupload - start "{folder}" - gitdisable "{gitdisable}"')
if folder is None:
folder = "" # improve this later
dirpath = Path(settings.DRAWINGS_DATA)
urlfile = '/dwgdataraw'
urldir = '/dwgupload'
else:
dirpath = Path(settings.DRAWINGS_DATA, folder)
urlfile = Path('/dwgdataraw/') / folder
urldir = Path('/dwgupload/') / folder
form = FilesForm()
if request.method == 'POST':
form = FilesForm(request.POST,request.FILES)
if form.is_valid():
f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist('uploadfiles')
fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder))
actual_saved = []
refused = []
# GIT see also core/views/expo.py editexpopage()
# GIT see also core/models/cave.py writetrogglefile()
if gitdisable != 'yes': # set in url 'dwguploadnogit/'
git = settings.GIT
else:
git = 'echo'
if multiple:
for f in multiple:
if dwgvalid(f.name):
saved_filename = fs.save(f.name, content=f)
actual_saved.append(saved_filename)
if gitdisable != 'yes':
dr_add = subprocess.run([git, "add", saved_filename], cwd=dirpath, capture_output=True, text=True)
if dr_add.returncode != 0:
msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout + '\n\nreturn code: ' + str(dr_add.returncode)
message = f'CANNOT git on server for this file {saved_filename}. Edits saved but not added to git.\n\n' + msgdata
return render(request,'errors/generic.html', {'message': message})
dwgfile, created = DrawingFile.objects.get_or_create(dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size)
dwgfile.save()
else:
refused.append(f.name)
print(f'REFUSED {f.name}')
if actual_saved: # maybe all were refused by the suffix test in dwgvalid()
filesaved = True
if len(actual_saved) > 1:
dots = "..."
else:
dots = ""
if gitdisable != 'yes':
dr_commit = subprocess.run([git, "commit", "-m", f'Drawings upload - {actual_saved[0]}{dots}'], cwd=dirpath, capture_output=True, text=True)
# This produces return code = 1 if it commits OK
if dr_commit.returncode != 0:
msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout + '\n\nreturn code: ' + str(dr_commit.returncode)
message = f'Error code with git on server for this {actual_saved[0]}{dots}. Edits saved, added to git, but NOT committed.\n\n' + msgdata
return render(request,'errors/generic.html', {'message': message})
files = []
dirs = []
# print(f'! - FORM dwgupload - start {folder} \n"{dirpath}" \n"{dirpath.parent}" \n"{dirpath.exists()}"')
try:
for f in dirpath.iterdir():
if f.is_dir():
if f.name not in ['.git' ]:
dirs.append(f.name)
continue
if f.is_file():
if dwgvaliddisp(f.name):
files.append(f.name)
continue
except FileNotFoundError:
doesnotexist = True
if files:
files = sorted(files)
if dirs:
dirs = sorted(dirs)
return render(request, 'dwguploadform.html',
{'form': form, 'doesnotexist': doesnotexist, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved, 'refused': refused})
Reference in New Issue View Git Blame Copy Permalink