forked from expo/troggle
373 lines
15 KiB
Python
373 lines
15 KiB
Python
import re, os
|
|
import subprocess
|
|
import json
|
|
import settings
|
|
|
|
from pathlib import Path
|
|
|
|
from django import forms
|
|
|
|
from django.conf import settings
|
|
from django.urls import reverse
|
|
from django.db.models import Q
|
|
from django.http import HttpResponse, HttpResponseRedirect
|
|
from django.shortcuts import render
|
|
from django.template import Context, loader
|
|
from django.core.files.storage import FileSystemStorage, default_storage
|
|
|
|
#from troggle import settings
|
|
from troggle.parsers.imports import import_caves, import_people, import_surveyscans
|
|
from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
|
|
from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml
|
|
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
|
|
from troggle.core.models.troggle import DataIssue
|
|
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
|
|
from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
|
|
from troggle.core.models.survex import DrawingFile
|
|
from .auth import login_required_if_public
|
|
#from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt
|
|
|
|
'''File upload 'views'
|
|
'''
|
|
|
|
todo = '''
|
|
- Write equivalent photo upload form system, similar to scanupload() but in expofiles/photos/
|
|
Need to validate it as being a valid image file, not a dubious script or hack
|
|
|
|
- Write equivalent GPX upload form system, similar to scanupload() but in expofiles/gpslogs/
|
|
Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
|
|
|
|
- Validate Tunnel & Therion files using an XML parser in dwgupload()
|
|
|
|
- Validate image files using a magic recogniser in scanupload()
|
|
|
|
- Enable folder creation in dwguploads or as a separate form
|
|
|
|
- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
|
|
'''
|
|
|
|
class FilesForm(forms.Form): # not a model-form, just a form-form
|
|
uploadfiles = forms.FileField()
|
|
|
|
class TextForm(forms.Form): # not a model-form, just a form-form
|
|
photographer = forms.CharField(strip=True)
|
|
|
|
@login_required_if_public
|
|
def scanupload(request, wallet=None):
|
|
'''Upload scanned image files into a wallet on /expofiles
|
|
This does NOT use a Django model linked to a Django form. Just a simple Django form.
|
|
You will find the Django documentation on forms very confusing, This is simpler.
|
|
'''
|
|
filesaved = False
|
|
actual_saved = []
|
|
# print(f'! - FORM scanupload - start {wallet}')
|
|
if wallet is None:
|
|
wallet = "2021#01" # improve this later
|
|
if not re.match('(19|20)\d\d:\d\d', wallet):
|
|
wallet = "2021:01" # improve this later
|
|
|
|
year = wallet[:4]
|
|
if int(year) < 1977:
|
|
year = "1977"
|
|
if int(year) > 2050:
|
|
year = "2050"
|
|
nexty = f'{int(year)+1}'
|
|
prevy = f'{int(year)-1}'
|
|
|
|
wnumber = wallet[5:]
|
|
next = f'{int(wnumber)+1:02d}'
|
|
prev = f'{int(wnumber)-1:02d}'
|
|
|
|
if int(wnumber) == 0:
|
|
prev = f'{int(wnumber):02d}'
|
|
|
|
wurl = f"/scanupload/{wallet}"
|
|
wallet = wallet.replace(':','#')
|
|
dirpath = Path(settings.SURVEY_SCANS, year, wallet)
|
|
contents_path = dirpath / contentsjson
|
|
walletdata = dirpath / contentsjson
|
|
|
|
form = FilesForm()
|
|
|
|
if request.method == 'POST':
|
|
form = FilesForm(request.POST,request.FILES)
|
|
#print(f'! - FilesForm POSTED')
|
|
|
|
if form.is_valid():
|
|
f = request.FILES["uploadfiles"]
|
|
multiple = request.FILES.getlist('uploadfiles')
|
|
fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary
|
|
|
|
actual_saved = []
|
|
if multiple:
|
|
for f in multiple:
|
|
actual_saved.append( fs.save(f.name, content=f) )
|
|
# print(f'! - FORM scanupload multiple {actual_saved}')
|
|
filesaved = True
|
|
|
|
# Wallet folder created, but index and contents.json need to be created.
|
|
|
|
if not contents_path.is_file(): # double-check
|
|
with open(contents_path, "w") as json_file:
|
|
json.dump(wallet_blank_json, json_file, sort_keys=True, indent = 1)
|
|
index_path = dirpath / indexhtml
|
|
if not index_path.is_file(): # double-check
|
|
thishtml = wallet_blank_html.replace("YEAR", str(year))
|
|
thishtml = thishtml.replace("WALLET", str(wallet))
|
|
with open(index_path, "w") as html_file:
|
|
html_file.write(thishtml )
|
|
|
|
files = []
|
|
dirs = []
|
|
# print(f'! - FORM scanupload - start {wallet} {dirpath}')
|
|
if dirpath.is_dir():
|
|
create = False
|
|
try:
|
|
for f in dirpath.iterdir():
|
|
if f.is_dir():
|
|
dirs.append(f.name)
|
|
if f.is_file():
|
|
if f.name != 'contents.json' and f.name != 'walletindex.html':
|
|
files.append(f.name)
|
|
except FileNotFoundError:
|
|
files.append('(no wallet yet. It would be created if you upload a scan)')
|
|
else:
|
|
create = True
|
|
|
|
if len(files) >0 :
|
|
files = sorted(files)
|
|
|
|
if dirs:
|
|
dirs = sorted(dirs)
|
|
|
|
waldata = []
|
|
if contents_path.is_file():
|
|
with open(contents_path) as json_file:
|
|
try:
|
|
waldata = json.load(json_file)
|
|
except:
|
|
message = f"! Failed to load {contents_path} JSON file"
|
|
print(message)
|
|
DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
|
|
raise
|
|
if not waldata["people"]:
|
|
waldata["people"]=["NOBODY"]
|
|
|
|
cave =""
|
|
psg = ""
|
|
if waldata["cave"]:
|
|
cave = waldata["cave"]
|
|
if waldata["name"]:
|
|
psg = waldata["name"]
|
|
if waldata["survex file"]:
|
|
if not isinstance(waldata["survex file"], list):
|
|
waldata["survex file"] = [waldata["survex file"]]
|
|
|
|
for svx in waldata["survex file"]:
|
|
print(f'{svx}')
|
|
if not (Path(settings.SURVEX_DATA) / svx).is_file():
|
|
message = f"! Incorrect survex file in wallet data: {svx} not found in LOSER repo"
|
|
print(message)
|
|
DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
|
|
|
|
|
|
context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty,
|
|
'files': files, 'dirs': dirs, 'waldata': waldata, 'create': create,
|
|
'filesaved': filesaved, 'actual_saved': actual_saved }
|
|
|
|
return render(request, 'scanuploadform.html',
|
|
{'form': form, 'wallet': wallet, **context, 'cave': cave, 'psg': psg})
|
|
|
|
@login_required_if_public
|
|
def photoupload(request, folder=None):
|
|
'''Upload photo image files into /expofiles/photos/<year>/<photographer>/
|
|
This does NOT use a Django model linked to a Django form. Just a simple Django form.
|
|
You will find the Django documentation on forms very confusing, This is simpler.
|
|
'''
|
|
year = settings.PHOTOS_YEAR
|
|
filesaved = False
|
|
actual_saved = []
|
|
|
|
context = {'year': year, 'placeholder': "AnathemaDevice"}
|
|
|
|
yearpath = Path(settings.PHOTOS_ROOT, year)
|
|
|
|
if folder == str(year) or folder == str(year) +"/":
|
|
folder = None
|
|
|
|
if folder is None:
|
|
folder = "" # improve this later
|
|
dirpath = Path(settings.PHOTOS_ROOT, year)
|
|
urlfile = f'/expofiles/photos/{year}'
|
|
urldir = f'/photoupload/{year}'
|
|
else: # it will contain the year as well as the photographer
|
|
dirpath = Path(settings.PHOTOS_ROOT, folder)
|
|
if dirpath.is_dir():
|
|
urlfile = f'/expofiles/photos/{folder}'
|
|
urldir = Path('/photoupload') / folder
|
|
else:
|
|
folder = "" # improve this later
|
|
dirpath = Path(settings.PHOTOS_ROOT, year)
|
|
urlfile = f'/expofiles/photos/{year}'
|
|
urldir = f'/photoupload/{year}'
|
|
|
|
|
|
form = FilesForm()
|
|
formd = TextForm()
|
|
|
|
if request.method == 'POST':
|
|
if "photographer" in request.POST:
|
|
formd = TextForm(request.POST)
|
|
if formd.is_valid():
|
|
newphotographer = request.POST["photographer"]
|
|
(yearpath / newphotographer).mkdir(exist_ok=True)
|
|
else:
|
|
form = FilesForm(request.POST,request.FILES)
|
|
if form.is_valid():
|
|
f = request.FILES["uploadfiles"]
|
|
multiple = request.FILES.getlist('uploadfiles')
|
|
fs = FileSystemStorage(dirpath)
|
|
|
|
actual_saved = []
|
|
if multiple:
|
|
for f in multiple:
|
|
actual_saved.append( fs.save(f.name, content=f) )
|
|
filesaved = True
|
|
files = []
|
|
dirs = []
|
|
try:
|
|
for f in dirpath.iterdir():
|
|
if f.is_dir():
|
|
dirs.append(f.name)
|
|
if f.is_file():
|
|
files.append(f.name)
|
|
except FileNotFoundError:
|
|
files.append('(no folder yet - would be created)')
|
|
if len(files) >0 :
|
|
files = sorted(files)
|
|
|
|
if dirs:
|
|
dirs = sorted(dirs)
|
|
|
|
return render(request, 'photouploadform.html',
|
|
{'form': form, **context, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved})
|
|
|
|
@login_required_if_public
|
|
def dwgupload(request, folder=None, gitdisable='no'):
|
|
'''Upload DRAWING files (tunnel or therion) into the upload folder in :drawings
|
|
AND registers it into the :drawings: git repo.
|
|
|
|
This does NOT use a Django model linked to a Django form. Just a simple Django form.
|
|
You will find the Django documentation on forms very confusing, This is simpler.
|
|
|
|
Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
|
|
|
|
|
|
We use get_or_create instead of simply creating a new object in case someone uploads the same file
|
|
several times in one session, and expects them to be overwritten in the database. Although
|
|
the actual file will be duplicated in the filesystem with different random name ending.
|
|
'''
|
|
def dwgvalid(name):
|
|
if name in [ '.gitignore', ]:
|
|
return False
|
|
if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt']:
|
|
return True # dangerous, we should check the actual file binary signature
|
|
return False
|
|
|
|
def dwgvaliddisp(name):
|
|
'''OK to display, even if we are not going to allow a new one to be uploaded
|
|
'''
|
|
if name in [ '.gitignore', ]:
|
|
return False
|
|
if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt', '.jpg', '.jpeg', '.png', '.pdf', '.topo']:
|
|
return True # dangerous, we should check the actual file binary signature
|
|
return False
|
|
|
|
filesaved = False
|
|
actual_saved = []
|
|
refused = []
|
|
doesnotexist = ''
|
|
# print(f'! - FORM dwgupload - start "{folder}" - gitdisable "{gitdisable}"')
|
|
if folder is None:
|
|
folder = "" # improve this later
|
|
dirpath = Path(settings.DRAWINGS_DATA)
|
|
urlfile = '/dwgdataraw'
|
|
urldir = '/dwgupload'
|
|
else:
|
|
dirpath = Path(settings.DRAWINGS_DATA, folder)
|
|
urlfile = Path('/dwgdataraw/') / folder
|
|
urldir = Path('/dwgupload/') / folder
|
|
|
|
form = FilesForm()
|
|
|
|
if request.method == 'POST':
|
|
form = FilesForm(request.POST,request.FILES)
|
|
if form.is_valid():
|
|
f = request.FILES["uploadfiles"]
|
|
multiple = request.FILES.getlist('uploadfiles')
|
|
fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder))
|
|
|
|
actual_saved = []
|
|
refused = []
|
|
|
|
# GIT see also core/views/expo.py editexpopage()
|
|
# GIT see also core/models/cave.py writetrogglefile()
|
|
if gitdisable != 'yes': # set in url 'dwguploadnogit/'
|
|
git = settings.GIT
|
|
else:
|
|
git = 'echo'
|
|
|
|
if multiple:
|
|
for f in multiple:
|
|
if dwgvalid(f.name):
|
|
saved_filename = fs.save(f.name, content=f)
|
|
actual_saved.append(saved_filename)
|
|
if gitdisable != 'yes':
|
|
dr_add = subprocess.run([git, "add", saved_filename], cwd=dirpath, capture_output=True, text=True)
|
|
if dr_add.returncode != 0:
|
|
msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout + '\n\nreturn code: ' + str(dr_add.returncode)
|
|
message = f'CANNOT git on server for this file {saved_filename}. Edits saved but not added to git.\n\n' + msgdata
|
|
return render(request,'errors/generic.html', {'message': message})
|
|
dwgfile, created = DrawingFile.objects.get_or_create(dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size)
|
|
dwgfile.save()
|
|
else:
|
|
refused.append(f.name)
|
|
print(f'REFUSED {f.name}')
|
|
if actual_saved: # maybe all were refused by the suffix test in dwgvalid()
|
|
filesaved = True
|
|
if len(actual_saved) > 1:
|
|
dots = "..."
|
|
else:
|
|
dots = ""
|
|
if gitdisable != 'yes':
|
|
dr_commit = subprocess.run([git, "commit", "-m", f'Drawings upload - {actual_saved[0]}{dots}'], cwd=dirpath, capture_output=True, text=True)
|
|
# This produces return code = 1 if it commits OK
|
|
if dr_commit.returncode != 0:
|
|
msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout + '\n\nreturn code: ' + str(dr_commit.returncode)
|
|
message = f'Error code with git on server for this {actual_saved[0]}{dots}. Edits saved, added to git, but NOT committed.\n\n' + msgdata
|
|
return render(request,'errors/generic.html', {'message': message})
|
|
|
|
files = []
|
|
dirs = []
|
|
# print(f'! - FORM dwgupload - start {folder} \n"{dirpath}" \n"{dirpath.parent}" \n"{dirpath.exists()}"')
|
|
try:
|
|
for f in dirpath.iterdir():
|
|
if f.is_dir():
|
|
if f.name not in ['.git' ]:
|
|
dirs.append(f.name)
|
|
continue
|
|
if f.is_file():
|
|
if dwgvaliddisp(f.name):
|
|
files.append(f.name)
|
|
continue
|
|
except FileNotFoundError:
|
|
doesnotexist = True
|
|
if files:
|
|
files = sorted(files)
|
|
|
|
if dirs:
|
|
dirs = sorted(dirs)
|
|
|
|
return render(request, 'dwguploadform.html',
|
|
{'form': form, 'doesnotexist': doesnotexist, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved, 'refused': refused})
|