forked from expo/troggle
236 lines
9.7 KiB
Python
236 lines
9.7 KiB
Python
import re, os
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
from django import forms
|
|
|
|
from django.conf import settings
|
|
from django.urls import reverse
|
|
from django.db.models import Q
|
|
from django.http import HttpResponse, HttpResponseRedirect
|
|
from django.shortcuts import render
|
|
from django.template import Context, loader
|
|
from django.core.files.storage import FileSystemStorage, default_storage
|
|
|
|
from troggle.parsers.imports import import_caves, import_people, import_surveyscans
|
|
from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
|
|
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
|
|
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
|
|
from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
|
|
from troggle.core.models.survex import DrawingFile
|
|
from .auth import login_required_if_public
|
|
|
|
'''File upload 'views'
|
|
'''
|
|
|
|
todo = '''
|
|
- Write equivalent photo upload form system, similar to scanupload() but in expofiles/photos/
|
|
Need to validate it as being a valid image file, not a dubious script or hack
|
|
|
|
- Write equivalent GPX upload form system, similar to scanupload() but in expofiles/gpslogs/
|
|
Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
|
|
|
|
- Validate Tunnel & Therion files using an XML parser in dwgupload()
|
|
|
|
- Validate image files using a magic recogniser in scanupload()
|
|
|
|
- Enable folder creation in dwguploads or as a separate form
|
|
|
|
- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
|
|
'''
|
|
|
|
class FilesForm(forms.Form): # not a model-form, just a form-form
|
|
uploadfiles = forms.FileField()
|
|
|
|
@login_required_if_public
|
|
def scanupload(request, wallet=None):
|
|
'''Upload scanned image files into a wallet on /expofiles
|
|
This does NOT use a Django model linked to a Django form. Just a simple Django form.
|
|
You will find the Django documentation on forms very confusing, This is simpler.
|
|
'''
|
|
filesaved = False
|
|
actual_saved = []
|
|
# print(f'! - FORM scanupload - start {wallet}')
|
|
if wallet is None:
|
|
wallet = "2021#01" # improve this later
|
|
if not re.match('(19|20)\d\d:\d\d', wallet):
|
|
wallet = "2021:01" # improve this later
|
|
|
|
year = wallet[:4]
|
|
if int(year) < 1977:
|
|
year = "1977"
|
|
if int(year) > 2050:
|
|
year = "2050"
|
|
nexty = f'{int(year)+1}'
|
|
prevy = f'{int(year)-1}'
|
|
|
|
wnumber = wallet[5:]
|
|
next = f'{int(wnumber)+1:02d}'
|
|
prev = f'{int(wnumber)-1:02d}'
|
|
|
|
if int(wnumber) == 0:
|
|
prev = f'{int(wnumber):02d}'
|
|
|
|
context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty}
|
|
|
|
wallet = wallet.replace(':','#')
|
|
dirpath = Path(settings.SURVEY_SCANS, year, wallet)
|
|
|
|
form = FilesForm()
|
|
|
|
if request.method == 'POST':
|
|
form = FilesForm(request.POST,request.FILES)
|
|
if form.is_valid():
|
|
f = request.FILES["uploadfiles"]
|
|
multiple = request.FILES.getlist('uploadfiles')
|
|
fs = FileSystemStorage(os.path.join(settings.SURVEY_SCANS, year, wallet))
|
|
|
|
actual_saved = []
|
|
if multiple:
|
|
for f in multiple:
|
|
actual_saved.append( fs.save(f.name, content=f) )
|
|
# print(f'! - FORM scanupload multiple {actual_saved}')
|
|
filesaved = True
|
|
|
|
files = []
|
|
dirs = []
|
|
# print(f'! - FORM scanupload - start {wallet} {dirpath}')
|
|
try:
|
|
for f in dirpath.iterdir():
|
|
if f.is_dir():
|
|
dirs.append(f.name)
|
|
if f.is_file():
|
|
if f.name != 'contents.json' and f.name != 'walletindex.html':
|
|
files.append(f.name)
|
|
except FileNotFoundError:
|
|
files.append('(no wallet yet - would be created)')
|
|
if len(files) ==0 :
|
|
files.append('(no image files in wallet)')
|
|
else:
|
|
files = sorted(files)
|
|
|
|
if dirs:
|
|
dirs = sorted(dirs)
|
|
|
|
return render(request, 'scanuploadform.html',
|
|
{'form': form, 'wallet': wallet, **context, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved})
|
|
|
|
@login_required_if_public
|
|
def dwgupload(request, folder=None, gitdisable='no'):
|
|
'''Upload DRAWING files (tunnel or therion) into the upload folder in :drawings
|
|
AND registers it into the :drawings: git repo.
|
|
|
|
This does NOT use a Django model linked to a Django form. Just a simple Django form.
|
|
You will find the Django documentation on forms very confusing, This is simpler.
|
|
|
|
Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
|
|
|
|
|
|
We use get_or_create instead of simply creating a new object in case someone uploads the same file
|
|
several times in one session, and expects them to be overwritten in the database. Although
|
|
the actual file will be duplicated in the filesystem with different random name ending.
|
|
'''
|
|
def dwgvalid(name):
|
|
if name in [ '.gitignore', ]:
|
|
return False
|
|
if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt']:
|
|
return True # dangerous, we should check the actual file binary signature
|
|
return False
|
|
|
|
def dwgvaliddisp(name):
|
|
'''OK to display, even if we are not going to allow a new one to be uploaded
|
|
'''
|
|
if name in [ '.gitignore', ]:
|
|
return False
|
|
if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt', '.jpg', '.jpeg', '.png', '.pdf']:
|
|
return True # dangerous, we should check the actual file binary signature
|
|
return False
|
|
|
|
filesaved = False
|
|
actual_saved = []
|
|
refused = []
|
|
doesnotexist = ''
|
|
# print(f'! - FORM dwgupload - start "{folder}" - gitdisable "{gitdisable}"')
|
|
if folder is None:
|
|
folder = "" # improve this later
|
|
dirpath = Path(settings.DRAWINGS_DATA)
|
|
urlfile = '/dwgdataraw'
|
|
urldir = '/dwgupload'
|
|
else:
|
|
dirpath = Path(settings.DRAWINGS_DATA, folder)
|
|
urlfile = Path('/dwgdataraw/') / folder
|
|
urldir = Path('/dwgupload/') / folder
|
|
|
|
form = FilesForm()
|
|
|
|
if request.method == 'POST':
|
|
form = FilesForm(request.POST,request.FILES)
|
|
if form.is_valid():
|
|
f = request.FILES["uploadfiles"]
|
|
multiple = request.FILES.getlist('uploadfiles')
|
|
fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder))
|
|
|
|
actual_saved = []
|
|
refused = []
|
|
|
|
# GIT see also core/views/expo.py editexpopage()
|
|
# GIT see also core/models/cave.py writetrogglefile()
|
|
if gitdisable != 'yes': # set in url 'dwguploadnogit/'
|
|
git = settings.GIT
|
|
else:
|
|
git = 'echo'
|
|
|
|
if multiple:
|
|
for f in multiple:
|
|
if dwgvalid(f.name):
|
|
saved_filename = fs.save(f.name, content=f)
|
|
actual_saved.append(saved_filename)
|
|
if gitdisable != 'yes':
|
|
dr_add = subprocess.run([git, "add", saved_filename], cwd=dirpath, capture_output=True, text=True)
|
|
if dr_add.returncode != 0:
|
|
msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout + '\n\nreturn code: ' + str(dr_add.returncode)
|
|
message = f'CANNOT git on server for this file {saved_filename}. Edits saved but not added to git.\n\n' + msgdata
|
|
return render(request,'errors/generic.html', {'message': message})
|
|
dwgfile, created = DrawingFile.objects.get_or_create(dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size)
|
|
dwgfile.save()
|
|
else:
|
|
refused.append(f.name)
|
|
print(f'REFUSED {f.name}')
|
|
if actual_saved: # maybe all were refused by the suffix test in dwgvalid()
|
|
filesaved = True
|
|
if len(actual_saved) > 1:
|
|
dots = "..."
|
|
else:
|
|
dots = ""
|
|
if gitdisable != 'yes':
|
|
dr_commit = subprocess.run([git, "commit", "-m", f'Drawings upload - {actual_saved[0]}{dots}'], cwd=dirpath, capture_output=True, text=True)
|
|
# This produces return code = 1 if it commits OK
|
|
if dr_commit.returncode != 0:
|
|
msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout + '\n\nreturn code: ' + str(dr_commit.returncode)
|
|
message = f'Error code with git on server for this {actual_saved[0]}{dots}. Edits saved, added to git, but NOT committed.\n\n' + msgdata
|
|
return render(request,'errors/generic.html', {'message': message})
|
|
|
|
files = []
|
|
dirs = []
|
|
# print(f'! - FORM dwgupload - start {folder} \n"{dirpath}" \n"{dirpath.parent}" \n"{dirpath.exists()}"')
|
|
try:
|
|
for f in dirpath.iterdir():
|
|
if f.is_dir():
|
|
if f.name not in ['.git' ]:
|
|
dirs.append(f.name)
|
|
continue
|
|
if f.is_file():
|
|
if dwgvaliddisp(f.name):
|
|
files.append(f.name)
|
|
continue
|
|
except FileNotFoundError:
|
|
doesnotexist = True
|
|
if files:
|
|
files = sorted(files)
|
|
|
|
if dirs:
|
|
dirs = sorted(dirs)
|
|
|
|
return render(request, 'dwguploadform.html',
|
|
{'form': form, 'doesnotexist': doesnotexist, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved, 'refused': refused})
|