troggle/core/views/uploads.py

import re, os
import subprocess
import json
import settings
import urllib

from pathlib import Path

from django import forms

from django.conf import settings
from django.urls import reverse
from django.db.models import Q
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.template import Context, loader
from django.core.files.storage import FileSystemStorage, default_storage


#from troggle import settings
from troggle.parsers.imports import import_caves, import_people, import_surveyscans
from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import DataIssue
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
from troggle.core.models.survex import DrawingFile
from troggle.core.views.scans import oldwallet, walletindex

from .auth import login_required_if_public
#from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt

'''File upload 'views' 
'''

todo = '''
- Need to validate uploaded file as being a valid image file, not a dubious script or hack

- Write equivalent GPX upload form system, similar to scanupload() but in expofiles/gpslogs/
  Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack
  
- Validate Tunnel & Therion files using an XML parser in dwgupload(). Though Julian says tunnel is only mostly correct XML

- Validate image files using a magic recogniser in scanupload() https://pypi.org/project/reportlab/ or 
  https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file

- Enable folder creation in dwguploads or as a separate form

- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
'''

class FilesForm(forms.Form):  # not a model-form, just a form-form
    uploadfiles = forms.FileField() 

class TextForm(forms.Form):  # not a model-form, just a form-form
    photographer = forms.CharField(strip=True) 

class WalletForm(forms.Form):  # not a model-form, just a form-form
    descriptionw = forms.CharField(strip=True, required=False) 
    people = forms.CharField(strip=True, required=False) 
    survexnr = forms.CharField(strip=True, required=False) 
    qmsw = forms.CharField(strip=True, required=False) 
    date = forms.CharField(strip=True, required=True) # the only required field
    websiteupt = forms.CharField(strip=True, required=False) 
    elevnr = forms.CharField(strip=True, required=False) 
    cave = forms.CharField(strip=True, required=False) 
    psg = forms.CharField(strip=True, required=False) 
    plannr = forms.CharField(strip=True, required=False) 
    electronic = forms.CharField(strip=True, required=False) 
    pland = forms.CharField(strip=True, required=False) 
    elevd = forms.CharField(strip=True, required=False) 
    url = forms.CharField(strip=True, required=False) 
    survex = forms.CharField(strip=True, required=False) 

xlate = {"url": "description url",
        "descriptionw": "description written",
        "people": "people",
        "date": "date",
        "cave": "cave",
        "people": "people",
        "plannr": "plan not required",
        "survexnr": "survex not required",
        "qmsw": "qms written",
        "elevnr": "elev not required",
        "websiteupt": "website updated",
        "electronic": "electronic survey",
        "pland": "plan drawn",
        "elevd": "elev drawn",
        "psg": "name",
        "survex": "survex file",
        }

# @login_required_if_public
def scanupload(request, path=None):
    '''Upload scanned image files into a wallet on /expofiles
    Also display and edit the contents.json data in the wallet.
    
    This does NOT use a Django model linked to a Django form. Just a simple Django form.
    You will find the Django documentation on forms very confusing, This is simpler.
    '''
    filesaved = False
    actual_saved = []
    
    checkboxes = ["description written", "survex not required", "qms written", "website updated",
            "plan not required", "plan drawn", "elev not required", "elev drawn", "electronic survey" ]
    if path:
        wallet = urllib.parse.unquote(path)
    else:
        wallet = "2022#01" # improve this later

    year = wallet[:4]
    if int(year) <= 1998: 
        #print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET')
        return(oldwallet(request, path))
    
    if str(wallet).lower().endswith('indexpages'):
        # print(f'! - FORM scanupload - start {wallet} REDIRECT TO OLDWALLET')
        return(walletindex(request, path))

    if not re.match('(19|20)\d\d[:#]\d\d', wallet):
        wallet = "2022:01" # improve this later
    # print(f'! - FORM scanupload - start {wallet}')
    
    if path:
        pass
        # print(f'! - FORM scanupload - start wallet:{wallet}: path:{path}:')
    if int(year) < 1977: 
        year = "1977"
    if int(year) > 2050: 
        year = "2050"
    nexty = f'{int(year)+1}'
    prevy = f'{int(year)-1}'


    wnumber = wallet[5:]
    next = f'{int(wnumber)+1:02d}'
    prev = f'{int(wnumber)-1:02d}'
    
    if int(wnumber) == 0:
        prev = f'{int(wnumber):02d}'

    wurl = f"/scanupload/{wallet}".replace('#', ':')     
    wallet = wallet.replace(':','#')
    dirpath = Path(settings.SCANS_ROOT, year, wallet)
    contents_path = dirpath / contentsjson
    walletdata = dirpath / contentsjson
    
    form = FilesForm()
   
    if request.method == 'POST':
        if "psg" in request.POST:              
            formj = WalletForm(request.POST)
            # Beware. All fields returned as strings. Must re-type them as lists etc. before using or re-saving
            # Also lots of hassle with lists of strings interpreted as a single string
            # Unset checkboxes do not return any value, checked ones return "True". So need initialising to False
            if formj.is_valid():
                #print(f'--- JSON Update form is VALID, saving to {contents_path}')
                posted = request.POST.copy()
                posted.pop("csrfmiddlewaretoken") # discard this
                wd = wallet_blank_json
                for f in checkboxes:
                    wd[f] = False
                    #print(f'--- wd ${f}$  - {wd[f]}')
                for f in posted:
                    wd[xlate[f]] = posted[f].replace("\'", "\"")

                    if posted[f] =="True":
                        wd[xlate[f]] = True

                wd["people"] = wd["people"][1:-1].replace("\"", "").split(",")
                for i, elem in enumerate(wd["people"]):
                    wd["people"][i] = elem.strip()
                    
                #print(f'--- ${wd["survex file"]}$  - {type(wd["survex file"])}')
                if wd["survex file"]: # allow for no survex file at all
                    if wd["survex file"][0] == '[':
                        wd["survex file"] = wd["survex file"][1:-1]
                    wd["survex file"] = wd["survex file"].replace("\"", "").split(",")
                    for i, elem in enumerate(wd["survex file"]):
                       wd["survex file"][i] = elem.strip()
                    #print(f'--- {wd["survex file"]} - {type(wd["survex file"])}')

                with open(contents_path, "w") as jfile:
                    json.dump(wd, jfile, indent = 1)
                # print(f'--- FINISHED saving to JSON\n')
            else:
                print(f'--- INVALID JSON Update form submitted')
                print(formj.errors)
                return render(request,'errors/generic.html', {'message': formj.errors})

        else:
            form = FilesForm(request.POST,request.FILES)
     
            if form.is_valid():
                f = request.FILES["uploadfiles"]
                multiple = request.FILES.getlist('uploadfiles')
                fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary
               
                actual_saved = []
                if multiple:
                    for f in multiple:
                        try: # crashes in Django os.chmod call if on WSL, but does save file!
                            saved_filename = fs.save(f.name, content=f) 
                        except:
                            print(f'\n !! Permissions failure ?! on attempting to save file {f.name}')
                            if 'saved_filename' in locals():
                                if saved_filename.is_file():
                                    actual_saved.append(saved_filename)
                                    # print(f'! - FORM scanupload multiple {actual_saved}')
                                    filesaved = True
                
                if not contents_path.is_file(): # double-check
                    with open(contents_path, "w") as json_file:
                        json.dump(wallet_blank_json, json_file, sort_keys=True, indent = 1) 
                index_path = dirpath / indexhtml
                if not index_path.is_file(): # double-check
                    thishtml = wallet_blank_html.replace("YEAR", str(year))
                    thishtml = thishtml.replace("WALLET", str(wallet))
                    with open(index_path, "w") as html_file:
                        html_file.write(thishtml ) 
      
    files = []
    dirs = []
    # print(f'! - FORM scanupload - start {wallet} {dirpath}')
    if dirpath.is_dir():
        create = False
        try:
            for f in dirpath.iterdir():
                if f.is_dir():
                    dirs.append(f.name)
                if f.is_file():
                    if f.name != 'contents.json' and f.name != 'walletindex.html':
                        files.append(f.name)
        except FileNotFoundError:
            files.append('(no wallet yet. It would be created if you upload a scan)')
    else:
           create = True
         
    if len(files) >0 :
        files = sorted(files)
        
    if dirs:
        dirs = sorted(dirs)
        
    waldata = {}   
    if contents_path.is_file():
        with open(contents_path) as json_file:
            try:
                waldata = json.load(json_file)
            except:
                message = f"! {wallet} Failed to load {contents_path} JSON file"
                print(message)
                DataIssue.objects.create(parser='scans', message=message, url=wurl)  # set URL to this wallet folder
                raise
    cave =""
    psg = ""
    chkplannr = ""
    chkpland = ""
    svxfiles = []
    checked = {}
    if waldata:
        if not waldata["people"]:
            waldata["people"]=["NOBODY"]
        if not type(waldata["people"])==list: 
            if waldata["people"][0] == '"':
                waldata["people"] = waldata["people"][1:-1]
            waldata["people"] = list(waldata["people"])

        if not waldata["description url"]:
            waldata["description url"]=""
        if waldata["cave"]:
            cave =  waldata["cave"]
        if waldata["name"]:
            psg =  waldata["name"]

        if waldata["survex file"]:
            #print(f'+++ ${waldata["survex file"]}$ {type(waldata["survex file"])}')
            if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
                #print(f'+++ NOT A LIST: {waldata["survex file"]} {type(waldata["survex file"])}')
                waldata["survex file"] = [waldata["survex file"]]
                 
        for svx in waldata["survex file"]:
            svxfiles.append(svx)
            if not (Path(settings.SURVEX_DATA) / svx).is_file():
                    message = f"! {wallet} Incorrect survex file in wallet data: {svx} not found in LOSER repo"
                    print(message)
                    DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder

        for f in checkboxes:
             if waldata[f]:
                 checked[f] = "checked"
             
        context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty,
                'files': files, 'dirs': dirs, 'waldata': waldata, 'svxfiles': svxfiles,
                'checked': checked,
                'create': create, 
                'people':  waldata["people"], 'peoplesize': str(len(str(waldata["people"]))),
                'filesaved': filesaved, 'actual_saved': actual_saved }

        return render(request, 'scanuploadform.html', 
            {'form': form, 'wallet': wallet, **context, 
            'date': waldata["date"],
            'url':  waldata["description url"], 'urlsize': str(len(str(waldata["description url"]))),
            'survex': waldata["survex file"], 'survexsize': str(len(str(waldata["survex file"]))),
            'cave': cave, 'psg': psg, 'psgsize': str(max(12,len(str(psg))))})
    else: # no wallet here
        context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty,
                'files': files, 'dirs': dirs, 'waldata': waldata, 'svxfiles': svxfiles,
                'checked': checked,
                'create': create, 
                'people':  "", 'peoplesize': 12,
                'filesaved': filesaved, 'actual_saved': actual_saved }
        
        return render(request, 'scanuploadform.html', 
            {'form': form, 'wallet': wallet, **context, 
            'date': "",
            'url':  "", 'urlsize': 12,
            'survex': "", 'survexsize': 12,
            'cave': cave, 'psg': psg, 'psgsize': 12})

@login_required_if_public
def photoupload(request, folder=None):
    '''Upload photo image files into /expofiles/photos/<year>/<photographer>/
    This does NOT use a Django model linked to a Django form. Just a simple Django form.
    You will find the Django documentation on forms very confusing, This is simpler.
    '''
    year = settings.PHOTOS_YEAR
    filesaved = False
    actual_saved = []
 
    context = {'year': year, 'placeholder': "AnathemaDevice"}
    
    yearpath = Path(settings.PHOTOS_ROOT, year)
    
    if folder == str(year) or folder == str(year) +"/":
        folder = None
    
    if folder is None:
        folder = "" # improve this later
        dirpath = Path(settings.PHOTOS_ROOT, year)
        urlfile = f'/expofiles/photos/{year}'
        urldir  = f'/photoupload/{year}'
    else: # it will contain the year as well as the photographer
        dirpath = Path(settings.PHOTOS_ROOT, folder)
        if dirpath.is_dir():
            urlfile = f'/expofiles/photos/{folder}'
            urldir  = Path('/photoupload') / folder
        else:
            folder = "" # improve this later
            dirpath = Path(settings.PHOTOS_ROOT, year)
            urlfile = f'/expofiles/photos/{year}'
            urldir  = f'/photoupload/{year}'
      
        
    form = FilesForm()
    formd = TextForm()  
     
    if request.method == 'POST':
        if "photographer" in request.POST:              
            formd = TextForm(request.POST)
            if formd.is_valid():
                newphotographer = request.POST["photographer"]
                (yearpath / newphotographer).mkdir(exist_ok=True)
        else:
            form = FilesForm(request.POST,request.FILES)
            if form.is_valid():
                f = request.FILES["uploadfiles"]
                multiple = request.FILES.getlist('uploadfiles')
                # NO CHECK that the files being uploaded are image files
                fs = FileSystemStorage(dirpath)
               
                actual_saved = []
                if multiple:
                    for f in multiple:
                        try: # crashes in Django os.chmod call if on WSL, but does save file!
                            saved_filename = fs.save(f.name, content=f) 
                        except:
                            print(f'\n !! Permissions failure ?! on attempting to save file {f.name}')
                            if 'saved_filename' in locals():
                                if saved_filename.is_file():
                                    actual_saved.append(saved_filename)
                                    filesaved = True    
    files = []
    dirs = []
    try:
        for f in dirpath.iterdir():
            if f.is_dir():
                dirs.append(f.name)
            if f.is_file():
                files.append(f.name)           
    except FileNotFoundError:
        files.append('(no folder yet - would be created)')
    if len(files) >0 :
        files = sorted(files)
        
    if dirs:
        dirs = sorted(dirs)

    return render(request, 'photouploadform.html', 
        {'form': form, **context, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved})

@login_required_if_public
def dwgupload(request, folder=None, gitdisable='no'):
    '''Upload DRAWING files (tunnel or therion) into the upload folder in :drawings
    AND registers it into the :drawings: git repo.
    
    This does NOT use a Django model linked to a Django form. Just a simple Django form.
    You will find the Django documentation on forms very confusing, This is simpler.
    
    Need to validate it as being a valid GPX file using an XML parser, not a dubious script or hack

    
    We use get_or_create instead of simply creating a new object in case someone uploads the same file
    several times in one session, and expects them to be overwritten in the database. Although
    the actual file will be duplicated in the filesystem with different random name ending.
    '''
    def dwgvalid(name):
        if name in [ '.gitignore', ]:
            return False
        if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt']:
            return True # dangerous, we should check the actual file binary signature
        return False

    def dwgvaliddisp(name):
        '''OK to display, even if we are not going to allow a new one to be uploaded
        '''
        if name in [ '.gitignore', ]:
            return False
        if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt', '.jpg', '.jpeg', '.png', '.pdf', '.top', '.topo']:
            return True # dangerous, we should check the actual file binary signature
        return False

    filesaved = False
    actual_saved = []
    refused = []
    doesnotexist = ''
    # print(f'! - FORM dwgupload - start "{folder}" - gitdisable "{gitdisable}"')
    if folder is None:
        folder = "" # improve this later
        dirpath = Path(settings.DRAWINGS_DATA)
        urlfile = '/dwgdataraw'
        urldir  = '/dwgupload'
    else:
        dirpath = Path(settings.DRAWINGS_DATA, folder)
        urlfile = Path('/dwgdataraw/') / folder
        urldir  = Path('/dwgupload/') / folder
    
    form = FilesForm()
    
    if request.method == 'POST':
        form = FilesForm(request.POST,request.FILES)
        if form.is_valid():
            f = request.FILES["uploadfiles"]
            multiple = request.FILES.getlist('uploadfiles')
            fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder))
           
            actual_saved = []
            refused = []
            
            # GIT see also core/views/expo.py editexpopage()
            # GIT see also core/models/cave.py writetrogglefile()
            if gitdisable != 'yes': # set in url 'dwguploadnogit/'
                git = settings.GIT
            else:
                git = 'echo'

            if multiple:
                for f in multiple:
                    if dwgvalid(f.name):
                        try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file!
                            saved_filename = fs.save(f.name, content=f) 
                        except:
                            print(f'\n !! Permissions failure ?! on attempting to save file {f.name}')
                            if 'saved_filename' in locals():
                                if saved_filename.is_file():
                                    actual_saved.append(saved_filename)
                                    if gitdisable != 'yes':
                                        dr_add = subprocess.run([git, "add", saved_filename], cwd=dirpath, capture_output=True, text=True)
                                    if dr_add.returncode != 0:
                                        msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout  + '\n\nreturn code: ' + str(dr_add.returncode)
                                        message = f'CANNOT git on server for this file {saved_filename}. Edits saved but not added to git.\n\n' + msgdata
                                        return render(request,'errors/generic.html', {'message': message})
                                    dwgfile, created = DrawingFile.objects.get_or_create(dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size)  
                                    dwgfile.save()
                    else:
                        refused.append(f.name)
                        print(f'REFUSED {f.name}')
            if actual_saved: # maybe all were refused by the suffix test in dwgvalid()
                filesaved = True
                if len(actual_saved) > 1:
                    dots = "..."
                else:
                    dots = ""
                if gitdisable != 'yes':
                    dr_commit = subprocess.run([git, "commit", "-m", f'Drawings upload - {actual_saved[0]}{dots}'], cwd=dirpath, capture_output=True, text=True)
                    # This produces return code = 1 if it commits OK
                    if dr_commit.returncode != 0:
                        msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout  + '\n\nreturn code: ' + str(dr_commit.returncode)
                        message = f'Error code with git on server for this {actual_saved[0]}{dots}. Edits saved, added to git, but NOT committed.\n\n' + msgdata
                        return render(request,'errors/generic.html', {'message': message})

    files = []
    dirs = []
    # print(f'! - FORM dwgupload - start {folder} \n"{dirpath}" \n"{dirpath.parent}" \n"{dirpath.exists()}"')
    try:
        for f in dirpath.iterdir():
            if f.is_dir():
                if f.name not in ['.git' ]:
                    dirs.append(f.name)
                    continue
            if f.is_file():
                if dwgvaliddisp(f.name): 
                    files.append(f.name)
                    continue
    except FileNotFoundError:
        doesnotexist = True
    if files:
        files = sorted(files)
        
    if dirs:
        dirs = sorted(dirs)

    return render(request, 'dwguploadform.html', 
        {'form': form, 'doesnotexist': doesnotexist, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved, 'refused': refused})