re-fettled scan upload, creates Wallet object earlier

core/views/uploads.py

@@ -23,7 +23,7 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 #from troggle import settings
 from troggle.parsers.imports import import_caves, import_people, import_surveyscans
 from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
-from troggle.parsers.scans import wallet_blank_json, contentsjson, CopyWalletData
+from troggle.parsers.scans import wallet_blank_json, contentsjson
 # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
 from troggle.core.models.troggle import DataIssue
 from troggle.core.models.troggle import Expedition, Person, PersonExpedition
@@ -203,12 +203,12 @@ def scanupload(request, path=None):
     '''Upload scanned image files into a wallet on /expofiles
     Also display AND EDIT the contents.json data in the wallet.
     
-    The Wallet object and the contents.json file are only created when the user
+    The Wallet object and the contents.json file are created when the user
-    edits the JSON data, not when they upload files. This may be a bad idea
+    first uploads files. 
-    as the wallet will not appear in reports untilt his is done.
     
     This does NOT use a Django model linked to a Django form. Just a simple Django form.
-    You will find the Django documentation on forms very confusing, This is simpler.
+    You will find the Django documentation on forms very confusing,
+    as it covers many very differnet things we do not need. This is simpler.
     
     This subsumes much of the code which was in the old wallets.py script and so this function is very long
     indeed and needs refactoring.
@@ -219,6 +219,48 @@ def scanupload(request, path=None):
     filesaved = False
     actual_saved = []
     
+    def save_json(jsondict):
+        newfolder = contents_path.parent
+        print(f'---  wallet directory in :drawings: repo {newfolder=}')
+        if not os.path.exists(contents_path.parent):
+            print(f'--- No wallet directory in :drawings: repo, so creating it')
+            os.makedirs(contents_path.parent)
+        
+        with open(contents_path, "w") as jfile:
+            json.dump(jsondict, jfile, indent = 1)
+        print(f'--- FINISHED saving to JSON at {contents_path}')
+        
+    def make_wallet(walletname):
+        '''We need a wallet Object so that the django template stuff can find the files
+        '''
+        try:
+            w, created = Wallet.objects.get_or_create(walletname=walletname)
+            print(f'wallet string {walletname}, wallet object {w} created new?: {created}')   
+            if created:
+                w.fpath = Path(settings.SCANS_ROOT, walletname[0:4], walletname)
+                w.save()
+        except:
+             print(f'wallet string {walletname}, FAIL TO GET or create WALLET OBJECT')
+             raise
+
+    def commit_json():
+        destfolder = contents_path.parent
+        dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
+        if dr_add.returncode != 0:
+            msgdata = 'Ask a nerd to fix this.\n--' + dr_add.stderr + '\n--' + dr_add.stdout  + '\n--return code: ' + str(dr_add.returncode)
+            message = f'CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n' + msgdata
+            print(message)
+            return render(request,'errors/generic.html', {'message': message})
+        else:
+            dr_commit = subprocess.run([git, "commit", "-m", f'Update of {contentsjson} in wallet'], cwd=destfolder, capture_output=True, text=True)
+            # This produces return code = 1 if it commits OK
+            if dr_commit.returncode != 0:
+                msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout  + '\n\nreturn code: ' + str(dr_commit.returncode)
+                message = f'Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n' + msgdata
+                print(message)
+                return render(request,'errors/generic.html', {'message': message})
+
+    
     checkboxes = ["description written", "survex not required", "qms written", "website updated",
             "plan not required", "plan drawn", "elev not required", "elev drawn", "electronic survey" ]
     if path:
@@ -301,45 +343,9 @@ def scanupload(request, path=None):
                        wd["survex file"][i] = elem.strip()
                     #print(f'--- {wd["survex file"]} - {type(wd["survex file"])}')
 
-                newfolder = contents_path.parent
+                save_json(wd)
-                print(f'---  wallet directory in :drawings: repo {newfolder=}')
+                make_wallet(wallet)
-                if not os.path.exists(contents_path.parent):
+                commit_json()
-                    print(f'--- No wallet directory in :drawings: repo, so creating it')
-                    os.makedirs(contents_path.parent)
-                
-                with open(contents_path, "w") as jfile:
-                    json.dump(wd, jfile, indent = 1)
-                print(f'--- FINISHED saving to JSON at {contents_path}')
-                
-                # This commits it,
-                # needs the troggle object wallet, not a string
- 
-                try:
-                    w, created = Wallet.objects.get_or_create(walletname=wallet)
-                    print(f'wallet string {wallet}, wallet object {w} created new?: {created}')   
-                    if created:
-                        w.fpath = Path(settings.SCANS_ROOT, wallet[0:4], wallet)
-                        w.save()
-                except:
-                     print(f'wallet string {wallet}, FAIL TO GET or create WALLET OBJECT')
-                     raise
-                
-                # now we do the git add and commit
-                destfolder = contents_path.parent
-                dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
-                if dr_add.returncode != 0:
-                    msgdata = 'Ask a nerd to fix this.\n--' + dr_add.stderr + '\n--' + dr_add.stdout  + '\n--return code: ' + str(dr_add.returncode)
-                    message = f'CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n' + msgdata
-                    print(message)
-                    return render(request,'errors/generic.html', {'message': message})
-                else:
-                    dr_commit = subprocess.run([git, "commit", "-m", f'Update of {contentsjson} in wallet'], cwd=destfolder, capture_output=True, text=True)
-                    # This produces return code = 1 if it commits OK
-                    if dr_commit.returncode != 0:
-                        msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout  + '\n\nreturn code: ' + str(dr_commit.returncode)
-                        message = f'Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n' + msgdata
-                        print(message)
-                        return render(request,'errors/generic.html', {'message': message})
                 
             else:
                 print(f'--- INVALID JSON Update form submitted')
@@ -357,7 +363,7 @@ def scanupload(request, path=None):
             form = FilesForm(request.POST,request.FILES)
      
             if form.is_valid():
-                f = request.FILES["uploadfiles"]
+                #f = request.FILES["uploadfiles"]
                 multiple = request.FILES.getlist('uploadfiles')
                 fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary
                
@@ -373,20 +379,9 @@ def scanupload(request, path=None):
                                     actual_saved.append(saved_filename)
                                     # print(f'! - FORM scanupload multiple {actual_saved}')
                                     filesaved = True
-                
+                    save_json(wallet_blank_json)
-                # Don't do this when uploading a file. 
+                    make_wallet(wallet)
-                # The contents.json file will be created when the wallet data is edited for the first time.
+                    commit_json()
-                # if not contents_path.is_file(): # double-check: it doesn't exist yet so create it when uploading a file.
-                    # with open(contents_path, "w") as json_file:
-                        # json.dump(wallet_blank_json, json_file, sort_keys=True, indent = 1) 
-                        
-                # index_path = dirpath / indexhtml
-                # if not index_path.is_file(): # double-check it doesn't exist yet
-                    # thishtml = wallet_blank_html.replace("YEAR", str(year))
-                    # thishtml = thishtml.replace("WALLET", str,(wallet))
-                    # with open(index_path, "w") as html_file:
-                        # html_file.write(thishtml ) 
-      
     files = []
     dirs = []
     # print(f'! - FORM scanupload - start {wallet} {dirpath}')
@@ -399,7 +394,7 @@ def scanupload(request, path=None):
                 if f.is_file():
                     files.append(f.name)
         except FileNotFoundError:
-            files.append('(no wallet yet. It would be created if you upload a scan)')
+            files.append('(no wallet yet. It would be created if you upload a scan and then save the form with a date.)')
     else:
            create = True
          

parsers/scans.py

@@ -28,7 +28,7 @@ git = settings.GIT
 wallet_blank_json = {
  "cave": "", 
  "date": "", 
- "description url": "/caves", 
+ "description url": "1623/XXX", 
  "description written": False, 
  "electronic survey": False, 
  "elev drawn": False, 
@@ -44,22 +44,6 @@ wallet_blank_json = {
  "survex not required": False, 
  "website updated": False}
 
-# wallet_blank_html = '''<html><body><H1>Wallet WALLET</H1>
-# <p>List of trips: <a href="http://expo.survex.com/expedition/YEAR">expedition/YEAR</a> 
-# - troggle-processed .svx files and logbook entries on server</p>
-# <p>Date: </p><p>People:  Unknown,</p>
-# <p>Cave <a href='http://expo.survex.com/caves/'>Guidebook description</a> 
-# - A description is indicated as being needed, so may need adding into this cave page. 
-# <p>Survex file: not identified yet
-# <H2>Issues</H2>
-# <p>The description needs writing</p>
-# <p>The QMs needs writing</p><p>The website is marked as needing updating (using the guidebook description)</p>
-# <p>Tunnel / Therion drawing files need drawing</p>
-# <H2>Files</H2>
-# <UL>
-# </UL>
-# </body></html>
-# '''
                     
 def CheckEmptyDate(wallet):
     '''If date is not set, get it from a linked survex file. 
@@ -100,48 +84,6 @@ def LoadListScansFile(wallet):
             if c>=10:
                 print(".", end='')
                 c = 0
-def CopyWalletData(wallet):
-    '''Copies all the contents.json to a parallel set of folders in the drawings repo
-    refreshes everything during a full import, but it should all be up to date as every time
-    wallet data gets saved it should also be copied across and committed.
-    '''
-    # not needed now the drawinsg repo is the master
-    return
-    
-    year = wallet.walletname[0:4]
-    destfolder = Path(settings.DRAWINGS_DATA,'walletjson', year, wallet.walletname)
-    destjson = destfolder / contentsjson
-    sourcejson = Path(wallet.fpath, contentsjson)
-    if not os.path.exists(Path(destfolder)):
-        try:
-            os.makedirs(destfolder)
-            print(f' -  created folder {destfolder}..')
-        except PermissionError:
-            print(f"CANNOT  save this JSON file.\nPERMISSIONS incorrectly set on server for this folder {destfolder}. Ask a nerd to fix this.")
-    if os.path.isfile(sourcejson):
-        try:
-            if not os.path.isfile(destjson) or not filecmp.cmp(sourcejson, destjson):
-                shutil.copy(sourcejson, destjson)
-                print(f' -  Copied {sourcejson} to {destjson}')
-                dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
-                if dr_add.returncode != 0:
-                    msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout  + '\n\nreturn code: ' + str(dr_add.returncode)
-                    message = f'CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n' + msgdata
-                    print(message)
-                else:
-                    # ideally we would commit many chnages to many wallets just once. But most of the time only a couple of files will change.
-                    dr_commit = subprocess.run([git, "commit", "-m", f'Update of {contentsjson} in wallet'], cwd=destfolder, capture_output=True, text=True)
-                    # This produces return code = 1 if it commits OK
-                    if dr_commit.returncode != 0:
-                        msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout  + '\n\nreturn code: ' + str(dr_commit.returncode)
-                        message = f'Error code with git on server for this {contentsjson}. File is copied, added to git, but NOT committed.\n\n' + msgdata
-                        print(message)
-
-        except PermissionError:
-            print(f"CANNOT  copy this JSON file.\nPERMISSIONS incorrectly set on server for this file {destjson}. Ask a nerd to fix this.")
-
-
- 
         
 def load_all_scans():
     '''This iterates through the scans directories (either here or on the remote server)
@@ -186,7 +128,7 @@ def load_all_scans():
                     CheckEmptyPeople(wallet)
                     wallet.save()
                     LoadListScansFile(wallet)
-                    CopyWalletData(wallet)
+
  
         else:
             # but We should load all the scans, even for nonstandard names.