forked from expo/troggle
re-fettled scan upload, creates Wallet object earlier
This commit is contained in:
Philip Sargent
parent
284e044a03
commit
3c13f62bd1
@ -23,7 +23,7 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
|
||||
#from troggle import settings
|
||||
from troggle.parsers.imports import import_caves, import_people, import_surveyscans
|
||||
from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
|
||||
from troggle.parsers.scans import wallet_blank_json, contentsjson, CopyWalletData
|
||||
from troggle.parsers.scans import wallet_blank_json, contentsjson
|
||||
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
|
||||
from troggle.core.models.troggle import DataIssue
|
||||
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
|
||||
@ -203,12 +203,12 @@ def scanupload(request, path=None):
|
||||
'''Upload scanned image files into a wallet on /expofiles
|
||||
Also display AND EDIT the contents.json data in the wallet.
|
||||
|
||||
The Wallet object and the contents.json file are only created when the user
|
||||
edits the JSON data, not when they upload files. This may be a bad idea
|
||||
as the wallet will not appear in reports untilt his is done.
|
||||
The Wallet object and the contents.json file are created when the user
|
||||
first uploads files.
|
||||
|
||||
This does NOT use a Django model linked to a Django form. Just a simple Django form.
|
||||
You will find the Django documentation on forms very confusing, This is simpler.
|
||||
You will find the Django documentation on forms very confusing,
|
||||
as it covers many very differnet things we do not need. This is simpler.
|
||||
|
||||
This subsumes much of the code which was in the old wallets.py script and so this function is very long
|
||||
indeed and needs refactoring.
|
||||
@ -219,6 +219,48 @@ def scanupload(request, path=None):
|
||||
filesaved = False
|
||||
actual_saved = []
|
||||
|
||||
def save_json(jsondict):
|
||||
newfolder = contents_path.parent
|
||||
print(f'--- wallet directory in :drawings: repo {newfolder=}')
|
||||
if not os.path.exists(contents_path.parent):
|
||||
print(f'--- No wallet directory in :drawings: repo, so creating it')
|
||||
os.makedirs(contents_path.parent)
|
||||
|
||||
with open(contents_path, "w") as jfile:
|
||||
json.dump(jsondict, jfile, indent = 1)
|
||||
print(f'--- FINISHED saving to JSON at {contents_path}')
|
||||
|
||||
def make_wallet(walletname):
|
||||
'''We need a wallet Object so that the django template stuff can find the files
|
||||
'''
|
||||
try:
|
||||
w, created = Wallet.objects.get_or_create(walletname=walletname)
|
||||
print(f'wallet string {walletname}, wallet object {w} created new?: {created}')
|
||||
if created:
|
||||
w.fpath = Path(settings.SCANS_ROOT, walletname[0:4], walletname)
|
||||
w.save()
|
||||
except:
|
||||
print(f'wallet string {walletname}, FAIL TO GET or create WALLET OBJECT')
|
||||
raise
|
||||
|
||||
def commit_json():
|
||||
destfolder = contents_path.parent
|
||||
dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
|
||||
if dr_add.returncode != 0:
|
||||
msgdata = 'Ask a nerd to fix this.\n--' + dr_add.stderr + '\n--' + dr_add.stdout + '\n--return code: ' + str(dr_add.returncode)
|
||||
message = f'CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n' + msgdata
|
||||
print(message)
|
||||
return render(request,'errors/generic.html', {'message': message})
|
||||
else:
|
||||
dr_commit = subprocess.run([git, "commit", "-m", f'Update of {contentsjson} in wallet'], cwd=destfolder, capture_output=True, text=True)
|
||||
# This produces return code = 1 if it commits OK
|
||||
if dr_commit.returncode != 0:
|
||||
msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout + '\n\nreturn code: ' + str(dr_commit.returncode)
|
||||
message = f'Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n' + msgdata
|
||||
print(message)
|
||||
return render(request,'errors/generic.html', {'message': message})
|
||||
|
||||
|
||||
checkboxes = ["description written", "survex not required", "qms written", "website updated",
|
||||
"plan not required", "plan drawn", "elev not required", "elev drawn", "electronic survey" ]
|
||||
if path:
|
||||
@ -301,45 +343,9 @@ def scanupload(request, path=None):
|
||||
wd["survex file"][i] = elem.strip()
|
||||
#print(f'--- {wd["survex file"]} - {type(wd["survex file"])}')
|
||||
|
||||
newfolder = contents_path.parent
|
||||
print(f'--- wallet directory in :drawings: repo {newfolder=}')
|
||||
if not os.path.exists(contents_path.parent):
|
||||
print(f'--- No wallet directory in :drawings: repo, so creating it')
|
||||
os.makedirs(contents_path.parent)
|
||||
|
||||
with open(contents_path, "w") as jfile:
|
||||
json.dump(wd, jfile, indent = 1)
|
||||
print(f'--- FINISHED saving to JSON at {contents_path}')
|
||||
|
||||
# This commits it,
|
||||
# needs the troggle object wallet, not a string
|
||||
|
||||
try:
|
||||
w, created = Wallet.objects.get_or_create(walletname=wallet)
|
||||
print(f'wallet string {wallet}, wallet object {w} created new?: {created}')
|
||||
if created:
|
||||
w.fpath = Path(settings.SCANS_ROOT, wallet[0:4], wallet)
|
||||
w.save()
|
||||
except:
|
||||
print(f'wallet string {wallet}, FAIL TO GET or create WALLET OBJECT')
|
||||
raise
|
||||
|
||||
# now we do the git add and commit
|
||||
destfolder = contents_path.parent
|
||||
dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
|
||||
if dr_add.returncode != 0:
|
||||
msgdata = 'Ask a nerd to fix this.\n--' + dr_add.stderr + '\n--' + dr_add.stdout + '\n--return code: ' + str(dr_add.returncode)
|
||||
message = f'CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n' + msgdata
|
||||
print(message)
|
||||
return render(request,'errors/generic.html', {'message': message})
|
||||
else:
|
||||
dr_commit = subprocess.run([git, "commit", "-m", f'Update of {contentsjson} in wallet'], cwd=destfolder, capture_output=True, text=True)
|
||||
# This produces return code = 1 if it commits OK
|
||||
if dr_commit.returncode != 0:
|
||||
msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout + '\n\nreturn code: ' + str(dr_commit.returncode)
|
||||
message = f'Error code with git on server for this {contentsjson}. File is added to git, but NOT committed.\n' + msgdata
|
||||
print(message)
|
||||
return render(request,'errors/generic.html', {'message': message})
|
||||
save_json(wd)
|
||||
make_wallet(wallet)
|
||||
commit_json()
|
||||
|
||||
else:
|
||||
print(f'--- INVALID JSON Update form submitted')
|
||||
@ -357,7 +363,7 @@ def scanupload(request, path=None):
|
||||
form = FilesForm(request.POST,request.FILES)
|
||||
|
||||
if form.is_valid():
|
||||
f = request.FILES["uploadfiles"]
|
||||
#f = request.FILES["uploadfiles"]
|
||||
multiple = request.FILES.getlist('uploadfiles')
|
||||
fs = FileSystemStorage(os.path.join(dirpath)) # creates wallet folder if necessary
|
||||
|
||||
@ -373,20 +379,9 @@ def scanupload(request, path=None):
|
||||
actual_saved.append(saved_filename)
|
||||
# print(f'! - FORM scanupload multiple {actual_saved}')
|
||||
filesaved = True
|
||||
|
||||
# Don't do this when uploading a file.
|
||||
# The contents.json file will be created when the wallet data is edited for the first time.
|
||||
# if not contents_path.is_file(): # double-check: it doesn't exist yet so create it when uploading a file.
|
||||
# with open(contents_path, "w") as json_file:
|
||||
# json.dump(wallet_blank_json, json_file, sort_keys=True, indent = 1)
|
||||
|
||||
# index_path = dirpath / indexhtml
|
||||
# if not index_path.is_file(): # double-check it doesn't exist yet
|
||||
# thishtml = wallet_blank_html.replace("YEAR", str(year))
|
||||
# thishtml = thishtml.replace("WALLET", str,(wallet))
|
||||
# with open(index_path, "w") as html_file:
|
||||
# html_file.write(thishtml )
|
||||
|
||||
save_json(wallet_blank_json)
|
||||
make_wallet(wallet)
|
||||
commit_json()
|
||||
files = []
|
||||
dirs = []
|
||||
# print(f'! - FORM scanupload - start {wallet} {dirpath}')
|
||||
@ -399,7 +394,7 @@ def scanupload(request, path=None):
|
||||
if f.is_file():
|
||||
files.append(f.name)
|
||||
except FileNotFoundError:
|
||||
files.append('(no wallet yet. It would be created if you upload a scan)')
|
||||
files.append('(no wallet yet. It would be created if you upload a scan and then save the form with a date.)')
|
||||
else:
|
||||
create = True
|
||||
|
||||
|
||||
@ -28,7 +28,7 @@ git = settings.GIT
|
||||
wallet_blank_json = {
|
||||
"cave": "",
|
||||
"date": "",
|
||||
"description url": "/caves",
|
||||
"description url": "1623/XXX",
|
||||
"description written": False,
|
||||
"electronic survey": False,
|
||||
"elev drawn": False,
|
||||
@ -43,23 +43,7 @@ wallet_blank_json = {
|
||||
"survex file": [],
|
||||
"survex not required": False,
|
||||
"website updated": False}
|
||||
|
||||
# wallet_blank_html = '''<html><body><H1>Wallet WALLET</H1>
|
||||
# <p>List of trips: <a href="http://expo.survex.com/expedition/YEAR">expedition/YEAR</a>
|
||||
# - troggle-processed .svx files and logbook entries on server</p>
|
||||
# <p>Date: </p><p>People: Unknown,</p>
|
||||
# <p>Cave <a href='http://expo.survex.com/caves/'>Guidebook description</a>
|
||||
# - A description is indicated as being needed, so may need adding into this cave page.
|
||||
# <p>Survex file: not identified yet
|
||||
# <H2>Issues</H2>
|
||||
# <p>The description needs writing</p>
|
||||
# <p>The QMs needs writing</p><p>The website is marked as needing updating (using the guidebook description)</p>
|
||||
# <p>Tunnel / Therion drawing files need drawing</p>
|
||||
# <H2>Files</H2>
|
||||
# <UL>
|
||||
# </UL>
|
||||
# </body></html>
|
||||
# '''
|
||||
|
||||
|
||||
def CheckEmptyDate(wallet):
|
||||
'''If date is not set, get it from a linked survex file.
|
||||
@ -100,48 +84,6 @@ def LoadListScansFile(wallet):
|
||||
if c>=10:
|
||||
print(".", end='')
|
||||
c = 0
|
||||
def CopyWalletData(wallet):
|
||||
'''Copies all the contents.json to a parallel set of folders in the drawings repo
|
||||
refreshes everything during a full import, but it should all be up to date as every time
|
||||
wallet data gets saved it should also be copied across and committed.
|
||||
'''
|
||||
# not needed now the drawinsg repo is the master
|
||||
return
|
||||
|
||||
year = wallet.walletname[0:4]
|
||||
destfolder = Path(settings.DRAWINGS_DATA,'walletjson', year, wallet.walletname)
|
||||
destjson = destfolder / contentsjson
|
||||
sourcejson = Path(wallet.fpath, contentsjson)
|
||||
if not os.path.exists(Path(destfolder)):
|
||||
try:
|
||||
os.makedirs(destfolder)
|
||||
print(f' - created folder {destfolder}..')
|
||||
except PermissionError:
|
||||
print(f"CANNOT save this JSON file.\nPERMISSIONS incorrectly set on server for this folder {destfolder}. Ask a nerd to fix this.")
|
||||
if os.path.isfile(sourcejson):
|
||||
try:
|
||||
if not os.path.isfile(destjson) or not filecmp.cmp(sourcejson, destjson):
|
||||
shutil.copy(sourcejson, destjson)
|
||||
print(f' - Copied {sourcejson} to {destjson}')
|
||||
dr_add = subprocess.run([git, "add", contentsjson], cwd=destfolder, capture_output=True, text=True)
|
||||
if dr_add.returncode != 0:
|
||||
msgdata = 'Ask a nerd to fix this.\n\n' + dr_add.stderr + '\n\n' + dr_add.stdout + '\n\nreturn code: ' + str(dr_add.returncode)
|
||||
message = f'CANNOT git on server for this file {contentsjson}. Edits saved but not added to git.\n\n' + msgdata
|
||||
print(message)
|
||||
else:
|
||||
# ideally we would commit many chnages to many wallets just once. But most of the time only a couple of files will change.
|
||||
dr_commit = subprocess.run([git, "commit", "-m", f'Update of {contentsjson} in wallet'], cwd=destfolder, capture_output=True, text=True)
|
||||
# This produces return code = 1 if it commits OK
|
||||
if dr_commit.returncode != 0:
|
||||
msgdata = 'Ask a nerd to fix this.\n\n' + dr_commit.stderr + '\n\n' + dr_commit.stdout + '\n\nreturn code: ' + str(dr_commit.returncode)
|
||||
message = f'Error code with git on server for this {contentsjson}. File is copied, added to git, but NOT committed.\n\n' + msgdata
|
||||
print(message)
|
||||
|
||||
except PermissionError:
|
||||
print(f"CANNOT copy this JSON file.\nPERMISSIONS incorrectly set on server for this file {destjson}. Ask a nerd to fix this.")
|
||||
|
||||
|
||||
|
||||
|
||||
def load_all_scans():
|
||||
'''This iterates through the scans directories (either here or on the remote server)
|
||||
@ -186,7 +128,7 @@ def load_all_scans():
|
||||
CheckEmptyPeople(wallet)
|
||||
wallet.save()
|
||||
LoadListScansFile(wallet)
|
||||
CopyWalletData(wallet)
|
||||
|
||||
|
||||
else:
|
||||
# but We should load all the scans, even for nonstandard names.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user