mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2024-11-25 08:41:51 +00:00
297 lines
12 KiB
Python
297 lines
12 KiB
Python
import logging
|
|
import random
|
|
import resource
|
|
import subprocess
|
|
import os
|
|
from decimal import getcontext
|
|
from pathlib import Path
|
|
|
|
getcontext().prec = 2 # use 2 significant figures for decimal calculations
|
|
|
|
|
|
import settings
|
|
|
|
"""This file declares TROG a globally visible object for caches.
|
|
|
|
TROG is a dictionary holding globally visible indexes and cache functions.
|
|
It is a Global Object, see https://python-patterns.guide/python/module-globals/
|
|
troggle.utils.TROG
|
|
|
|
chaosmonkey(n) - used by survex import to regenerate some .3d files
|
|
save_carefully() - core function that saves troggle objects in the database
|
|
|
|
various git add/commit functions that need refactoring together
|
|
|
|
NOTE that TROG is not serialized! Two users can update it and conflict !!
|
|
This needs to be in a multi-user database with transactions. However it is
|
|
useful when doing a data import with databaseReset.py as that has a single
|
|
thread.
|
|
|
|
"""
|
|
|
|
TROG = {"pagecache": {"expedition": {}}, "caves": {"gcavelookup": {}, "gcavecount": {}}}
|
|
|
|
# This is module-level executable. This is a Bad Thing. Especially when it touches the file system.
|
|
try:
|
|
logging.basicConfig(level=logging.DEBUG, filename=settings.LOGFILE, filemode="w")
|
|
except:
|
|
# Opening of file for writing is going to fail currently, so decide it doesn't matter for now
|
|
pass
|
|
|
|
|
|
def get_process_memory():
|
|
usage = resource.getrusage(resource.RUSAGE_SELF)
|
|
return usage[2] / 1024.0
|
|
|
|
|
|
def chaosmonkey(n):
|
|
"""returns True once every n calls - randomly"""
|
|
if random.randrange(0, n) != 0:
|
|
return False
|
|
# print("CHAOS strikes !", file=sys.stderr)
|
|
return True
|
|
|
|
|
|
def only_commit(fname, message):
|
|
"""Only used to commit a survex file edited and saved in view/survex.py"""
|
|
git = settings.GIT
|
|
cwd = fname.parent
|
|
filename = fname.name
|
|
# print(f'{fname=} ')
|
|
|
|
try:
|
|
cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True)
|
|
if cp_add.returncode != 0:
|
|
msgdata = f"Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}"
|
|
raise WriteAndCommitError(
|
|
f"CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n" + msgdata
|
|
)
|
|
|
|
cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True)
|
|
# This produces return code = 1 if it commits OK, but when the local repo still needs to be pushed to origin/loser
|
|
# which will be the case when running a test troggle system on a development machine
|
|
devok_text = """On branch master
|
|
Your branch is ahead of 'origin/master' by 1 commit.
|
|
(use "git push" to publish your local commits)
|
|
|
|
nothing to commit, working tree clean
|
|
"""
|
|
if cp_commit.returncode == 1 and cp_commit.stdout == devok_text:
|
|
pass
|
|
else:
|
|
if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith(
|
|
"nothing to commit, working tree clean"
|
|
):
|
|
msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}'
|
|
print(msgdata)
|
|
raise WriteAndCommitError(
|
|
f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n"
|
|
+ msgdata
|
|
)
|
|
|
|
except subprocess.SubprocessError:
|
|
raise WriteAndCommitError(
|
|
f"CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this."
|
|
)
|
|
|
|
|
|
def write_and_commit(files, message):
|
|
"""Writes the content to the filepath and adds and commits the file to git. If this fails, a WriteAndCommitError is raised.
|
|
This does not create any needed intermediate folders, which is what we do when writing survex files, so functionality here
|
|
is duplicated in only_commit()
|
|
|
|
These need refactoring
|
|
"""
|
|
print(files)
|
|
git = settings.GIT
|
|
commands = []
|
|
try:
|
|
for filepath, content, encoding in files:
|
|
cwd = filepath.parent
|
|
filename = filepath.name
|
|
# GIT see also core/views/uploads.py dwgupload()
|
|
# GIT see also core/views/expo.py editexpopage()
|
|
os.makedirs(os.path.dirname(filepath), exist_ok = True)
|
|
if encoding:
|
|
mode = "w"
|
|
kwargs = {"encoding": encoding}
|
|
else:
|
|
mode = "wb"
|
|
kwargs = {}
|
|
try:
|
|
with open(filepath, mode, **kwargs) as f:
|
|
print(f"WRITING {cwd}---{filename} ")
|
|
# as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
|
|
# os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
|
|
f.write(content)
|
|
except PermissionError:
|
|
raise WriteAndCommitError(
|
|
f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this."
|
|
)
|
|
cmd_diff = [git, "diff", filename]
|
|
cp_diff = subprocess.run(cmd_diff, cwd=cwd, capture_output=True, text=True)
|
|
commands.append(cmd_diff)
|
|
if cp_diff.returncode == 0:
|
|
cmd_add = [git, "add", filename]
|
|
cp_add = subprocess.run(cmd_add, cwd=cwd, capture_output=True, text=True)
|
|
commands.append(cmd_add)
|
|
if cp_add.returncode != 0:
|
|
msgdata = (
|
|
"Ask a nerd to fix this.\n\n"
|
|
+ cp_add.stderr
|
|
+ "\n\n"
|
|
+ cp_add.stdout
|
|
+ "\n\nreturn code: "
|
|
+ str(cp_add.returncode)
|
|
)
|
|
raise WriteAndCommitError(
|
|
f"CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n"
|
|
+ msgdata
|
|
)
|
|
else:
|
|
print(f"No change {filepath}")
|
|
filepaths = [filepath for filepath, content, encoding in files]
|
|
cmd_commit = [git, "commit"] + filepaths + ["-m", message]
|
|
cm_status = subprocess.run(cmd_commit, cwd=cwd, capture_output=True, text=True)
|
|
commands.append(cmd_commit)
|
|
if cm_status == 0:
|
|
msgdata = (
|
|
"Commands: " + str(commands) +
|
|
"Ask a nerd to fix this.\n\n"
|
|
+ "Stderr: " + cp_status.stderr
|
|
+ "\n\n"
|
|
+ "Stdout: " + cp_status.stdout
|
|
+ "\n\nreturn code: " + str(cp_status.returncode)
|
|
)
|
|
raise WriteAndCommitError(
|
|
f"Error committing. Edits saved, added to git, but NOT committed.\n\n"
|
|
+ msgdata
|
|
)
|
|
cmd_status = [git, "status"] + filepaths
|
|
cp_status = subprocess.run(cmd_status, cwd=cwd, capture_output=True, text=True)
|
|
commands.append(cp_status)
|
|
#This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb
|
|
if (not cp_status.stdout) or len(cp_status.stdout) < 2 or cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean":
|
|
msgdata = (
|
|
str(commands) +
|
|
"Ask a nerd to fix this.\n\n"
|
|
+ "Stderr: " + cp_status.stderr
|
|
+ "\n\n"
|
|
+ "Stdout: " + cp_status.stdout
|
|
+ "\n\nreturn code: " + str(cp_status.returncode)
|
|
)
|
|
raise WriteAndCommitError(
|
|
f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n"
|
|
+ msgdata
|
|
)
|
|
except subprocess.SubprocessError:
|
|
raise WriteAndCommitError(
|
|
f"CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this."
|
|
)
|
|
|
|
|
|
class WriteAndCommitError(Exception):
|
|
"""Exception class for errors writing files and comitting them to git"""
|
|
|
|
def __init__(self, message):
|
|
self.message = message
|
|
|
|
def __str__(self):
|
|
return f"WriteAndCommitError: {self.message}"
|
|
|
|
|
|
def writetrogglefile(filepath, filecontent):
|
|
"""Commit the new saved file to git
|
|
Callers to cave.writeDataFile() or entrance.writeDataFile() should handle the exception PermissionsError explicitly
|
|
"""
|
|
# GIT see also core/views/expo.py editexpopage()
|
|
# GIT see also core/views/uploads.py dwgupload()
|
|
# Called from core/models/caves.py Cave.writeDataFile() Entrance.writeDataFile()
|
|
filepath = Path(filepath)
|
|
cwd = filepath.parent
|
|
filename = filepath.name
|
|
git = settings.GIT
|
|
|
|
# as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
|
|
# do not trap exceptions, pass them up to the view that called this function
|
|
print(f"WRITING{cwd}---{filename} ")
|
|
with open(filepath, "w") as f:
|
|
f.write(filecontent)
|
|
# os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
|
|
sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True)
|
|
if sp.returncode != 0:
|
|
out = sp.stdout
|
|
if len(out) > 160:
|
|
out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
|
|
print(f"git ADD {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode))
|
|
|
|
sp = subprocess.run(
|
|
[git, "commit", "-m", f"Troggle online: cave or entrance edit -{filename}"],
|
|
cwd=cwd,
|
|
capture_output=True,
|
|
check=True,
|
|
text=True,
|
|
)
|
|
if sp.returncode != 0:
|
|
out = sp.stdout
|
|
if len(out) > 160:
|
|
out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
|
|
print(f"git COMMIT {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode))
|
|
# not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that.
|
|
|
|
|
|
def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
|
|
"""Looks up instance using lookupAttribs and carries out the following:
|
|
-if instance does not exist in DB: add instance to DB, return (new instance, True)
|
|
-if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False)
|
|
-if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False)
|
|
|
|
The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field
|
|
defined in core.models.TroggleModel.
|
|
|
|
We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false.
|
|
|
|
NOTE: this takes twice as long as simply creating a new object with the given values.
|
|
|
|
As of Jan.2023 this function is not used anywhere in troggle.
|
|
|
|
"""
|
|
try:
|
|
instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
|
|
except:
|
|
print(" !! - FAIL in SAVE CAREFULLY ===================", objectType)
|
|
print(" !! - -- objects.get_or_create()")
|
|
print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
|
|
raise
|
|
if not created and not instance.new_since_parsing:
|
|
for k, v in list(
|
|
nonLookupAttribs.items()
|
|
): # overwrite the existing attributes from the logbook text (except date and title)
|
|
setattr(instance, k, v)
|
|
try:
|
|
instance.save()
|
|
except:
|
|
print(" !! - SAVE CAREFULLY ===================", objectType)
|
|
print(" !! - -- instance.save()")
|
|
print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
|
|
raise
|
|
try:
|
|
str(instance)
|
|
except:
|
|
pass
|
|
if created:
|
|
logging.info(str(instance) + " was just added to the database for the first time. \n")
|
|
|
|
if not created and instance.new_since_parsing:
|
|
logging.info(
|
|
str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n"
|
|
)
|
|
|
|
if not created and not instance.new_since_parsing:
|
|
logging.info(
|
|
" instance:<"
|
|
+ str(instance)
|
|
+ "> existed in the database unchanged since last parse. It have been overwritten."
|
|
)
|
|
return (instance, created)
|