expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-12-01 06:11:51 +00:00
Code Issues 19 Projects 3 Activity
29879fdde2
troggle/core/utils.py

333 lines
14 KiB
Python
Raw Blame History

import hashlib
import logging
import os
import random
import resource
import string
import subprocess
from decimal import getcontext
from pathlib import Path
from troggle.core.models.troggle import Expedition
getcontext().prec = 2 # use 2 significant figures for decimal calculations
import settings
"""This file declares TROG a globally visible object for caches.
TROG is a dictionary holding globally visible indexes and cache functions.
It is a Global Object, see https://python-patterns.guide/python/module-globals/
troggle.utils.TROG
chaosmonkey(n) - used by survex import to regenerate some .3d files
save_carefully() - core function that saves troggle objects in the database
various git add/commit functions that need refactoring together
NOTE that TROG is not serialized! Two users can update it and conflict !!
This needs to be in a multi-user database with transactions. However it is
useful when doing a data import with databaseReset.py as that has a single
thread.
"""
TROG = {"pagecache": {"expedition": {}}, "caves": {"gcavelookup": {}, "gcavecount": {}}}
alphabet = []
sha = hashlib.new('sha256')
# This is module-level executable. This is a Bad Thing. Especially when it touches the file system.
try:
logging.basicConfig(level=logging.DEBUG, filename=settings.LOGFILE, filemode="w")
except:
# Opening of file for writing is going to fail currently, so decide it doesn't matter for now
pass
def sanitize_name(name):
"""Filenames sould not caontain these characters as then the system barf when it tries to use them in URLs
"""
return name.replace("#","-").replace("?","=").replace("&","+").replace(":","^")
def get_process_memory():
usage = resource.getrusage(resource.RUSAGE_SELF)
return usage[2] / 1024.0
def chaosmonkey(n):
"""returns True once every n calls - randomly"""
if random.randrange(0, n) != 0:
return False
# print("CHAOS strikes !", file=sys.stderr)
return True
def unique_slug(text, n):
"""This gives an almost-unique id based on the text,
2 hex digits would seem adequate, but we might get a collision.
Not used anywhere.
"""
sha.update(text.encode('utf-8'))
return sha.hexdigest()[0:n]
def alphabet_suffix(n):
"""This is called repeatedly during initial parsing import, hence the cached list
"""
global alphabet
if not alphabet:
alphabet = list(string.ascii_lowercase)
if n < len(alphabet) and n > 0:
suffix = alphabet[n-1]
else:
suffix = "_X_" + random.choice(string.ascii_lowercase) + random.choice(string.ascii_lowercase)
return suffix
def current_expo():
expos = Expedition.objects.all().order_by('-year')
if expos:
return expos[0].year
else:
return settings.EPOCH.year
def only_commit(fname, message):
"""Only used to commit a survex file edited and saved in view/survex.py"""
git = settings.GIT
cwd = fname.parent
filename = fname.name
# print(f'{fname=} ')
try:
cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True)
if cp_add.returncode != 0:
msgdata = f"Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}"
raise WriteAndCommitError(
f"CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n" + msgdata
)
cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True)
# This produces return code = 1 if it commits OK, but when the local repo still needs to be pushed to origin/loser
# which will be the case when running a test troggle system on a development machine
devok_text = """On branch master
Your branch is ahead of 'origin/master' by 1 commit.
(use "git push" to publish your local commits)
nothing to commit, working tree clean
"""
if cp_commit.returncode == 1 and cp_commit.stdout == devok_text:
pass
else:
if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith(
"nothing to commit, working tree clean"
):
msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}'
print(msgdata)
raise WriteAndCommitError(
f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n"
+ msgdata
)
except subprocess.SubprocessError:
raise WriteAndCommitError(
f"CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this."
)
def write_and_commit(files, message):
"""Writes the content to the filepath and adds and commits the file to git. If this fails, a WriteAndCommitError is raised.
This does not create any needed intermediate folders, which is what we do when writing survex files, so functionality here
is duplicated in only_commit()
These need refactoring
"""
git = settings.GIT
commands = []
try:
for filepath, content, encoding in files:
cwd = filepath.parent
filename = filepath.name
# GIT see also core/views/uploads.py dwgupload()
# GIT see also core/views/expo.py editexpopage()
os.makedirs(os.path.dirname(filepath), exist_ok = True)
if encoding:
mode = "w"
kwargs = {"encoding": encoding}
else:
mode = "wb"
kwargs = {}
try:
with open(filepath, mode, **kwargs) as f:
print(f"WRITING {cwd}---{filename} ")
# as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
# os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
f.write(content)
except PermissionError:
raise WriteAndCommitError(
f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this."
)
cmd_diff = [git, "diff", filename]
cp_diff = subprocess.run(cmd_diff, cwd=cwd, capture_output=True, text=True)
commands.append(cmd_diff)
if cp_diff.returncode == 0:
cmd_add = [git, "add", filename]
cp_add = subprocess.run(cmd_add, cwd=cwd, capture_output=True, text=True)
commands.append(cmd_add)
if cp_add.returncode != 0:
msgdata = (
"Ask a nerd to fix this.\n\n"
+ cp_add.stderr
+ "\n\n"
+ cp_add.stdout
+ "\n\nreturn code: "
+ str(cp_add.returncode)
)
raise WriteAndCommitError(
f"CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n"
+ msgdata
)
else:
print(f"No change {filepath}")
filepaths = [filepath for filepath, content, encoding in files]
cmd_commit = [git, "commit"] + filepaths + ["-m", message]
cm_status = subprocess.run(cmd_commit, cwd=cwd, capture_output=True, text=True)
commands.append(cmd_commit)
if cm_status == 0:
msgdata = (
"Commands: " + str(commands) +
"Ask a nerd to fix this.\n\n"
+ "Stderr: " + cp_status.stderr
+ "\n\n"
+ "Stdout: " + cp_status.stdout
+ "\n\nreturn code: " + str(cp_status.returncode)
)
raise WriteAndCommitError(
f"Error committing. Edits saved, added to git, but NOT committed.\n\n"
+ msgdata
)
cmd_status = [git, "status"] + filepaths
cp_status = subprocess.run(cmd_status, cwd=cwd, capture_output=True, text=True)
commands.append(cp_status)
#This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb
if (not cp_status.stdout) or len(cp_status.stdout) < 2 or cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean":
msgdata = (
str(commands) +
"Ask a nerd to fix this.\n\n"
+ "Stderr: " + cp_status.stderr
+ "\n\n"
+ "Stdout: " + cp_status.stdout
+ "\n\nreturn code: " + str(cp_status.returncode)
)
raise WriteAndCommitError(
f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n"
+ msgdata
)
except subprocess.SubprocessError:
raise WriteAndCommitError(
f"CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this."
)
class WriteAndCommitError(Exception):
"""Exception class for errors writing files and comitting them to git"""
def __init__(self, message):
self.message = message
def __str__(self):
return f"WriteAndCommitError: {self.message}"
def writetrogglefile(filepath, filecontent):
"""Commit the new saved file to git
Callers to cave.writeDataFile() or entrance.writeDataFile() should handle the exception PermissionsError explicitly
"""
# GIT see also core/views/expo.py editexpopage()
# GIT see also core/views/uploads.py dwgupload()
# Called from core/models/caves.py Cave.writeDataFile() Entrance.writeDataFile()
filepath = Path(filepath)
cwd = filepath.parent
filename = filepath.name
git = settings.GIT
# as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
# do not trap exceptions, pass them up to the view that called this function
print(f"WRITING{cwd}---{filename} ")
with open(filepath, "w") as f:
f.write(filecontent)
# os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True)
if sp.returncode != 0:
out = sp.stdout
if len(out) > 160:
out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
print(f"git ADD {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode))
sp = subprocess.run(
[git, "commit", "-m", f"Troggle online: cave or entrance edit -{filename}"],
cwd=cwd,
capture_output=True,
check=True,
text=True,
)
if sp.returncode != 0:
out = sp.stdout
if len(out) > 160:
out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
print(f"git COMMIT {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode))
# not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that.
def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
"""Looks up instance using lookupAttribs and carries out the following:
-if instance does not exist in DB: add instance to DB, return (new instance, True)
-if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False)
-if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False)
The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field
defined in core.models.TroggleModel.
We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false.
NOTE: this takes twice as long as simply creating a new object with the given values.
As of Jan.2023 this function is not used anywhere in troggle.
"""
try:
instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
except:
print(" !! - FAIL in SAVE CAREFULLY ===================", objectType)
print(" !! - -- objects.get_or_create()")
print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
raise
if not created and not instance.new_since_parsing:
for k, v in list(
nonLookupAttribs.items()
): # overwrite the existing attributes from the logbook text (except date and title)
setattr(instance, k, v)
try:
instance.save()
except:
print(" !! - SAVE CAREFULLY ===================", objectType)
print(" !! - -- instance.save()")
print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
raise
try:
str(instance)
except:
pass
if created:
logging.info(str(instance) + " was just added to the database for the first time. \n")
if not created and instance.new_since_parsing:
logging.info(
str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n"
)
if not created and not instance.new_since_parsing:
logging.info(
" instance:<"
+ str(instance)
+ "> existed in the database unchanged since last parse. It have been overwritten."
)
return (instance, created)
Reference in New Issue View Git Blame Copy Permalink