xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

ran 'black' to reformat all the core files

Browse Source
This commit is contained in:
Philip Sargent
2023-01-30 19:04:36 +00:00
parent d06dd3d166
commit 7808005498
28 changed files with 3844 additions and 3075 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
core/utils.py
View File

@@ -10,7 +10,7 @@ from decimal import Decimal, getcontext
from pathlib import Path
from urllib.parse import urljoin
getcontext().prec=2 #use 2 significant figures for decimal calculations
getcontext().prec = 2 # use 2 significant figures for decimal calculations
from django.conf import settings
from django.contrib import admin
@@ -22,7 +22,7 @@ from django.urls import reverse
import settings
'''This file declares TROG a globally visible object for caches.
"""This file declares TROG a globally visible object for caches.
TROG is a dictionary holding globally visible indexes and cache functions.
It is a Global Object, see https://python-patterns.guide/python/module-globals/
@@ -38,78 +38,79 @@ This needs to be in a multi-user database with transactions. However it is
useful when doing a data import with databaseReset.py as that has a single
thread.
'''
"""
TROG = {
'pagecache' : {
'expedition' : {}
},
'caves' : {
'gcavelookup' : {},
'gcavecount' : {}
}
}
TROG = {"pagecache": {"expedition": {}}, "caves": {"gcavelookup": {}, "gcavecount": {}}}
# This is module-level executable. This is a Bad Thing. Especially when it touches the file system.
try:
logging.basicConfig(level=logging.DEBUG,
filename=settings.LOGFILE,
filemode='w')
logging.basicConfig(level=logging.DEBUG, filename=settings.LOGFILE, filemode="w")
except:
# Opening of file for writing is going to fail currently, so decide it doesn't matter for now
# Opening of file for writing is going to fail currently, so decide it doesn't matter for now
pass
def get_process_memory():
usage=resource.getrusage(resource.RUSAGE_SELF)
return usage[2]/1024.0
usage = resource.getrusage(resource.RUSAGE_SELF)
return usage[2] / 1024.0
def chaosmonkey(n):
'''returns True once every n calls - randomly'''
if random.randrange(0,n) != 0:
"""returns True once every n calls - randomly"""
if random.randrange(0, n) != 0:
return False
# print("CHAOS strikes !", file=sys.stderr)
return True
def only_commit(fname, message):
'''Only used to commit a survex file edited and saved in view/survex.py
'''
"""Only used to commit a survex file edited and saved in view/survex.py"""
git = settings.GIT
cwd = fname.parent
filename = fname.name
#print(f'{fname=} ')
# print(f'{fname=} ')
try:
cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True)
if cp_add.returncode != 0:
msgdata = f'Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}'
raise WriteAndCommitError(f'CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n' + msgdata)
msgdata = f"Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}"
raise WriteAndCommitError(
f"CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n" + msgdata
)
cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True)
# This produces return code = 1 if it commits OK, but when the local repo still needs to be pushed to origin/loser
# which will be the case when running a test troggle system on a development machine
devok_text ='''On branch master
devok_text = """On branch master
Your branch is ahead of 'origin/master' by 1 commit.
(use "git push" to publish your local commits)
nothing to commit, working tree clean
'''
"""
if cp_commit.returncode == 1 and cp_commit.stdout == devok_text:
pass
else:
if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith('nothing to commit, working tree clean'):
msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}'
if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith(
"nothing to commit, working tree clean"
):
msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}'
print(msgdata)
raise WriteAndCommitError(f'Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n' + msgdata)
raise WriteAndCommitError(
f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n"
+ msgdata
)
except subprocess.SubprocessError:
raise WriteAndCommitError(f'CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this.')
raise WriteAndCommitError(
f"CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this."
)
def write_and_commit(files, message):
"""Writes the content to the filepath and adds and commits the file to git. If this fails, a WriteAndCommitError is raised.
This does not create any needed intermediate folders, which is what we do when writing survex files, so functionality here
is duplicated in only_commit()
These need refactoring
"""
git = settings.GIT
@@ -125,46 +126,73 @@ def write_and_commit(files, message):
kwargs = {"encoding": encoding}
else:
mode = "wb"
kwargs = {}
kwargs = {}
try:
with open(filepath, mode, **kwargs) as f:
print(f'WRITING{cwd}---{filename} ')
print(f"WRITING{cwd}---{filename} ")
# as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
# os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
f.write(content)
except PermissionError:
raise WriteAndCommitError(f'CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this.')
raise WriteAndCommitError(
f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this."
)
cp_diff = subprocess.run([git, "diff", filename], cwd=cwd, capture_output=True, text=True)
if cp_diff.returncode == 0:
cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True)
if cp_add.returncode != 0:
msgdata = 'Ask a nerd to fix this.\n\n' + cp_add.stderr + '\n\n' + cp_add.stdout + '\n\nreturn code: ' + str(cp_add.returncode)
raise WriteAndCommitError(f'CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n' + msgdata)
msgdata = (
"Ask a nerd to fix this.\n\n"
+ cp_add.stderr
+ "\n\n"
+ cp_add.stdout
+ "\n\nreturn code: "
+ str(cp_add.returncode)
)
raise WriteAndCommitError(
f"CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n"
+ msgdata
)
else:
print(f"No change {filepah}")
cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True)
cp_status = subprocess.run([git, "status"], cwd=cwd, capture_output=True, text=True)
# This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb
if cp_status.stdout.split("\n")[-2] != 'nothing to commit, working tree clean':
if cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean":
print("FOO: ", cp_status.stdout.split("\n")[-2])
msgdata = 'Ask a nerd to fix this.\n\n' + cp_status.stderr + '\n\n' + cp_status.stdout + '\n\nreturn code: ' + str(cp_status.returncode)
raise WriteAndCommitError(f'Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n' + msgdata)
msgdata = (
"Ask a nerd to fix this.\n\n"
+ cp_status.stderr
+ "\n\n"
+ cp_status.stdout
+ "\n\nreturn code: "
+ str(cp_status.returncode)
)
raise WriteAndCommitError(
f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n"
+ msgdata
)
except subprocess.SubprocessError:
raise WriteAndCommitError(f'CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this.')
raise WriteAndCommitError(
f"CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this."
)
class WriteAndCommitError(Exception):
"""Exception class for errors writing files and comitting them to git"""
def __init__(self, message):
self.message = message
def __str__(self):
return f'WriteAndCommitError: {self.message}'
return f"WriteAndCommitError: {self.message}"
def writetrogglefile(filepath, filecontent):
'''Commit the new saved file to git
"""Commit the new saved file to git
Callers to cave.writeDataFile() or entrance.writeDataFile() should handle the exception PermissionsError explicitly
'''
"""
# GIT see also core/views/expo.py editexpopage()
# GIT see also core/views/uploads.py dwgupload()
# Called from core/models/caves.py Cave.writeDataFile() Entrance.writeDataFile()
@@ -175,41 +203,47 @@ def writetrogglefile(filepath, filecontent):
# as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
# do not trap exceptions, pass them up to the view that called this function
print(f'WRITING{cwd}---{filename} ')
print(f"WRITING{cwd}---{filename} ")
with open(filepath, "w") as f:
f.write(filecontent)
#os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
# os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True)
if sp.returncode != 0:
out = sp.stdout
if len(out) > 160:
out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
print(f'git ADD {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out + '\n\nreturn code: ' + str(sp.returncode))
print(f"git ADD {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode))
sp = subprocess.run([git, "commit", "-m", f'Troggle online: cave or entrance edit -{filename}'], cwd=cwd, capture_output=True, check=True, text=True)
sp = subprocess.run(
[git, "commit", "-m", f"Troggle online: cave or entrance edit -{filename}"],
cwd=cwd,
capture_output=True,
check=True,
text=True,
)
if sp.returncode != 0:
out = sp.stdout
if len(out) > 160:
out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
print(f'git COMMIT {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out + '\n\nreturn code: ' + str(sp.returncode))
print(f"git COMMIT {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode))
# not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that.
def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
"""Looks up instance using lookupAttribs and carries out the following:
-if instance does not exist in DB: add instance to DB, return (new instance, True)
-if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False)
-if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False)
The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field
defined in core.models.TroggleModel.
We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false.
NOTE: this takes twice as long as simply creating a new object with the given values.
As of Jan.2023 this function is not used anywhere in troggle.
-if instance does not exist in DB: add instance to DB, return (new instance, True)
-if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False)
-if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False)
The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field
defined in core.models.TroggleModel.
We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false.
NOTE: this takes twice as long as simply creating a new object with the given values.
As of Jan.2023 this function is not used anywhere in troggle.
"""
try:
instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
@@ -219,7 +253,9 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
raise
if not created and not instance.new_since_parsing:
for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title)
for k, v in list(
nonLookupAttribs.items()
): # overwrite the existing attributes from the logbook text (except date and title)
setattr(instance, k, v)
try:
instance.save()
@@ -233,18 +269,17 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
except:
msg = f"FAULT getting __str__ for instance with lookupattribs: {lookupAttribs}:"
if created:
logging.info(str(instance) + ' was just added to the database for the first time. \n')
logging.info(str(instance) + " was just added to the database for the first time. \n")
if not created and instance.new_since_parsing:
logging.info(str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n")
logging.info(
str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n"
)
if not created and not instance.new_since_parsing:
logging.info(" instance:<"+ str(instance) + "> existed in the database unchanged since last parse. It have been overwritten.")
logging.info(
" instance:<"
+ str(instance)
+ "> existed in the database unchanged since last parse. It have been overwritten."
)
return (instance, created)