import datetime import logging import os import random import re import resource import string import subprocess from decimal import Decimal, getcontext from pathlib import Path from urllib.parse import urljoin getcontext().prec = 2 # use 2 significant figures for decimal calculations from django.conf import settings from django.contrib import admin from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.db import models from django.template import Context, loader from django.urls import reverse import settings """This file declares TROG a globally visible object for caches. TROG is a dictionary holding globally visible indexes and cache functions. It is a Global Object, see https://python-patterns.guide/python/module-globals/ troggle.utils.TROG chaosmonkey(n) - used by survex import to regenerate some .3d files save_carefully() - core function that saves troggle objects in the database various git add/commit functions that need refactoring together NOTE that TROG is not serialized! Two users can update it and conflict !! This needs to be in a multi-user database with transactions. However it is useful when doing a data import with databaseReset.py as that has a single thread. """ TROG = {"pagecache": {"expedition": {}}, "caves": {"gcavelookup": {}, "gcavecount": {}}} # This is module-level executable. This is a Bad Thing. Especially when it touches the file system. try: logging.basicConfig(level=logging.DEBUG, filename=settings.LOGFILE, filemode="w") except: # Opening of file for writing is going to fail currently, so decide it doesn't matter for now pass def get_process_memory(): usage = resource.getrusage(resource.RUSAGE_SELF) return usage[2] / 1024.0 def chaosmonkey(n): """returns True once every n calls - randomly""" if random.randrange(0, n) != 0: return False # print("CHAOS strikes !", file=sys.stderr) return True def only_commit(fname, message): """Only used to commit a survex file edited and saved in view/survex.py""" git = settings.GIT cwd = fname.parent filename = fname.name # print(f'{fname=} ') try: cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True) if cp_add.returncode != 0: msgdata = f"Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}" raise WriteAndCommitError( f"CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n" + msgdata ) cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True) # This produces return code = 1 if it commits OK, but when the local repo still needs to be pushed to origin/loser # which will be the case when running a test troggle system on a development machine devok_text = """On branch master Your branch is ahead of 'origin/master' by 1 commit. (use "git push" to publish your local commits) nothing to commit, working tree clean """ if cp_commit.returncode == 1 and cp_commit.stdout == devok_text: pass else: if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith( "nothing to commit, working tree clean" ): msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}' print(msgdata) raise WriteAndCommitError( f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n" + msgdata ) except subprocess.SubprocessError: raise WriteAndCommitError( f"CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this." ) def write_and_commit(files, message): """Writes the content to the filepath and adds and commits the file to git. If this fails, a WriteAndCommitError is raised. This does not create any needed intermediate folders, which is what we do when writing survex files, so functionality here is duplicated in only_commit() These need refactoring """ git = settings.GIT try: for filepath, content, encoding in files: cwd = filepath.parent filename = filepath.name # GIT see also core/views/uploads.py dwgupload() # GIT see also core/views/expo.py editexpopage() if encoding: mode = "w" kwargs = {"encoding": encoding} else: mode = "wb" kwargs = {} try: with open(filepath, mode, **kwargs) as f: print(f"WRITING{cwd}---{filename} ") # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod. # os.chmod(filepath, 0o664) # set file permissions to rw-rw-r-- f.write(content) except PermissionError: raise WriteAndCommitError( f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this." ) cp_diff = subprocess.run([git, "diff", filename], cwd=cwd, capture_output=True, text=True) if cp_diff.returncode == 0: cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True) if cp_add.returncode != 0: msgdata = ( "Ask a nerd to fix this.\n\n" + cp_add.stderr + "\n\n" + cp_add.stdout + "\n\nreturn code: " + str(cp_add.returncode) ) raise WriteAndCommitError( f"CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n" + msgdata ) else: print(f"No change {filepah}") cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True) cp_status = subprocess.run([git, "status"], cwd=cwd, capture_output=True, text=True) # This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb if cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean": print("FOO: ", cp_status.stdout.split("\n")[-2]) msgdata = ( "Ask a nerd to fix this.\n\n" + cp_status.stderr + "\n\n" + cp_status.stdout + "\n\nreturn code: " + str(cp_status.returncode) ) raise WriteAndCommitError( f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n" + msgdata ) except subprocess.SubprocessError: raise WriteAndCommitError( f"CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this." ) class WriteAndCommitError(Exception): """Exception class for errors writing files and comitting them to git""" def __init__(self, message): self.message = message def __str__(self): return f"WriteAndCommitError: {self.message}" def writetrogglefile(filepath, filecontent): """Commit the new saved file to git Callers to cave.writeDataFile() or entrance.writeDataFile() should handle the exception PermissionsError explicitly """ # GIT see also core/views/expo.py editexpopage() # GIT see also core/views/uploads.py dwgupload() # Called from core/models/caves.py Cave.writeDataFile() Entrance.writeDataFile() filepath = Path(filepath) cwd = filepath.parent filename = filepath.name git = settings.GIT # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod. # do not trap exceptions, pass them up to the view that called this function print(f"WRITING{cwd}---{filename} ") with open(filepath, "w") as f: f.write(filecontent) # os.chmod(filepath, 0o664) # set file permissions to rw-rw-r-- sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True) if sp.returncode != 0: out = sp.stdout if len(out) > 160: out = out[:75] + "\n \n" + out[-75:] print(f"git ADD {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode)) sp = subprocess.run( [git, "commit", "-m", f"Troggle online: cave or entrance edit -{filename}"], cwd=cwd, capture_output=True, check=True, text=True, ) if sp.returncode != 0: out = sp.stdout if len(out) > 160: out = out[:75] + "\n \n" + out[-75:] print(f"git COMMIT {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode)) # not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that. def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): """Looks up instance using lookupAttribs and carries out the following: -if instance does not exist in DB: add instance to DB, return (new instance, True) -if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False) -if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False) The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field defined in core.models.TroggleModel. We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false. NOTE: this takes twice as long as simply creating a new object with the given values. As of Jan.2023 this function is not used anywhere in troggle. """ try: instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs) except: print(" !! - FAIL in SAVE CAREFULLY ===================", objectType) print(" !! - -- objects.get_or_create()") print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}") raise if not created and not instance.new_since_parsing: for k, v in list( nonLookupAttribs.items() ): # overwrite the existing attributes from the logbook text (except date and title) setattr(instance, k, v) try: instance.save() except: print(" !! - SAVE CAREFULLY ===================", objectType) print(" !! - -- instance.save()") print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}") raise try: msg = str(instance) except: msg = f"FAULT getting __str__ for instance with lookupattribs: {lookupAttribs}:" if created: logging.info(str(instance) + " was just added to the database for the first time. \n") if not created and instance.new_since_parsing: logging.info( str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n" ) if not created and not instance.new_since_parsing: logging.info( " instance:<" + str(instance) + "> existed in the database unchanged since last parse. It have been overwritten." ) return (instance, created)