troggle/core/utils.py

import hashlib
import logging
import os
import random
import resource
import string
import subprocess
from decimal import getcontext
from pathlib import Path

from troggle.core.models.troggle import Expedition

getcontext().prec = 2  # use 2 significant figures for decimal calculations

import settings

"""This file declares TROG a globally visible object for caches.

TROG is a dictionary holding globally visible indexes and cache functions.
It is a Global Object, see https://python-patterns.guide/python/module-globals/
troggle.utils.TROG

chaosmonkey(n) - used by survex import to regenerate some .3d files
save_carefully() - core function that saves troggle objects in the database

various git add/commit functions that need refactoring together

NOTE that TROG is not serialized! Two users can update it and conflict !! 
This needs to be in a multi-user database with transactions. However it is
useful when doing a data import with databaseReset.py as that has a single
thread.

"""

TROG = {"pagecache": {"expedition": {}}, "caves": {"gcavelookup": {}, "gcavecount": {}}}
alphabet = []
sha = hashlib.new('sha256')

# This is module-level executable. This is a Bad Thing. Especially when it touches the file system.
try:
    logging.basicConfig(level=logging.DEBUG, filename=settings.LOGFILE, filemode="w")
except:
    # Opening of file for writing is going to fail currently, so decide it doesn't matter for now
    pass


def sanitize_name(name):
    """Filenames sould not caontain these characters as then the system barf when it tries to use them in URLs
    """
    return name.replace("#","-").replace("?","=").replace("&","+").replace(":","^")

def get_process_memory():
    usage = resource.getrusage(resource.RUSAGE_SELF)
    return usage[2] / 1024.0


def chaosmonkey(n):
    """returns True once every n calls - randomly"""
    if random.randrange(0, n) != 0:
        return False
    # print("CHAOS strikes !", file=sys.stderr)
    return True
    
def unique_slug(text, n):
    """This gives an almost-unique id based on the text, 
    2 hex digits would seem adequate, but we might get a collision.
    Not used anywhere.
    """
    sha.update(text.encode('utf-8'))
    return sha.hexdigest()[0:n]
    
def alphabet_suffix(n):
    """This is called repeatedly during initial parsing import, hence the cached list
    """
    global alphabet
    if not alphabet:
        alphabet = list(string.ascii_lowercase)
        
    if n < len(alphabet) and n > 0:
        suffix = alphabet[n-1]
    else:
        suffix = "_X_" + random.choice(string.ascii_lowercase) + random.choice(string.ascii_lowercase)
    return suffix   

def current_expo():
    expos = Expedition.objects.all().order_by('-year')
    if expos:
        return expos[0].year
    else:
        return settings.EPOCH.year

def only_commit(fname, message):
    """Only used to commit a survex file edited and saved in view/survex.py"""
    git = settings.GIT
    cwd = fname.parent
    filename = fname.name
    # print(f'{fname=} ')

    try:
        cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True)
        if cp_add.returncode != 0:
            msgdata = f"Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}"
            raise WriteAndCommitError(
                f"CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n" + msgdata
            )

        cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True)
        # This produces return code = 1 if it commits OK, but when the local repo still needs to be pushed to origin/loser
        # which will be the case when running a test troggle system on a development machine
        devok_text = """On branch master
Your branch is ahead of 'origin/master' by 1 commit.
  (use "git push" to publish your local commits)

nothing to commit, working tree clean
"""
        if cp_commit.returncode == 1 and cp_commit.stdout == devok_text:
            pass
        else:
            if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith(
                "nothing to commit, working tree clean"
            ):
                msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}'
                print(msgdata)
                raise WriteAndCommitError(
                    f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n"
                    + msgdata
                )

    except subprocess.SubprocessError:
        raise WriteAndCommitError(
            f"CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this."
        )


def write_and_commit(files, message):
    """Writes the content to the filepath and adds and commits the file to git.  If this fails, a WriteAndCommitError is raised.
    This does not create any needed intermediate folders, which is what we do when writing survex files, so functionality here
    is duplicated in only_commit()

    These need refactoring
    """
    git = settings.GIT
    commands = []
    try:
        for filepath, content, encoding in files:
            cwd = filepath.parent
            filename = filepath.name
            # GIT see also core/views/uploads.py dwgupload()
            # GIT see also core/views/expo.py editexpopage()
            os.makedirs(os.path.dirname(filepath), exist_ok = True)
            if encoding:
                mode = "w"
                kwargs = {"encoding": encoding}
            else:
                mode = "wb"
                kwargs = {}
            try:
                with open(filepath, mode, **kwargs) as f:
                    print(f"WRITING {cwd}---{filename} ")
                    # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
                    # os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
                    f.write(content)
            except PermissionError:
                raise WriteAndCommitError(
                    f"CANNOT  save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this."
                )
            cmd_diff = [git, "diff", filename]
            cp_diff = subprocess.run(cmd_diff, cwd=cwd, capture_output=True, text=True)
            commands.append(cmd_diff)
            if cp_diff.returncode == 0:
                cmd_add = [git, "add", filename]
                cp_add = subprocess.run(cmd_add, cwd=cwd, capture_output=True, text=True)
                commands.append(cmd_add)
                if cp_add.returncode != 0:
                    msgdata = (
                        "Ask a nerd to fix this.\n\n"
                        + cp_add.stderr
                        + "\n\n"
                        + cp_add.stdout
                        + "\n\nreturn code: "
                        + str(cp_add.returncode)
                    )
                    raise WriteAndCommitError(
                        f"CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n"
                        + msgdata
                    )
            else:
                print(f"No change {filepath}")
        filepaths = [filepath for filepath, content, encoding in files]
        cmd_commit = [git, "commit"] + filepaths + ["-m", message]
        cm_status = subprocess.run(cmd_commit, cwd=cwd, capture_output=True, text=True)
        commands.append(cmd_commit)
        if cm_status == 0:
            msgdata = (
                "Commands: " + str(commands) +
                "Ask a nerd to fix this.\n\n"
                + "Stderr: " + cp_status.stderr
                + "\n\n"
                + "Stdout: " + cp_status.stdout
                + "\n\nreturn code: " + str(cp_status.returncode)
            )
            raise WriteAndCommitError(
                f"Error committing. Edits saved, added to git, but NOT committed.\n\n"
                + msgdata
            )     
        cmd_status = [git, "status"] + filepaths   
        cp_status = subprocess.run(cmd_status, cwd=cwd, capture_output=True, text=True)
        commands.append(cp_status)
        #This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb
        if (not cp_status.stdout) or len(cp_status.stdout) < 2 or cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean":
            msgdata = (
                str(commands) +
                "Ask a nerd to fix this.\n\n"
                + "Stderr: " + cp_status.stderr
                + "\n\n"
                + "Stdout: " + cp_status.stdout
                + "\n\nreturn code: " + str(cp_status.returncode)
            )
            raise WriteAndCommitError(
                f"Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n"
                + msgdata
            )
    except subprocess.SubprocessError:
        raise WriteAndCommitError(
            f"CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this."
        )


class WriteAndCommitError(Exception):
    """Exception class for errors writing files and comitting them to git"""

    def __init__(self, message):
        self.message = message

    def __str__(self):
        return f"WriteAndCommitError: {self.message}"


def writetrogglefile(filepath, filecontent):
    """Commit the new saved file to git
    Callers to cave.writeDataFile() or entrance.writeDataFile() should handle the exception PermissionsError explicitly
    """
    # GIT see also core/views/expo.py editexpopage()
    # GIT see also core/views/uploads.py dwgupload()
    # Called from core/models/caves.py Cave.writeDataFile() Entrance.writeDataFile()
    filepath = Path(filepath)
    cwd = filepath.parent
    filename = filepath.name
    git = settings.GIT

    # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
    # do not trap exceptions, pass them up to the view that called this function
    print(f"WRITING{cwd}---{filename} ")
    with open(filepath, "w") as f:
        f.write(filecontent)
    # os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
    sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True)
    if sp.returncode != 0:
        out = sp.stdout
        if len(out) > 160:
            out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
        print(f"git ADD {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode))

    sp = subprocess.run(
        [git, "commit", "-m", f"Troggle online: cave or entrance edit -{filename}"],
        cwd=cwd,
        capture_output=True,
        check=True,
        text=True,
    )
    if sp.returncode != 0:
        out = sp.stdout
        if len(out) > 160:
            out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
        print(f"git COMMIT {cwd}:\n\n" + str(sp.stderr) + "\n\n" + out + "\n\nreturn code: " + str(sp.returncode))
    # not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that.


def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
    """Looks up instance using lookupAttribs and carries out the following:
        -if instance does not exist in DB: add instance to DB, return (new instance, True)
        -if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False)
        -if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False)

    The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field
    defined in core.models.TroggleModel.

    We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false.

    NOTE: this takes twice as long as simply creating a new object with the given values.

    As of Jan.2023 this function is not used anywhere in troggle.

    """
    try:
        instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
    except:
        print(" !! - FAIL in SAVE CAREFULLY   ===================", objectType)
        print(" !! - -- objects.get_or_create()")
        print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
        raise
    if not created and not instance.new_since_parsing:
        for k, v in list(
            nonLookupAttribs.items()
        ):  # overwrite the existing attributes from the logbook text (except date and title)
            setattr(instance, k, v)
        try:
            instance.save()
        except:
            print(" !! - SAVE CAREFULLY   ===================", objectType)
            print(" !! - -- instance.save()")
            print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
            raise
    try:
        str(instance)
    except:
        pass
    if created:
        logging.info(str(instance) + " was just added to the database for the first time. \n")

    if not created and instance.new_since_parsing:
        logging.info(
            str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n"
        )

    if not created and not instance.new_since_parsing:
        logging.info(
            " instance:<"
            + str(instance)
            + "> existed in the database unchanged since last parse. It have been overwritten."
        )
    return (instance, created)