troggle/core/utils.py

import datetime
import logging
import os
import random
import re
import resource
import string
import subprocess
from decimal import Decimal, getcontext
from pathlib import Path
from urllib.parse import urljoin

getcontext().prec=2 #use 2 significant figures for decimal calculations

from django.conf import settings
from django.contrib import admin
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.template import Context, loader
from django.urls import reverse

import settings

'''This file declares TROG a globally visible object for caches.

TROG is a dictionary holding globally visible indexes and cache functions.
It is a Global Object, see https://python-patterns.guide/python/module-globals/
troggle.utils.TROG

chaosmonkey(n) - used by survex import to regenerate some .3d files
save_carefully() - core function that saves troggle objects in the database

various git add/commit functions that need refactoring together

'''

TROG = {
    'pagecache' : {
        'expedition' : {}
    },
    'issues' : {
        'logdataissues' : {}
    },
    'caves' : {
        'gcavelookup' : {},
        'gcavecount' : {}
    }

}

# This is module-level executable. This is a Bad Thing. Especially when it touches the file system.
try:
    logging.basicConfig(level=logging.DEBUG,
        filename=settings.LOGFILE,
        filemode='w')
except:
    # Opening of file for writing is going to fail currently, so decide it doesn't matter for now    
    pass

def get_process_memory():
    usage=resource.getrusage(resource.RUSAGE_SELF)
    return usage[2]/1024.0 

def chaosmonkey(n):
    '''returns True once every n calls - randomly'''
    if random.randrange(0,n) != 0:
        return False
    # print("CHAOS strikes !", file=sys.stderr)
    return True

def only_commit(fname, message):
    '''Only used to commit a survex file edited and saved in view/survex.py
    '''
    git = settings.GIT
    cwd = fname.parent
    filename = fname.name
    #print(f'{fname=} ')

    try:
        cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True)
        if cp_add.returncode != 0:
            msgdata = f'Ask a nerd to fix this problem in only_commit().\n--{cp_add.stderr}\n--{cp_add.stdout}\n--return code:{str(cp_add.returncode)}' 
            raise WriteAndCommitError(f'CANNOT git ADD on server for this file {filename}. Edits saved but not added to git.\n\n' + msgdata)

        cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True)
        # This produces return code = 1 if it commits OK, but when the local repo still needs to be pushed to origin/loser
        # which will be the case when running a test troggle system on a development machine
        devok_text ='''On branch master
Your branch is ahead of 'origin/master' by 1 commit.
  (use "git push" to publish your local commits)

nothing to commit, working tree clean
'''
        if cp_commit.returncode == 1 and cp_commit.stdout == devok_text:
            pass
        else:
            if cp_commit.returncode != 0 and not cp_commit.stdout.strip().endswith('nothing to commit, working tree clean'):
                msgdata = f'--Ask a nerd to fix this problem in only_commit().\n--{cp_commit.stderr}\n--"{cp_commit.stdout}"\n--return code:{str(cp_commit.returncode)}' 
                print(msgdata)
                raise WriteAndCommitError(f'Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n' + msgdata)

    except subprocess.SubprocessError:
        raise WriteAndCommitError(f'CANNOT git COMMIT on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this.')

def write_and_commit(files, message):
    """Writes the content to the filepath and adds and commits the file to git.  If this fails, a WriteAndCommitError is raised.
    This does not create any needed intermediate folders, which is what we do when writing survex files, so functionality here
    is duplicated in only_commit()
    
    These need refactoring
    """
    git = settings.GIT
    try:
        for filepath, content, encoding in files:
            cwd = filepath.parent
            filename = filepath.name
            # GIT see also core/views/uploads.py dwgupload()
            # GIT see also core/views/expo.py editexpopage()

            if encoding:
                mode = "w"
                kwargs = {"encoding": encoding}
            else:
                mode = "wb"
                kwargs = {}    
            try:
                with open(filepath, mode, **kwargs) as f:
                    print(f'WRITING{cwd}---{filename} ')
                    # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
                    # os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
                    f.write(content)
            except PermissionError:
                raise WriteAndCommitError(f'CANNOT  save this file.\nPERMISSIONS incorrectly set on server for this file {filename}. Ask a nerd to fix this.')

            cp_diff = subprocess.run([git, "diff", filename], cwd=cwd, capture_output=True, text=True)
            if cp_diff.returncode == 0:
                cp_add = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, text=True)
                if cp_add.returncode != 0:
                    msgdata = 'Ask a nerd to fix this.\n\n' + cp_add.stderr + '\n\n' + cp_add.stdout  + '\n\nreturn code: ' + str(cp_add.returncode)
                    raise WriteAndCommitError(f'CANNOT git on server for this file {filename}. Edits saved but not added to git.\n\n' + msgdata)
            else:
                print(f"No change {filepah}")
        cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True)
        cp_status = subprocess.run([git, "status"], cwd=cwd, capture_output=True, text=True)
        # This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb
        if cp_status.stdout.split("\n")[-2] != 'nothing to commit, working tree clean':
            print("FOO: ", cp_status.stdout.split("\n")[-2])
            msgdata = 'Ask a nerd to fix this.\n\n' + cp_status.stderr + '\n\n' + cp_status.stdout  + '\n\nreturn code: ' + str(cp_status.returncode)
            raise WriteAndCommitError(f'Error code with git on server for this file {filename}. Edits saved, added to git, but NOT committed.\n\n' + msgdata)
    except subprocess.SubprocessError:
        raise WriteAndCommitError(f'CANNOT git on server for this file {filename}. Subprocess error. Edits not saved.\nAsk a nerd to fix this.')

class WriteAndCommitError(Exception):
    """Exception class for errors writing files and comitting them to git"""
    def __init__(self, message):
        self.message = message

    def __str__(self):
        return f'WriteAndCommitError: {self.message}'
        
def writetrogglefile(filepath, filecontent):
    '''Commit the new saved file to git
    Callers to cave.writeDataFile() or entrance.writeDataFile() should handle the exception PermissionsError explicitly
    '''
    # GIT see also core/views/expo.py editexpopage()
    # GIT see also core/views/uploads.py dwgupload()
    # Called from core/models/caves.py Cave.writeDataFile() Entrance.writeDataFile()
    filepath = Path(filepath)
    cwd = filepath.parent
    filename = filepath.name
    git = settings.GIT

    # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
    # do not trap exceptions, pass them up to the view that called this function
    print(f'WRITING{cwd}---{filename} ')
    with open(filepath, "w") as f:
        f.write(filecontent)
    #os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
    sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True)
    if sp.returncode != 0:
        out = sp.stdout
        if len(out) > 160:
            out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
        print(f'git ADD {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out  + '\n\nreturn code: ' + str(sp.returncode))

    sp = subprocess.run([git, "commit", "-m", f'Troggle online: cave or entrance edit -{filename}'], cwd=cwd, capture_output=True, check=True, text=True)
    if sp.returncode != 0:
        out = sp.stdout
        if len(out) > 160:
            out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
        print(f'git COMMIT {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out  + '\n\nreturn code: ' + str(sp.returncode))
    # not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that.


def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
    """Looks up instance using lookupAttribs and carries out the following:
            -if instance does not exist in DB: add instance to DB, return (new instance, True)
            -if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False)
            -if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False)
            
        The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field
        defined in core.models.TroggleModel.
        
        We are not using new_since_parsing - it is a fossil from Aaron Curtis's design in 2006. So it is always false.
    
    """
    try:
        instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
    except:
        print(" !! - FAIL in SAVE CAREFULLY   ===================", objectType)
        print(" !! - -- objects.get_or_create()")
        print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
        raise
    if not created and not instance.new_since_parsing:
        for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title)
            setattr(instance, k, v)
        try:
            instance.save()
        except:
            print(" !! - SAVE CAREFULLY   ===================", objectType)
            print(" !! - -- instance.save()")
            print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
            raise
    try:
        msg = str(instance)
    except:
        msg = f"FAULT getting __str__ for instance with lookupattribs: {lookupAttribs}:"
    if created:
        logging.info(str(instance) + ' was just added to the database for the first time. \n')
    
    if not created and instance.new_since_parsing:
        logging.info(str(instance) + " has been modified using Troggle since parsing, so the current script left it as is. \n")

    if not created and not instance.new_since_parsing:
        logging.info(" instance:<"+ str(instance) + "> existed in the database unchanged since last parse. It have been overwritten.")
    return (instance, created)