import string import os import datetime import re import resource import random import logging from subprocess import call from urllib.parse import urljoin from decimal import Decimal, getcontext getcontext().prec=2 #use 2 significant figures for decimal calculations import settings from django.db import models from django.contrib import admin from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.conf import settings from django.urls import reverse from django.template import Context, loader '''This file declares TROG a globally visible object for caches. TROG is a dictionary holding gloablly visible indexes and cache functions. It is a Global Object, see https://python-patterns.guide/python/module-globals/ troggle.utils.TROG chaosmonkey(n) - used by survex import to regenerate some .3d files save_carefully() - core function that saves troggle objects in the database ''' TROG = { 'pagecache' : { 'expedition' : {} }, 'issues' : { 'logdataissues' : {} } } # This is module-level executable. This is a Bad Thing. Especially when it touches the file system. try: logging.basicConfig(level=logging.DEBUG, filename=settings.LOGFILE, filemode='w') except: # Opening of file for writing is going to fail currently, so decide it doesn't matter for now pass def get_process_memory(): usage=resource.getrusage(resource.RUSAGE_SELF) return usage[2]/1024.0 def chaosmonkey(n): # returns True once every n calls - randomly if random.randrange(0,n) != 0: return False # print("CHAOS strikes !", file=sys.stderr) return True # handles url or file, so we can refer to a set of scans (not drawings) on another server def GetListDir(sdir): res = [ ] if sdir[:7] == "http://": # s = urllib.request.urlopen(sdir) message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]" print(message) DataIssue.objects.create(parser='Drawings', message=message) sdir[:7] = "" for f in os.listdir(sdir): if f[0] != ".": ff = os.path.join(sdir, f) res.append((f, ff, os.path.isdir(ff))) return res def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): """Looks up instance using lookupAttribs and carries out the following: -if instance does not exist in DB: add instance to DB, return (new instance, True) -if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False) -if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False) The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field defined in core.models.TroggleModel. """ try: instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs) except: print(" !! - FAIL in SAVE CAREFULLY ===================", objectType) print(" !! - -- objects.get_or_create()") print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs)) raise if not created and not instance.new_since_parsing: for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title) setattr(instance, k, v) try: instance.save() except: print(" !! - SAVE CAREFULLY ===================", objectType) print(" !! - -- instance.save()") print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs)) raise try: msg = str(instance) except: msg = "FAULT getting __str__ for instance with lookupattribs: {}:".format(lookupAttribs) if created: logging.info(str(instance) + ' was just added to the database for the first time. \n') if not created and instance.new_since_parsing: logging.info(str(instance) + " has been modified using Troggle, so the current script left it as is. \n") if not created and not instance.new_since_parsing: logging.info(str(instance) + " existed in the database unchanged since last parse. It was overwritten by the current script. \n") return (instance, created)