creating core/utils.py

core/models.py

@@ -22,6 +22,7 @@ from django.urls import reverse
 from django.template import Context, loader
 
 import troggle.core.models_survex 
+from troggle.core.utils import get_process_memory
 
 """This file declares TroggleModel which inherits from django.db.models.Model 
 All TroggleModel subclasses inherit persistence in the django relational database. This is known as
@@ -29,43 +30,6 @@ the django Object Relational Mapping (ORM).
 There are more subclasses define in models_caves.py models_survex.py etc.
 """
 
-# This variable is a dictionary holding gloablly visible indexes and cache functions.
-# It is a Global Object, see https://python-patterns.guide/python/module-globals/
-# troggle.models.TROG
-TROG = {
-    'pagecache' : {
-        'expedition' : {}
-    }
-}
-
-def get_process_memory():
-    usage=resource.getrusage(resource.RUSAGE_SELF)
-    return usage[2]/1024.0 
-
-
-# def get_related_by_wikilinks(wiki_text):
-    # found=re.findall(settings.QM_PATTERN,wiki_text)
-    # res=[]
-    # for wikilink in found:
-        # qmdict={'urlroot':settings.URL_ROOT,'cave':wikilink[2],'year':wikilink[1],'number':wikilink[3]}
-        # try:
-            # cave_slugs = models_caves.CaveSlug.objects.filter(cave__kataster_number = qmdict['cave'])
-            # qm=QM.objects.get(found_by__cave_slug__in = cave_slugs,
-                              # found_by__date__year = qmdict['year'],
-                              # number = qmdict['number'])
-            # res.append(qm)         
-        # except QM.DoesNotExist:
-            # print(('fail on '+str(wikilink)))
-    
-    # return res
-
-try:
-      logging.basicConfig(level=logging.DEBUG,
-                           filename=settings.LOGFILE,
-                           filemode='w')
-except:
-# Opening of file for writing is going to fail currently, so decide it doesn't matter for now    
-      pass
 
 #This class is for adding fields and methods which all of our models will have.
 class TroggleModel(models.Model):

core/utils.py

@@ -0,0 +1,84 @@
+import string
+import os
+import datetime
+import logging
+import re
+import resource
+from subprocess import call
+
+from urllib.parse import urljoin
+from decimal import Decimal, getcontext
+getcontext().prec=2 #use 2 significant figures for decimal calculations
+
+import settings
+
+from django.db import models
+from django.contrib import admin
+from django.contrib.auth.models import User
+from django.contrib.contenttypes.models import ContentType
+from django.conf import settings
+
+from django.urls import reverse
+from django.template import Context, loader
+
+#import troggle.core.models_survex 
+
+'''This file declares TROG a globally visible object for caches.
+
+TROG is a dictionary holding gloablly visible indexes and cache functions.
+It is a Global Object, see https://python-patterns.guide/python/module-globals/
+troggle.models.TROG
+
+chaosmonkey(n) - used by survex import to regenerate some .3d files
+save_carefully() - core function that saves troggle objects in the database
+
+'''
+
+TROG = {
+    'pagecache' : {
+        'expedition' : {}
+    }
+}
+
+# This is module-level executable. This is a Bad Thing.
+try:
+      logging.basicConfig(level=logging.DEBUG,
+                           filename=settings.LOGFILE,
+                           filemode='w')
+except:
+# Opening of file for writing is going to fail currently, so decide it doesn't matter for now    
+      pass
+
+def get_process_memory():
+    usage=resource.getrusage(resource.RUSAGE_SELF)
+    return usage[2]/1024.0 
+
+def chaosmonkey(n):
+    # returns True once every n calls - randomly
+    if random.randrange(0,n) != 0:
+        return False
+    # print("CHAOS strikes !", file=sys.stderr)
+    return True
+
+# def get_related_by_wikilinks(wiki_text):
+    # found=re.findall(settings.QM_PATTERN,wiki_text)
+    # res=[]
+    # for wikilink in found:
+        # qmdict={'urlroot':settings.URL_ROOT,'cave':wikilink[2],'year':wikilink[1],'number':wikilink[3]}
+        # try:
+            # cave_slugs = models_caves.CaveSlug.objects.filter(cave__kataster_number = qmdict['cave'])
+            # qm=QM.objects.get(found_by__cave_slug__in = cave_slugs,
+                              # found_by__date__year = qmdict['year'],
+                              # number = qmdict['number'])
+            # res.append(qm)         
+        # except QM.DoesNotExist:
+            # print(('fail on '+str(wikilink)))
+    
+    # return res
+
+
+
+
+
+
+

core/views/logbooks.py

@@ -14,7 +14,8 @@ from django.views.generic.list import ListView
 
 #import troggle.parsers.logbooks as logbookparsers
 from troggle.core.forms import getTripForm  # , get_name, PersonForm
-from troggle.core.models import Expedition, Person, PersonExpedition, TROG
+from troggle.core.models import Expedition, Person, PersonExpedition
+from troggle.core.utils import TROG
 from troggle.core.models_caves import LogbookEntry, PersonTrip
 from troggle.core.models_survex import SurvexBlock
 from .login import login_required_if_public

parsers/logbooks.py

@@ -10,7 +10,8 @@ from django.conf import settings
 from django.template.defaultfilters import slugify
 from django.utils.timezone import get_current_timezone, make_aware
 
-from troggle.core.models import DataIssue, Expedition, TROG
+from troggle.core.models import DataIssue, Expedition
+from troggle.core.utils import TROG
 from troggle.core.models_caves import Cave, LogbookEntry, PersonTrip, GetCaveLookup
 from parsers.people import GetPersonExpeditionNameLookup
 from utils import save_carefully

parsers/survex.py

@@ -15,7 +15,7 @@ import troggle.settings as settings
 import troggle.core.models as models
 import troggle.core.models_caves as models_caves
 import troggle.core.models_survex as models_survex
-from troggle.utils import ChaosMonkey
+from troggle.core.utils import get_process_memory, chaosmonkey
 from troggle.parsers.people import GetPersonExpeditionNameLookup
 from troggle.parsers.logbooks import GetCaveLookup
 from troggle.core.views.caves import MapLocations
@@ -716,7 +716,7 @@ class LoadingSurvex():
         self.depthbegin = 0
         self.datastar = self.datastardefault
         blocklegs = self.legsnumber
-        print(self.insp+"  - MEM:{:.3f} Reading. parent:{}  <> {} ".format(models.get_process_memory(),survexblock.survexfile.path, survexfile.path))
+        print(self.insp+"  - MEM:{:.3f} Reading. parent:{}  <> {} ".format(get_process_memory(),survexblock.survexfile.path, survexfile.path))
         self.lineno = 0
         sys.stderr.flush();
         self.callcount +=1
@@ -761,7 +761,7 @@ class LoadingSurvex():
                 print(".", file=sys.stderr,end='')
             if blockcount % 200 ==0 :
                 print("\n", file=sys.stderr,end='')
-                print(" - MEM:{:7.3f} MB in use".format(models.get_process_memory()),file=sys.stderr)
+                print(" - MEM:{:7.3f} MB in use".format(get_process_memory()),file=sys.stderr)
                 print("    ", file=sys.stderr,end='')
             sys.stderr.flush()
 
@@ -1104,7 +1104,7 @@ class LoadingSurvex():
         if cav_t - log_t > 0:          # new version of cavern
             runcavern()
             return
-        if ChaosMonkey(200):
+        if chaosmonkey(200):
             runcavern()
 
 def FindAndLoadSurvex(survexblockroot):
@@ -1128,7 +1128,7 @@ def FindAndLoadSurvex(survexblockroot):
     indent=""
     fcollate = open(collatefilename, 'w')
 
-    mem0 = models.get_process_memory()
+    mem0 = get_process_memory()
     print(" - MEM:{:7.2f} MB START".format(mem0),file=sys.stderr)
     flinear = open('svxlinear.log', 'w')
     flinear.write("    - MEM:{:7.2f} MB START {}\n".format(mem0,survexfileroot.path))
@@ -1142,7 +1142,7 @@ def FindAndLoadSurvex(survexblockroot):
     #----------------------------------------------------------------
     flinear.write("{:2} {} *edulcni {}\n".format(svx_scan.depthinclude, indent, survexfileroot.path))
     fcollate.write(";*edulcni {}\n".format(survexfileroot.path))
-    mem1 = models.get_process_memory()
+    mem1 = get_process_memory()
     flinear.write("\n    - MEM:{:.2f} MB STOP {}\n".format(mem1,survexfileroot.path))
     flinear.write("    - MEM:{:.3f} MB USED\n".format(mem1-mem0))
     svxfileslist = svx_scan.svxfileslist
@@ -1152,7 +1152,7 @@ def FindAndLoadSurvex(survexblockroot):
     svx_scan = None # Hmm. Does this actually delete all the instance variables if they are lists, dicts etc.?
     print("\n -  {:,} survex files in linear include list \n".format(len(svxfileslist)),file=sys.stderr)
 
-    mem1 = models.get_process_memory()
+    mem1 = get_process_memory()
     print(" - MEM:{:7.2f} MB END ".format(mem0),file=sys.stderr)
     print(" - MEM:{:7.3f} MB USED".format(mem1-mem0),file=sys.stderr)
     svxfileslist = [] # free memory
@@ -1180,7 +1180,7 @@ def FindAndLoadSurvex(survexblockroot):
     print(" - MEM:{:7.3f} MB USED".format(mem1-mem0),file=sys.stderr)
 
     legsnumber = svx_load.legsnumber
-    mem1 = models.get_process_memory()
+    mem1 = get_process_memory()
 
     print("  - Number of SurvexDirectories: {}".format(len(svx_load.survexdict)))
     tf=0
@@ -1227,11 +1227,11 @@ def LoadSurvexBlocks():
     survexblockroot.save()
 
     print(' - Loading Survex Blocks...')
-    memstart = models.get_process_memory()
+    memstart = get_process_memory()
     #----------------------------------------------------------------
     legsnumber = FindAndLoadSurvex(survexblockroot)
     #----------------------------------------------------------------
-    memend = models.get_process_memory()
+    memend = get_process_memory()
     print(" - MEMORY start:{:.3f} MB end:{:.3f} MB increase={:.3f} MB".format(memstart,memend, memend-memstart))
 
     survexblockroot.save()

utils.py

@@ -9,8 +9,6 @@ from django.shortcuts import render
 """Oddball mixture of critical, superfluous and useful functions which should 
 be re-located more sensibly to other modules:
 
-ChaosMonkey(n) - used by survex import to regenerate some .3d files
-save_carefully() - core function that saves troggle objects in the database
 
 various HTML/wiki functions presumably for logbooks?
 
@@ -19,12 +17,6 @@ weighted_choice(lst)
 randomLogbookSentence()
 """
 
-def ChaosMonkey(n):
-    # returns True once every n calls - randomly
-    if random.randrange(0,n) != 0:
-        return False
-    # print("CHAOS strikes !", file=sys.stderr)
-    return True
 
 def weighted_choice(lst):
     n = random.uniform(0,1)