pathlib for path management & cavelist fixes

core/views_survex.py

@@ -2,15 +2,17 @@ import re
 import os
 import datetime
 import difflib
+from pathlib import Path
 
 from django import forms
 from django.http import HttpResponseRedirect, HttpResponse
 from django.shortcuts import render_to_response, render
 #from django.core.context_processors import csrf
 from django.template.context_processors import csrf
-from django.core.exceptions import ObjectDoesNotExist
 from django.http import HttpResponse, Http404
 
+from django.core.exceptions import ObjectDoesNotExist
+
 import troggle.settings as settings
 import parsers.survex
 from troggle.core.models import Expedition, Person, PersonExpedition 
@@ -18,6 +20,7 @@ from troggle.core.models_survex import SurvexBlock, SurvexPersonRole, SurvexFile
 from troggle.core.models_caves import Cave, PersonTrip, LogbookEntry
 from troggle.parsers.people import GetPersonExpeditionNameLookup
 
+survexdatasetpath = Path(settings.SURVEX_DATA)
 
 survextemplatefile = """; *** THIS IS A TEMPLATE FILE NOT WHAT YOU MIGHT BE EXPECTING ***
 
@@ -99,8 +102,9 @@ class SvxForm(forms.Form):
     code = forms.CharField(widget=forms.Textarea(attrs={"cols":150, "rows":36}))
     
     def GetDiscCode(self):
-        fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
+        fname = survexdatasetpath / (self.data['filename'] + ".svx")
         if not os.path.isfile(fname):
+            print(">>> >>> WARNING - svx file not found, showiung TEMPLATE SVX",fname, flush=True)
             return survextemplatefile
         fin = open(fname, "rt",encoding='utf8',newline='')
         svxtext = fin.read() 
@@ -114,7 +118,7 @@ class SvxForm(forms.Form):
         return difflist
 
     def SaveCode(self, rcode):
-        fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
+        fname = survexdatasetpath / (self.data['filename'] + ".svx")
         if not os.path.isfile(fname):
             if re.search(r"\[|\]", rcode):   
                 return "Error: remove all []s from the text. They are only template guidance."
@@ -129,7 +133,7 @@ class SvxForm(forms.Form):
             fout = open(fname, "wt", encoding='utf8',newline='\n')
         except FileNotFoundError:
             pth = os.path.dirname(self.data['filename'])
-            newpath = os.path.join(settings.SURVEX_DATA, pth)
+            newpath = survexdatasetpath / pth
             if not os.path.exists(newpath):
                 os.makedirs(newpath) 
             fout = open(fname, "wt", encoding='utf8',newline='\n')
@@ -141,11 +145,13 @@ class SvxForm(forms.Form):
 
     def Process(self):
         print("....\n\n\n....Processing\n\n\n")
+        froox = os.fspath(survexdatasetpath / (self.data['filename'] + ".svx"))
+        froog = os.fspath(survexdatasetpath / (self.data['filename'] + ".log"))
         cwd = os.getcwd()
-        os.chdir(os.path.split(settings.SURVEX_DATA + self.data['filename'])[0])
+        os.chdir(os.path.split(froox)[0])
-        os.system(settings.CAVERN + " --log " + settings.SURVEX_DATA + self.data['filename'] + ".svx")
+        os.system(settings.CAVERN + " --log " + froox )
         os.chdir(cwd)
-        fin = open(settings.SURVEX_DATA + self.data['filename'] + ".log", "rt",encoding='utf8')
+        fin = open(froog, "rt",encoding='utf8')
         log = fin.read()
         fin.close()
         log = re.sub("(?s).*?(Survey contains)", "\\1", log)
@@ -206,7 +212,7 @@ def svx(request, survex_file):
     svxincludes = re.findall(r'\*include\s+(\S+)(?i)', form.data['code'] or "")
     
     vmap = {'settings': settings,
-            'has_3d': os.path.isfile(settings.SURVEX_DATA + survex_file + ".3d"),
+            'has_3d': os.path.isfile(survexdatasetpath / survex_file / ".3d"),
             'title': survex_file,
             'svxincludes': svxincludes,
             'difflist': difflist,
@@ -218,37 +224,37 @@ def svx(request, survex_file):
     return render_to_response('svxfile.html', vmap)
 
 def svxraw(request, survex_file):
-    svx = open(os.path.join(settings.SURVEX_DATA, survex_file+".svx"), "rt",encoding='utf8')
+    svx = open(os.path.join(survexdatasetpath /  survex_file / ".svx"), "rt",encoding='utf8')
     return HttpResponse(svx, content_type="text")
 
 
 # The cavern running function
 def process(survex_file):
     cwd = os.getcwd()
-    os.chdir(os.path.split(settings.SURVEX_DATA + survex_file)[0])
+    os.chdir(os.path.split(os.fspath(survexdatasetpath /  survex_file))[0])
-    os.system(settings.CAVERN + " --log " + settings.SURVEX_DATA + survex_file + ".svx")
+    os.system(settings.CAVERN + " --log " + survexdatasetpath /  survex_file / ".svx")
     os.chdir(cwd)
 
 
 def threed(request, survex_file):
     process(survex_file)
     try:
-        threed = open(settings.SURVEX_DATA + survex_file + ".3d", "rt",encoding='utf8')
+        threed = open(survexdatasetpath /  survex_file / ".3d", "rt",encoding='utf8')
         return HttpResponse(threed, content_type="model/3d")
     except:
-        log = open(settings.SURVEX_DATA + survex_file + ".log", "rt",encoding='utf8')
+        log = open(survexdatasetpath /  survex_file / ".log", "rt",encoding='utf8')
         return HttpResponse(log, content_type="text")
 
 
 def log(request, survex_file):
     process(survex_file)
-    log = open(settings.SURVEX_DATA + survex_file + ".log", "rt",encoding='utf8')
+    log = open(survexdatasetpath /  survex_file / ".log", "rt",encoding='utf8')
     return HttpResponse(log, content_type="text")
 
 
 def err(request, survex_file):
     process(survex_file)
-    err = open(settings.SURVEX_DATA + survex_file + ".err", "rt",encoding='utf8')
+    err = open(survexdatasetpath /  survex_file / ".err", "rt",encoding='utf8')
     return HttpResponse(err, content_type="text")
 
 
@@ -292,12 +298,36 @@ def identifycavedircontents(gcavedir):
         subsvx.insert(0, primesvx)
     return subdirs, subsvx
 
+def check_cave_registered(survex_cave):
+    '''Checks whether a cave has been properly registered when it is found in the Loser repo
+    This should be called by Databasereset not here in a view
+    Currently Caves are only registered if they are listed in :expoweb: settings.CAVEDESCRIPTIONS
+    so we need to add in any mroe here.
+    '''
+    try:
+        cave = Cave.objects.get(kataster_number=survex_cave)
+        return survex_cave
+    except ObjectDoesNotExist:
+        pass
+    try:
+        cave = Cave.objects.get(unofficial_number=survex_cave)
+        if cave.kataster_number:
+            return cave.kataster_number
+        else:
+            return None
+    except ObjectDoesNotExist:
+        pass
+
 
 # direct local non-database browsing through the svx file repositories
 # perhaps should use the database and have a reload button for it
 # why is caves-1623 HARD CODED here ?! That must be wrong..
 def survexcaveslist(request):
-    cavesdir = os.path.join(settings.SURVEX_DATA, "caves-1623")
+    '''This reads the entire list of caves in the Loser repo directory and produces a complete report.
+    It can find caves which have not yet been properly registered in the system by Databasereset.py because
+    someone may have uploaded the survex files without doing the rest of the integration process.
+    '''
+    cavesdir = survexdatasetpath / "caves-1623"
     #cavesdircontents = { }
     
     onefilecaves = [ ]
@@ -307,21 +337,25 @@ def survexcaveslist(request):
     # first sort the file list
     fnumlist = sorted([ (-int(re.match(r"\d*", f).group(0) or "0"), f)  for f in os.listdir(cavesdir) ])
     
-    print(fnumlist)
+    #print(fnumlist)
     
     # go through the list and identify the contents of each cave directory
     for num, cavedir in fnumlist:
         
         # these have sub dirs /cucc/ /arge/ /old/ but that is no reason to hide them in this webpage
-        # so these are now treated the same as 142 and 113 which also had a /cucc/ sub dir
+        # so these are now treated the same as 142 and 113 which also have a /cucc/ sub dir
         #if cavedir in ["144", "40"]: 
         #    continue
-            
+
+        # This all assumes that the first .svx file has the same name as the cave name, 
+        # which usually but not always true. e.g. caves-1623/78/allkaese.svx not caves-1623/78/78.svx
+        # which is why we now also pass through the cavedir
         gcavedir = os.path.join(cavesdir, cavedir)
         if os.path.isdir(gcavedir) and cavedir[0] != ".":
             subdirs, subsvx = identifycavedircontents(gcavedir)
-            survdirobj = [ ]
             
+            katast = check_cave_registered(cavedir) # should do this only once per database load or it will be slow
+            survdirobj = [ ]
             for lsubsvx in subsvx:
                 survdirobj.append(("caves-1623/"+cavedir+"/"+lsubsvx, lsubsvx))
             
@@ -340,7 +374,7 @@ def survexcaveslist(request):
             
             # multifile caves
             elif len(survdirobj) > 1:
-                multifilecaves.append((survdirobj[0], survdirobj[1:]))
+                multifilecaves.append((survdirobj[0], cavedir, survdirobj[1:]))
             # single file caves
             elif len(survdirobj) == 1:
                 onefilecaves.append(survdirobj[0])
@@ -349,16 +383,18 @@ def survexcaveslist(request):
 
 
 # parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people
-# doesn't use recursion.  just writes it twice
 
 # currently not showing Explorers or Titles. link test from SurvexFile page is "dates and explorers"
 # Should explicity fix the kataster number thing.
 def survexcavesingle(request, survex_cave):
+    print(">>>", survex_cave)
     breload = False
     if breload:
         parsers.survex.ReloadSurvexCave(survex_cave) # does not exit now, needs re-writing to work.
     try:
         cave = Cave.objects.get(kataster_number=survex_cave)
+        for survexdirectory in cave.survexdirectory_set.all:
+            print(">>> >>>", survexdirectory, flush=True)
         return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })
     except ObjectDoesNotExist:
         # can get here if the survex file is in a directory labelled with unofficial number not kataster number.

flatpages/views.py

@@ -1,5 +1,6 @@
 import os
 import re
+from pathlib import Path
 
 from django.shortcuts import render, redirect
 from django.http import HttpResponse, HttpResponseRedirect, Http404
@@ -52,15 +53,16 @@ def flatpage(request, path):
         print(("flat path noinfo", path))
         return HttpResponseRedirect(reverse("auth_login") + '?next=%s' % request.path)
 
+    expowebpath = Path(settings.EXPOWEB)
     if path.endswith("/") or path == "":
         #print(" - FLATPAGES the file: {} ENDSWITH ...".format(path))
 
         try:
-            o = open(os.path.normpath(settings.EXPOWEB + path + "index.html"), "rb")
+            o = open(os.path.normpath(expowebpath / path / "index.html"), "rb")
             path = path + "index.html"
         except IOError:
             try:
-                o = open(os.path.normpath(settings.EXPOWEB + path + "index.htm"), "rb")
+                o = open(os.path.normpath(expowebpath / path / "index.htm"), "rb")
                 path = path + "index.htm"
             except IOError:
                 return render(request, 'pagenotfound.html', {'path': path})
@@ -77,8 +79,8 @@ def flatpage(request, path):
                 path = path.replace("static", settings.MEDIA_ROOT)
                 filetobeopened = os.path.normpath(path)
             else:
-                #print(" - NO    _ROOT: {}  ...".format(settings.EXPOWEB))
+                #print(" - NO    _ROOT: {}  ...".format(expowebpath))
-                filetobeopened = os.path.normpath(settings.EXPOWEB + path)
+                filetobeopened = os.path.normpath(expowebpath / path)
             
             #print(" - FLATPAGES full path : {}  ...".format(filetobeopened))
             o = open(filetobeopened, "rb")

parsers/survex.py

@@ -4,7 +4,7 @@ import re
 import time
 import copy
 
-from unipath import Path
+from pathlib import Path
 from datetime import datetime, timedelta
 from subprocess import call, run
 
@@ -1257,7 +1257,7 @@ def LoadPositions():
         print(" -  Regenerating {} {}.3d  in  {}".format(settings.SURVEXPORT, topdata, settings.SURVEX_DATA))
         call([settings.SURVEXPORT, '--pos', '{}.3d'.format(topdata)], cwd = settings.SURVEX_DATA)
 
-    topdata = settings.SURVEX_DATA.child(settings.SURVEX_TOPNAME)
+    topdata = os.fspath(Path(settings.SURVEX_DATA) / settings.SURVEX_TOPNAME)
     print((' - Generating a list of Pos from %s.svx and then loading...' % (topdata)))
 
     found = 0

settings.py

@@ -27,7 +27,7 @@ print("*  importing troggle/settings.py")
 # read https://docs.djangoproject.com/en/3.0/topics/settings/
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR = os.path.dirname(os.path.dirname(__file__))
+#BASE_DIR = os.path.dirname(os.path.dirname(__file__))
 
 # Django settings for troggle project.
 

templates/svxfilecavelist.html

@@ -41,10 +41,10 @@
 <h2 id="cmult">Caves of multiple files</h2>
 <table>
 <tr><th>Dates and explorers</th><th>Survex files</th></tr>
-{% for primarycavefile, subcavefiles in multifilecaves %}
+{% for primarycavefile, cavedir, subcavefiles in multifilecaves %}
 <tr>
   <td>
-     <a href="{% url "survexcavessingle" primarycavefile.1 %}">{{primarycavefile.1}}</a>
+     <a href="{% url "survexcavessingle" cavedir %}">{{cavedir}}</a> <!-- formerly primarycavefile.1 -->
   </td>
   <td>
     <a href="{% url "svx" primarycavefile.0 %}">{{primarycavefile.1}}</a> -