SUrvexDirectory no longer necessary

core/views/survex.py

@@ -30,13 +30,12 @@ but also displays data on a cave or caves when there is ambiguity
 todo = """- survexcavesingle is not properly producing any result for Homecoming, 1626-359, 2018-dm-07
 even though there are dozens of surveys.
 
-- REFACTOR the very impenetrable code for scanningsubdirectories, replace with modern python pathlib
-
 - filter out the non-public caves  from display UNLESS LOGGED IN
 
 - Never actual uses the object for the survexfile, works entirely from the filepath! Make it check and validate
 
-- the primary survex file in each cave directory should be in a configuration, not buried in the code...
+- the primary survex file in each cave directory should be in a configuration? not buried in the code
+  and implicit in the order of *import statements ?
 
 - Save and re-parse an edited survexfile which already exists in the db, and update
   all its dependencies (work in progress)
@@ -145,7 +144,7 @@ def get_survexfile(filename):
         print("BUG - to be fixed in the survex parser - not critical..")
         print(f"Number of SurvexFile objects found: {len(refs)}")
         for s in refs:
-            print (s.path, s.survexdirectory, s.cave)
+            print (s.path, s.primary, s.cave)
     # print(type(survexfile), filename)
     return survexfile
 
@@ -369,6 +368,7 @@ def svx(request, survex_file):
 
     # collect all the survex blocks which actually have a valid date
     if svxfile:
+        #dirparent = Path(svxfile.primary.path).parent
         has_3d = (Path(SVXPATH) / Path(survex_file + ".3d")).is_file()
         try:
             svxblocks = svxfile.survexblock_set.filter(date__isnull=False).order_by('date')
@@ -405,6 +405,7 @@ def svx(request, survex_file):
         "logmessage": logmessage,
         "form": form,
         "events": events,
+        #"dirparent": dirparent,
     }
 
     if outputtype == "ajax": # used by CodeMirror ajax I think
@@ -649,7 +650,7 @@ def survexcaveslist(request):
 def survexdir(request):
     """report on all the SurvexDirectory objects
     We are trying to find out how mismatches have crept in.
-    and whether the whole SUrvexDirectory class is actually redundant
+    and whether the whole SurvexDirectory class is actually redundant
     as the info it holds is always embedded in the survexFile path directories
     """
     
@@ -665,28 +666,28 @@ def survexdir(request):
             
     survexfiles = SurvexFile.objects.all().order_by("cave")
     for f in survexfiles:
-        f.matchbad = True
+        if f.survexdirectory:
-        if f"{f.path}".startswith(str(f.survexdirectory.path)):
+            f.matchbad = True
-            f.matchbad = False
+            if f"{f.path}".startswith(str(f.survexdirectory.path)):
-        f.primarybad = True
+                f.matchbad = False
-        if f.primary:
+            f.primarybad = True
-            f.pathparent = Path(f.primary.path).parent
+            if f.primary:
-            if str(f.survexdirectory.path) == str(f.pathparent):
+                f.pathparent = Path(f.primary.path).parent
-                f.primarybad = False
+                if str(f.survexdirectory.path) == str(f.pathparent):
+                    f.primarybad = False
         f.pathbad = True
         if Path(settings.SURVEX_DATA, f"{f.path}.svx").is_file():
             f.pathbad = False
     return render(request, "survexdir.html", {"survexdirs": sds, "survexfiles": survexfiles})
 
-def get_directories(cave):
+def get_primaries(cave):
     sds = []
     sfs = cave.survexfile_set.all()
     for sf in sfs:
-        sds.append(sf.survexdirectory)
+        sp = sf.primary # survexfile object
+        sds.append(sp)
     return list(set(sds))
     
-    
-    
 def survexcavesingle(request, cave_shortname):
     """parsing all the survex files of a single cave and showing that it's consistent and can find all
     the files and people.   
@@ -696,7 +697,7 @@ def survexcavesingle(request, cave_shortname):
     if cave_shortname in Gcavelookup:
         cave = Gcavelookup[cave_shortname]
         # print(f"survexcavesingle {cave_shortname=} => {cave=}")
-        cave.sds = get_directories(cave)
+        cave.sds = get_primaries(cave)
         return render(request, "svxcaves.html", {"settings": settings, "caves": [cave]})
     else:
         # not a cave or an ambiguous short name, e.g. "122"
@@ -708,7 +709,7 @@ def survexcavesingle(request, cave_shortname):
         if len(caves) > 0:
             # print(f"many {cave_shortname=} => {caves=}")
             for cave in caves:
-                cave.sds = get_directories(cave)
+                cave.sds = get_primaries(cave)
                 # print(f"many {cave=} => {cave.sds=}")
             return render(request, "svxcaves.html", {"settings": settings, "caves": caves})
         else:

parsers/survex.py

@@ -332,9 +332,10 @@ class LoadingSurvex:
     stacksvxfiles = []
     svxfileslist = []
     svxdirs = {}
+    svxprim = {}
     uniquefile = {} # each key is a survex path, and its value is a list of parent files
     expos = {}
-    survexdict = {}  # each key is a directory, and its value is a list of files
+    #survexdict = {}  # each key is a directory, and its value is a list of files
     lineno = 0
     insp = ""
     callcount = 0
@@ -1175,19 +1176,25 @@ class LoadingSurvex:
         are doing.
         
         """
+        # all = SurvexDirectory.objects.all()
+        # if not all:
+            # sd0 = SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfiley)
+        # sd0 = SurvexDirectory.objects.filter(id=1)[0]
+        
         if not headpath:
             # This is normal for .svx file in the root of the :loser: repo
             # message = f" ! GetSurvexDirectory NO headpath given at {self.currentsurvexfile}"
             # print("\n"+message,file=sys.stderr)
             # stash_data_issue(parser="survex", message=message, url=f"/survexfile/{self.currentsurvexfile}")
-            return self.svxdirs[""], self.currentsurvexfile
+            return self.currentsurvexfile
             
-        if headpath.lower() not in self.svxdirs:
+        if headpath.lower() not in self.svxprim:
             primary = self.currentsurvexfile
-            self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower()
+            # self.svxdirs[headpath.lower()] = sd0 #SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower()
-            self.svxdirs[headpath.lower()].save()
+            # self.svxdirs[headpath.lower()].save()
-            self.survexdict[self.svxdirs[headpath.lower()]] = []  # list of the files in the directory
+            self.svxprim[headpath.lower()] = primary
-        return self.svxdirs[headpath.lower()], self.svxdirs[headpath.lower()].primarysurvexfile
+            #self.survexdict[self.svxdirs[headpath.lower()]] = []  # list of the files in the directory
+        return self.svxprim[headpath.lower()]
 
     def ReportNonCaveIncludes(self, headpath, includelabel, depth):
         """Ignore surface, kataser and gpx *include survex files"""
@@ -1278,17 +1285,17 @@ class LoadingSurvex:
         
         newfile.save()  # until we do this there is no internal id so no foreign key works
         self.currentsurvexfile = newfile
-        newdirectory, primary = self.GetSurvexDirectory(headpath)
+        primary = self.GetSurvexDirectory(headpath)
-        newdirectory.save()
+        #newdirectory.save()
-        newfile.survexdirectory = newdirectory
+        #newfile.survexdirectory = newdirectory
-        self.survexdict[newdirectory].append(newfile)
+        #self.survexdict[newdirectory].append(newfile)
         newfile.primary = primary
 
-        if not newdirectory:
+        # if not newdirectory:
-            message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})"
+            # message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})"
-            print(message)
+            # print(message)
-            print(message, file=sys.stderr)
+            # print(message, file=sys.stderr)
-            stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}")
+            # stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}")
 
         # REPLACE all this IdentifyCave() stuff with GCaveLookup ?
         cave = IdentifyCave(headpath)  # cave already exists in db
@@ -1302,23 +1309,23 @@ class LoadingSurvex:
             newfile.cave = cave
             # print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr)
  
-        if not newfile.survexdirectory:
+        # if not newfile.survexdirectory:
-            message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} "
+            # message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} "
-            print(message)
+            # print(message)
-            print(message, file=sys.stderr)
+            # print(message, file=sys.stderr)
-            stash_data_issue(parser="survex", message=message)
+            # stash_data_issue(parser="survex", message=message)
         if not newfile.primary:
             message = f" ! .primary NOT SET in new SurvexFile {svxid} "
             print(message)
             print(message, file=sys.stderr)
             stash_data_issue(parser="survex", message=message)
         self.currentsurvexfile.save()  # django insists on this although it is already saved !?
-        try:
+        # try:
-            newdirectory.save()
+            # newdirectory.save()
-        except:
+        # except:
-            print(newdirectory, file=sys.stderr)
+            # print(newdirectory, file=sys.stderr)
-            print(newdirectory.primarysurvexfile, file=sys.stderr)
+            # print(newdirectory.primarysurvexfile, file=sys.stderr)
-            raise
+            # raise
 
 
     def ProcessIncludeLine(self, included):
@@ -2311,9 +2318,9 @@ def FindAndLoadSurvex(survexblockroot):
     print("\n  - Loading All Survex Blocks (LinearLoad)", file=sys.stderr)
     svx_load = LoadingSurvex()
 
-    svx_load.survexdict[survexfileroot.survexdirectory] = []
+    #svx_load.survexdict[survexfileroot.survexdirectory] = []
-    svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
+    #svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
-    svx_load.svxdirs[""] = survexfileroot.survexdirectory
+    #svx_load.svxdirs[""] = survexfileroot.survexdirectory
 
     # pr2 = cProfile.Profile()
     # pr2.enable()
@@ -2339,10 +2346,10 @@ def FindAndLoadSurvex(survexblockroot):
     legsnumber = svx_load.legsnumber
     mem1 = get_process_memory()
 
-    print(f"  - Number of SurvexDirectories: {len(svx_load.survexdict):,}")
+    print(f"  - Number of SurvexDirectories: {len(svx_load.svxprim):,}")
     tf = 0
-    for d in svx_load.survexdict:
+    # for d in svx_load.survexdict:
-        tf += len(svx_load.survexdict[d])
+        # tf += len(svx_load.survexdict[d])
     print(f"  - Number of SurvexFiles: {tf:,}")
     print(f"  - Number of Survex legs: {legsnumber:,}")
     svx_load = None
@@ -2402,9 +2409,9 @@ def parse_one_file(fpath): # --------------------------------------in progress--
         print(f"  -  block_dummy now '{block_dummy}' {type(block_dummy)} id={block_dummy.id} f:{block_dummy.survexfile}")
 
         
-        svx_load.survexdict[svxfileroot.survexdirectory] = []
+        # svx_load.survexdict[svxfileroot.survexdirectory] = []
-        svx_load.survexdict[svxfileroot.survexdirectory].append(svxfileroot)
+        # svx_load.survexdict[svxfileroot.survexdirectory].append(svxfileroot)
-        svx_load.svxdirs[""] = svxfileroot.survexdirectory
+        #svx_load.svxdirs[""] = svxfileroot.survexdirectory
         
         # ----------------------------------------------------------------
         svx_load.LinearLoad(block_dummy, svxfileroot.path, fname)
@@ -2418,7 +2425,7 @@ def parse_one_file(fpath): # --------------------------------------in progress--
     global svx_load
     print(f"\n  - Loading One Survex file '{fpath}'", file=sys.stderr)
     svx_load = LoadingSurvex()
-    svx_load.survexdict = {}
+    #svx_load.survexdict = {}
     
     fname = Path(settings.SURVEX_DATA, (fpath + ".svx"))
     

templates/svxcaves.html

@@ -1,4 +1,4 @@
-<!-- svxcavesingle.html - this text visible because this template has been included -->
+<!-- svxcaves.html - this text visible because this template has been included -->
 {% extends "base.html" %}
 {% block title %}List of survex files{% endblock %}
 {% block content %}
@@ -13,8 +13,9 @@
 <p>Wallets: <a href="/cave/scans/{{cave|safe}}">{{cave|safe}}</a>
 </p>
 <p>
-{% for svxdir in cave.sds %} 
+{% for svxprim in cave.sds %} 
- <a href="#T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</a> &nbsp;
+ <a href="#T_{{svxprim}}">{{svxprim}}</a>
+ &nbsp;
     {% empty %}
     <p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps
     because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because <br>
@@ -39,11 +40,11 @@ to go to a form to correct the online data.
     Instructions for filling in this form are in this part 
     <a href="/handbook/survey/caveentry.html"> of the survey handbook</a>.
 </p>
-{% for svxdir in cave.sds %}
+{% for svxprim in cave.sds %}
-    <h3 id="T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</h3>
+    <h3 id="T_{{svxprim.path}}">{{svxprim.path}}</h3>
     <table>
     <tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr>
-    {% for survexfile in svxdir.survexfile_set.all %}
+    {% for survexfile in svxprim.primarysurvex.all %}
     <tr>
       {% if survexfile.exists %}
         <td rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}">
@@ -51,7 +52,7 @@ to go to a form to correct the online data.
         <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}">
       {% endif %}
 
-      {% if survexfile == svxdir.primarysurvexfile %}
+      {% if survexfile == svxprim.primarysurvexfile %}
         <a href="{% url "svx" survexfile.path %}"><b>{% url "svx" survexfile.path %}</b></a>
       {% else %}
         <a href="{% url "svx" survexfile.path %}"><i><small>{% url "svx" survexfile.path %}</small></i></a><!-- would like to extract only the last bit. Some javascript useful ?-->
@@ -98,6 +99,6 @@ to go to a form to correct the online data.
         {% endfor %}  <!-- survexblock -->  
     {% endfor %} <!-- survexfile -->  
     </table>
-{% endfor %} <!-- svxdir --> 
+{% endfor %} <!-- svxprim --> 
 {% endfor %} <!-- caves -->
 {% endblock %}

templates/svxfile.html

@@ -82,7 +82,7 @@ LOGMESSAGES
 </div>
 underground survey length: {{svxlength|floatformat:2}} metres<br />
 parent survex file <a href="/survexfile/{{ survexfile.cave.survex_file }}">{{survexfile.cave.survex_file}}</a> for this cave <br>
-survex directory <a href="/survexfile/{{survexfile.survexdirectory.primarysurvexfile}}">{{survexfile.survexdirectory}}</a> <br /> 
+survex directory <a href="/survexfile/{{survexfile.primary}}.svx">{{survexfile.primary}}</a> <br />
 {% for sb in svxblocks %}
 block <em>{{sb}}</em> has parent block <em>{{sb.parent}}</em><br />
 {% empty %}