moving primary survex file to each survexfile

core/models/caves.py

@@ -515,9 +515,6 @@ def GetCaveLookup():
     global Gcave_count
     Gcave_count = defaultdict(int)  # sets default value to int(0)
 
-    DataIssue.objects.filter(parser="aliases").delete()
-    DataIssue.objects.filter(parser="aliases ok").delete()
-
     for cave in Cave.objects.all():
         key = cave.official_name.lower()
         if key != "" and key != "unamed" and key != "unnamed":

core/models/survex.py

@@ -12,16 +12,17 @@ from django.urls import reverse
 
 
 class SurvexDirectory(models.Model):
-    """This relates a Cave to the primary SurvexFile which is the 'head' of the survex tree for
+    """This relates a survexfile (identified by path) to the primary SurvexFile 
-    that cave. Surely this could just be a property of Cave ? No. Several subdirectories
+    which is the 'head' of the survex tree for that cave. 
-    all relate to the same Cave
+    Surely this could just be a property of Cave ? No. Several subdirectories
+    all relate to the same Cave.
+    
+    But it *could* be a property of SurvexFile
     """
     path = models.CharField(max_length=200) 
-    # cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL) # apparently NEVER USED
     primarysurvexfile = models.ForeignKey(
         "SurvexFile", related_name="primarysurvexfile", blank=True, null=True, on_delete=models.SET_NULL
     )
-    # could also include files in directory but not referenced
 
     class Meta:
         ordering = ("id",)
@@ -37,6 +38,9 @@ class SurvexDirectory(models.Model):
 class SurvexFile(models.Model):
     path = models.CharField(max_length=200)
     survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True, on_delete=models.SET_NULL)
+    primary = models.ForeignKey(
+        "SurvexFile", related_name="primarysurvex", blank=True, null=True, on_delete=models.SET_NULL
+        )
     cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL)
 
     class Meta:

core/views/survex.py

@@ -563,7 +563,11 @@ def get_survexareapath(area):
 def survexcaveslist(request):
     """This reads the entire list of caves in the Loser repo directory and produces a complete report.
     It can find caves which have not yet been properly registered in the system by Databasereset.py because
-    someone may have uploaded the survex files without doing the rest of the integration process.
+    someone may have uploaded the survex files with git without doing the rest of the integration process.
+    
+    But maybe we don't care if someone has done that! 
+    In which case we don't need any of this reading the filesystem, we can generate it all from
+    what is already in the db, and just construct:  onefilecaves,  multifilecaves, subdircaves.
     
     It uses very impenetrable code in identifycavedircontents()
     """
@@ -651,23 +655,27 @@ def survexdir(request):
     
     sds = SurvexDirectory.objects.all() #.order_by("cave")
     for sd in sds:
-        sd.primarybad = True
+        sd.matchbad = True
         if f"{sd.primarysurvexfile}".startswith(str(sd.path)):
-            sd.primarybad = False
+            sd.matchbad = False
-            
-        # sd.cavebad = True
-        # munge = f"caves-{sd.cave}".lower()
-        # if str(sd.path).lower().replace("/","-").startswith(munge):
-            # sd.cavebad = False
            
         sd.pathbad = True
         if Path(settings.SURVEX_DATA, f"{sd.primarysurvexfile}.svx").is_file():
             sd.pathbad = False
             
     survexfiles = SurvexFile.objects.all().order_by("cave")
-    # for f in survexfiles:
+    for f in survexfiles:
-        # if f.cave:
+        f.matchbad = True
-            # print(f, f.cave)
+        if f"{f.path}".startswith(str(f.survexdirectory.path)):
+            f.matchbad = False
+        f.primarybad = True
+        if f.primary:
+            f.pathparent = Path(f.primary.path).parent
+            if str(f.survexdirectory.path) == str(f.pathparent):
+                f.primarybad = False
+        f.pathbad = True
+        if Path(settings.SURVEX_DATA, f"{f.path}.svx").is_file():
+            f.pathbad = False
     return render(request, "survexdir.html", {"survexdirs": sds, "survexfiles": survexfiles})
 
 def get_directories(cave):

parsers/caves.py

@@ -774,6 +774,9 @@ def readcaves():
     DataIssue.objects.filter(parser="areas").delete()
     DataIssue.objects.filter(parser="caves").delete()
     DataIssue.objects.filter(parser="caves ok").delete()
+    DataIssue.objects.filter(parser="aliases").delete()
+    DataIssue.objects.filter(parser="aliases ok").delete()
+
     #DataIssue.objects.filter(parser="entrances").delete()
     #DataIssue.objects.filter(parser="xEntrances").delete()
 

parsers/survex.py

@@ -1174,15 +1174,20 @@ class LoadingSurvex:
         in the directory, where first is defined by the *include ordering. Which is what we
         are doing.
         
-        This does NOT set the current cave id in the SurvexDirectory, that happens later
         """
         if not headpath:
-            return self.svxdirs[""]
+            # This is normal for .svx file in the root of the :loser: repo
+            # message = f" ! GetSurvexDirectory NO headpath given at {self.currentsurvexfile}"
+            # print("\n"+message,file=sys.stderr)
+            # stash_data_issue(parser="survex", message=message, url=f"/survexfile/{self.currentsurvexfile}")
+            return self.svxdirs[""], self.currentsurvexfile
+            
         if headpath.lower() not in self.svxdirs:
-            self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile)
+            primary = self.currentsurvexfile
+            self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower()
             self.svxdirs[headpath.lower()].save()
             self.survexdict[self.svxdirs[headpath.lower()]] = []  # list of the files in the directory
-        return self.svxdirs[headpath.lower()]
+        return self.svxdirs[headpath.lower()], self.svxdirs[headpath.lower()].primarysurvexfile
 
     def ReportNonCaveIncludes(self, headpath, includelabel, depth):
         """Ignore surface, kataser and gpx *include survex files"""
@@ -1238,7 +1243,6 @@ class LoadingSurvex:
         
     def LoadSurvexFile(self, svxid):
         """Creates SurvexFile in the database, and SurvexDirectory if needed
-        with links to 'cave'
         Creates a new current survexfile and valid .survexdirectory
         Inspects the parent folder of the survexfile and uses that to decide if this is 
         a cave we know.
@@ -1274,10 +1278,11 @@ class LoadingSurvex:
         
         newfile.save()  # until we do this there is no internal id so no foreign key works
         self.currentsurvexfile = newfile
-        newdirectory = self.GetSurvexDirectory(headpath)
+        newdirectory, primary = self.GetSurvexDirectory(headpath)
         newdirectory.save()
         newfile.survexdirectory = newdirectory
         self.survexdict[newdirectory].append(newfile)
+        newfile.primary = primary
 
         if not newdirectory:
             message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})"
@@ -1285,6 +1290,7 @@ class LoadingSurvex:
             print(message, file=sys.stderr)
             stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}")
 
+        # REPLACE all this IdentifyCave() stuff with GCaveLookup ?
         cave = IdentifyCave(headpath)  # cave already exists in db
         if not cave:
             # probably a surface survey, or a cave in a new area 
@@ -1293,12 +1299,16 @@ class LoadingSurvex:
         #try again
         cave = IdentifyCave(headpath)  
         if cave:
-            # newdirectory.cave = cave # turns uout to be never used
             newfile.cave = cave
             # print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr)
  
         if not newfile.survexdirectory:
-            message = f" ! SurvexDirectory NOT SET in new SurvexFile {svxid} "
+            message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} "
+            print(message)
+            print(message, file=sys.stderr)
+            stash_data_issue(parser="survex", message=message)
+        if not newfile.primary:
+            message = f" ! .primary NOT SET in new SurvexFile {svxid} "
             print(message)
             print(message, file=sys.stderr)
             stash_data_issue(parser="survex", message=message)
@@ -1310,12 +1320,6 @@ class LoadingSurvex:
             print(newdirectory.primarysurvexfile, file=sys.stderr)
             raise
 
-        if debugprint:
-            print(f" # datastack end LoadSurvexFile:{svxid} 'type':", end="")
-            for dict in self.datastack:
-                print(f"'{dict['type'].upper()}'   ", end="")
-            print("")
-
 
     def ProcessIncludeLine(self, included):
         """As we read the long linear file, we come across lines telling us that the
@@ -2500,7 +2504,6 @@ def MakeSurvexFileRoot():
 
     fileroot = SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
     fileroot.save()
-    #directoryroot = SurvexDirectory(path=settings.SURVEX_DATA, cave=smk[0], primarysurvexfile=fileroot)
     directoryroot = SurvexDirectory(path=settings.SURVEX_DATA, primarysurvexfile=fileroot)
     # MariaDB doesn't like this hack. Complains about non-null cave_id EVEN THOUGH our model file says this is OK:
     # cave = models.ForeignKey('Cave', blank=True, null=True,on_delete=models.SET_NULL)

templates/survexdir.html

@@ -12,20 +12,22 @@
 <tr><th>Dir Path</th><th>Primary svx</th></tr>
 {% for sd in survexdirs %}
     <tr>       
-        <td>{{sd.path}}</td>
+        <td><span{% if sd.matchbad %} style="color:red" {% endif %}>{{sd.path}}</span></td>
-        <td><a href="/survexfile/{{sd.primarysurvexfile}}"><span {% if sd.pathbad %} style="color:red" {% endif %}>{{sd.primarysurvexfile}}.svx</span></a><span {% if sd.primarybad %} style="color:blue"> <b>MISMATCH</b> {% endif %}</span></td>
+        <td><a href="/survexfile/{{sd.primarysurvexfile}}.svx"><span {% if sd.pathbad %} style="color:red" {% endif %}>{{sd.primarysurvexfile}}.svx</span></a></td>
     </tr>
 {% endfor %}
 </table>
 <p>
 <h2>All SurvexFiles</h2>
 <table>
-<tr><th>Cave</th><th>Dir Path</th><th>svx</th></tr>
+<tr><th>Cave</th><th>f.survexdirectory.path</th><th>f.primary.path.parent()</th><th>f.primary</th><th>f.path</th></tr>
 {% for f in survexfiles %}
     <tr>       
 <td>{{f.cave}}</td>
-<td>{{f.survexdirectory.path}}</td>
+<td><span {% if f.matchbad %} style="color:red" {% endif %}>{{f.survexdirectory.path}}</span></td>
-<td><a href="/survexfile/{{f.path}}">{{f.path}}.svx</a></td>
+<td><span {% if f.primarybad %} style="color:red" {% endif %}>{{f.pathparent}}</span></td>
+<td> {{f.primary}}</td>
+<td><span {% if f.pathbad %} style="color:red" {% endif %}><a href="/survexfile/{{f.path}}.svx">{{f.path}}.svx</a></span></td>
     </tr>
 {% endfor %}
 </table>

templates/svxcaves.html

@@ -13,8 +13,8 @@
 <p>Wallets: <a href="/cave/scans/{{cave|safe}}">{{cave|safe}}</a>
 </p>
 <p>
-{% for survexdirectory in cave.sds %} <!-- redo to use cave.survexfile_set.all() each of which has a .survexdirectory-->
+{% for svxdir in cave.sds %} 
- <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a> &nbsp;
+ <a href="#T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</a> &nbsp;
     {% empty %}
     <p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps
     because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because <br>
@@ -39,11 +39,11 @@ to go to a form to correct the online data.
     Instructions for filling in this form are in this part 
     <a href="/handbook/survey/caveentry.html"> of the survey handbook</a>.
 </p>
-{% for survexdirectory in cave.sds %}
+{% for svxdir in cave.sds %}
-    <h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
+    <h3 id="T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</h3>
     <table>
     <tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr>
-    {% for survexfile in survexdirectory.survexfile_set.all %}
+    {% for survexfile in svxdir.survexfile_set.all %}
     <tr>
       {% if survexfile.exists %}
         <td rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}">
@@ -51,7 +51,7 @@ to go to a form to correct the online data.
         <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}">
       {% endif %}
 
-      {% if survexfile == survexdirectory.primarysurvexfile %}
+      {% if survexfile == svxdir.primarysurvexfile %}
         <a href="{% url "svx" survexfile.path %}"><b>{% url "svx" survexfile.path %}</b></a>
       {% else %}
         <a href="{% url "svx" survexfile.path %}"><i><small>{% url "svx" survexfile.path %}</small></i></a><!-- would like to extract only the last bit. Some javascript useful ?-->
@@ -98,6 +98,6 @@ to go to a form to correct the online data.
         {% endfor %}  <!-- survexblock -->  
     {% endfor %} <!-- survexfile -->  
     </table>
-{% endfor %} <!-- survexdirectory --> 
+{% endfor %} <!-- svxdir --> 
 {% endfor %} <!-- caves -->
 {% endblock %}