diff --git a/core/models/caves.py b/core/models/caves.py
index e8cfeea..28b9d4c 100644
--- a/core/models/caves.py
+++ b/core/models/caves.py
@@ -515,9 +515,6 @@ def GetCaveLookup():
global Gcave_count
Gcave_count = defaultdict(int) # sets default value to int(0)
- DataIssue.objects.filter(parser="aliases").delete()
- DataIssue.objects.filter(parser="aliases ok").delete()
-
for cave in Cave.objects.all():
key = cave.official_name.lower()
if key != "" and key != "unamed" and key != "unnamed":
diff --git a/core/models/survex.py b/core/models/survex.py
index 835d31d..2ca7412 100644
--- a/core/models/survex.py
+++ b/core/models/survex.py
@@ -12,16 +12,17 @@ from django.urls import reverse
class SurvexDirectory(models.Model):
- """This relates a Cave to the primary SurvexFile which is the 'head' of the survex tree for
- that cave. Surely this could just be a property of Cave ? No. Several subdirectories
- all relate to the same Cave
+ """This relates a survexfile (identified by path) to the primary SurvexFile
+ which is the 'head' of the survex tree for that cave.
+ Surely this could just be a property of Cave ? No. Several subdirectories
+ all relate to the same Cave.
+
+ But it *could* be a property of SurvexFile
"""
path = models.CharField(max_length=200)
- # cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL) # apparently NEVER USED
primarysurvexfile = models.ForeignKey(
"SurvexFile", related_name="primarysurvexfile", blank=True, null=True, on_delete=models.SET_NULL
)
- # could also include files in directory but not referenced
class Meta:
ordering = ("id",)
@@ -37,6 +38,9 @@ class SurvexDirectory(models.Model):
class SurvexFile(models.Model):
path = models.CharField(max_length=200)
survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True, on_delete=models.SET_NULL)
+ primary = models.ForeignKey(
+ "SurvexFile", related_name="primarysurvex", blank=True, null=True, on_delete=models.SET_NULL
+ )
cave = models.ForeignKey("Cave", blank=True, null=True, on_delete=models.SET_NULL)
class Meta:
diff --git a/core/views/survex.py b/core/views/survex.py
index 3ecd523..10ebbd0 100644
--- a/core/views/survex.py
+++ b/core/views/survex.py
@@ -563,7 +563,11 @@ def get_survexareapath(area):
def survexcaveslist(request):
"""This reads the entire list of caves in the Loser repo directory and produces a complete report.
It can find caves which have not yet been properly registered in the system by Databasereset.py because
- someone may have uploaded the survex files without doing the rest of the integration process.
+ someone may have uploaded the survex files with git without doing the rest of the integration process.
+
+ But maybe we don't care if someone has done that!
+ In which case we don't need any of this reading the filesystem, we can generate it all from
+ what is already in the db, and just construct: onefilecaves, multifilecaves, subdircaves.
It uses very impenetrable code in identifycavedircontents()
"""
@@ -651,23 +655,27 @@ def survexdir(request):
sds = SurvexDirectory.objects.all() #.order_by("cave")
for sd in sds:
- sd.primarybad = True
+ sd.matchbad = True
if f"{sd.primarysurvexfile}".startswith(str(sd.path)):
- sd.primarybad = False
-
- # sd.cavebad = True
- # munge = f"caves-{sd.cave}".lower()
- # if str(sd.path).lower().replace("/","-").startswith(munge):
- # sd.cavebad = False
+ sd.matchbad = False
sd.pathbad = True
if Path(settings.SURVEX_DATA, f"{sd.primarysurvexfile}.svx").is_file():
sd.pathbad = False
survexfiles = SurvexFile.objects.all().order_by("cave")
- # for f in survexfiles:
- # if f.cave:
- # print(f, f.cave)
+ for f in survexfiles:
+ f.matchbad = True
+ if f"{f.path}".startswith(str(f.survexdirectory.path)):
+ f.matchbad = False
+ f.primarybad = True
+ if f.primary:
+ f.pathparent = Path(f.primary.path).parent
+ if str(f.survexdirectory.path) == str(f.pathparent):
+ f.primarybad = False
+ f.pathbad = True
+ if Path(settings.SURVEX_DATA, f"{f.path}.svx").is_file():
+ f.pathbad = False
return render(request, "survexdir.html", {"survexdirs": sds, "survexfiles": survexfiles})
def get_directories(cave):
diff --git a/parsers/caves.py b/parsers/caves.py
index 4e720db..c16a577 100644
--- a/parsers/caves.py
+++ b/parsers/caves.py
@@ -774,6 +774,9 @@ def readcaves():
DataIssue.objects.filter(parser="areas").delete()
DataIssue.objects.filter(parser="caves").delete()
DataIssue.objects.filter(parser="caves ok").delete()
+ DataIssue.objects.filter(parser="aliases").delete()
+ DataIssue.objects.filter(parser="aliases ok").delete()
+
#DataIssue.objects.filter(parser="entrances").delete()
#DataIssue.objects.filter(parser="xEntrances").delete()
diff --git a/parsers/survex.py b/parsers/survex.py
index a5a9959..ba2622c 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -1174,15 +1174,20 @@ class LoadingSurvex:
in the directory, where first is defined by the *include ordering. Which is what we
are doing.
- This does NOT set the current cave id in the SurvexDirectory, that happens later
"""
if not headpath:
- return self.svxdirs[""]
+ # This is normal for .svx file in the root of the :loser: repo
+ # message = f" ! GetSurvexDirectory NO headpath given at {self.currentsurvexfile}"
+ # print("\n"+message,file=sys.stderr)
+ # stash_data_issue(parser="survex", message=message, url=f"/survexfile/{self.currentsurvexfile}")
+ return self.svxdirs[""], self.currentsurvexfile
+
if headpath.lower() not in self.svxdirs:
- self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile)
+ primary = self.currentsurvexfile
+ self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower()
self.svxdirs[headpath.lower()].save()
self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory
- return self.svxdirs[headpath.lower()]
+ return self.svxdirs[headpath.lower()], self.svxdirs[headpath.lower()].primarysurvexfile
def ReportNonCaveIncludes(self, headpath, includelabel, depth):
"""Ignore surface, kataser and gpx *include survex files"""
@@ -1238,7 +1243,6 @@ class LoadingSurvex:
def LoadSurvexFile(self, svxid):
"""Creates SurvexFile in the database, and SurvexDirectory if needed
- with links to 'cave'
Creates a new current survexfile and valid .survexdirectory
Inspects the parent folder of the survexfile and uses that to decide if this is
a cave we know.
@@ -1274,10 +1278,11 @@ class LoadingSurvex:
newfile.save() # until we do this there is no internal id so no foreign key works
self.currentsurvexfile = newfile
- newdirectory = self.GetSurvexDirectory(headpath)
+ newdirectory, primary = self.GetSurvexDirectory(headpath)
newdirectory.save()
newfile.survexdirectory = newdirectory
self.survexdict[newdirectory].append(newfile)
+ newfile.primary = primary
if not newdirectory:
message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})"
@@ -1285,6 +1290,7 @@ class LoadingSurvex:
print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}")
+ # REPLACE all this IdentifyCave() stuff with GCaveLookup ?
cave = IdentifyCave(headpath) # cave already exists in db
if not cave:
# probably a surface survey, or a cave in a new area
@@ -1293,12 +1299,16 @@ class LoadingSurvex:
#try again
cave = IdentifyCave(headpath)
if cave:
- # newdirectory.cave = cave # turns uout to be never used
newfile.cave = cave
# print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr)
if not newfile.survexdirectory:
- message = f" ! SurvexDirectory NOT SET in new SurvexFile {svxid} "
+ message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} "
+ print(message)
+ print(message, file=sys.stderr)
+ stash_data_issue(parser="survex", message=message)
+ if not newfile.primary:
+ message = f" ! .primary NOT SET in new SurvexFile {svxid} "
print(message)
print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message)
@@ -1310,12 +1320,6 @@ class LoadingSurvex:
print(newdirectory.primarysurvexfile, file=sys.stderr)
raise
- if debugprint:
- print(f" # datastack end LoadSurvexFile:{svxid} 'type':", end="")
- for dict in self.datastack:
- print(f"'{dict['type'].upper()}' ", end="")
- print("")
-
def ProcessIncludeLine(self, included):
"""As we read the long linear file, we come across lines telling us that the
@@ -2500,7 +2504,6 @@ def MakeSurvexFileRoot():
fileroot = SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
fileroot.save()
- #directoryroot = SurvexDirectory(path=settings.SURVEX_DATA, cave=smk[0], primarysurvexfile=fileroot)
directoryroot = SurvexDirectory(path=settings.SURVEX_DATA, primarysurvexfile=fileroot)
# MariaDB doesn't like this hack. Complains about non-null cave_id EVEN THOUGH our model file says this is OK:
# cave = models.ForeignKey('Cave', blank=True, null=True,on_delete=models.SET_NULL)
@@ -2525,7 +2528,7 @@ def MakeFileRoot(fn):
try:
fileroot.survexdirectory = SurvexDirectory.objects.get(id=1) # default
except:
- fileroot.survexdirectory = None
+ fileroot.survexdirectory = None
# if cave:
diff --git a/templates/survexdir.html b/templates/survexdir.html
index f0cbf11..e2e2747 100644
--- a/templates/survexdir.html
+++ b/templates/survexdir.html
@@ -12,20 +12,22 @@
<tr><th>Dir Path</th><th>Primary svx</th></tr>
{% for sd in survexdirs %}
<tr>
- <td>{{sd.path}}</td>
- <td><a href="/survexfile/{{sd.primarysurvexfile}}"><span {% if sd.pathbad %} style="color:red" {% endif %}>{{sd.primarysurvexfile}}.svx</span></a><span {% if sd.primarybad %} style="color:blue"> <b>MISMATCH</b> {% endif %}</span></td>
+ <td><span{% if sd.matchbad %} style="color:red" {% endif %}>{{sd.path}}</span></td>
+ <td><a href="/survexfile/{{sd.primarysurvexfile}}.svx"><span {% if sd.pathbad %} style="color:red" {% endif %}>{{sd.primarysurvexfile}}.svx</span></a></td>
</tr>
{% endfor %}
</table>
<p>
<h2>All SurvexFiles</h2>
<table>
-<tr><th>Cave</th><th>Dir Path</th><th>svx</th></tr>
+<tr><th>Cave</th><th>f.survexdirectory.path</th><th>f.primary.path.parent()</th><th>f.primary</th><th>f.path</th></tr>
{% for f in survexfiles %}
<tr>
<td>{{f.cave}}</td>
-<td>{{f.survexdirectory.path}}</td>
-<td><a href="/survexfile/{{f.path}}">{{f.path}}.svx</a></td>
+<td><span {% if f.matchbad %} style="color:red" {% endif %}>{{f.survexdirectory.path}}</span></td>
+<td><span {% if f.primarybad %} style="color:red" {% endif %}>{{f.pathparent}}</span></td>
+<td> {{f.primary}}</td>
+<td><span {% if f.pathbad %} style="color:red" {% endif %}><a href="/survexfile/{{f.path}}.svx">{{f.path}}.svx</a></span></td>
</tr>
{% endfor %}
</table>
diff --git a/templates/svxcaves.html b/templates/svxcaves.html
index 01548d7..2f3506d 100644
--- a/templates/svxcaves.html
+++ b/templates/svxcaves.html
@@ -13,8 +13,8 @@
<p>Wallets: <a href="/cave/scans/{{cave|safe}}">{{cave|safe}}</a>
</p>
<p>
-{% for survexdirectory in cave.sds %} <!-- redo to use cave.survexfile_set.all() each of which has a .survexdirectory-->
- <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>
+{% for svxdir in cave.sds %}
+ <a href="#T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</a>
{% empty %}
<p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps
because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because <br>
@@ -39,11 +39,11 @@ to go to a form to correct the online data.
Instructions for filling in this form are in this part
<a href="/handbook/survey/caveentry.html"> of the survey handbook</a>.
</p>
-{% for survexdirectory in cave.sds %}
- <h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
+{% for svxdir in cave.sds %}
+ <h3 id="T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</h3>
<table>
<tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr>
- {% for survexfile in survexdirectory.survexfile_set.all %}
+ {% for survexfile in svxdir.survexfile_set.all %}
<tr>
{% if survexfile.exists %}
<td rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}">
@@ -51,7 +51,7 @@ to go to a form to correct the online data.
<td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}">
{% endif %}
- {% if survexfile == survexdirectory.primarysurvexfile %}
+ {% if survexfile == svxdir.primarysurvexfile %}
<a href="{% url "svx" survexfile.path %}"><b>{% url "svx" survexfile.path %}</b></a>
{% else %}
<a href="{% url "svx" survexfile.path %}"><i><small>{% url "svx" survexfile.path %}</small></i></a><!-- would like to extract only the last bit. Some javascript useful ?-->
@@ -98,6 +98,6 @@ to go to a form to correct the online data.
{% endfor %} <!-- survexblock -->
{% endfor %} <!-- survexfile -->
</table>
-{% endfor %} <!-- survexdirectory -->
+{% endfor %} <!-- svxdir -->
{% endfor %} <!-- caves -->
{% endblock %}