From 1ddb8248dff14b819c0037b9f59d68aedeba9829 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@gmail.com>
Date: Wed, 6 Sep 2023 21:38:45 +0300
Subject: [PATCH] SUrvexDirectory no longer necessary
---
core/views/survex.py | 39 ++++++++++----------
parsers/survex.py | 81 ++++++++++++++++++++++-------------------
templates/svxcaves.html | 17 +++++----
templates/svxfile.html | 2 +-
4 files changed, 74 insertions(+), 65 deletions(-)
diff --git a/core/views/survex.py b/core/views/survex.py
index 10ebbd0..d916e56 100644
--- a/core/views/survex.py
+++ b/core/views/survex.py
@@ -30,13 +30,12 @@ but also displays data on a cave or caves when there is ambiguity
todo = """- survexcavesingle is not properly producing any result for Homecoming, 1626-359, 2018-dm-07
even though there are dozens of surveys.
-- REFACTOR the very impenetrable code for scanningsubdirectories, replace with modern python pathlib
-
- filter out the non-public caves from display UNLESS LOGGED IN
- Never actual uses the object for the survexfile, works entirely from the filepath! Make it check and validate
-- the primary survex file in each cave directory should be in a configuration, not buried in the code...
+- the primary survex file in each cave directory should be in a configuration? not buried in the code
+ and implicit in the order of *import statements ?
- Save and re-parse an edited survexfile which already exists in the db, and update
all its dependencies (work in progress)
@@ -145,7 +144,7 @@ def get_survexfile(filename):
print("BUG - to be fixed in the survex parser - not critical..")
print(f"Number of SurvexFile objects found: {len(refs)}")
for s in refs:
- print (s.path, s.survexdirectory, s.cave)
+ print (s.path, s.primary, s.cave)
# print(type(survexfile), filename)
return survexfile
@@ -369,6 +368,7 @@ def svx(request, survex_file):
# collect all the survex blocks which actually have a valid date
if svxfile:
+ #dirparent = Path(svxfile.primary.path).parent
has_3d = (Path(SVXPATH) / Path(survex_file + ".3d")).is_file()
try:
svxblocks = svxfile.survexblock_set.filter(date__isnull=False).order_by('date')
@@ -405,6 +405,7 @@ def svx(request, survex_file):
"logmessage": logmessage,
"form": form,
"events": events,
+ #"dirparent": dirparent,
}
if outputtype == "ajax": # used by CodeMirror ajax I think
@@ -649,7 +650,7 @@ def survexcaveslist(request):
def survexdir(request):
"""report on all the SurvexDirectory objects
We are trying to find out how mismatches have crept in.
- and whether the whole SUrvexDirectory class is actually redundant
+ and whether the whole SurvexDirectory class is actually redundant
as the info it holds is always embedded in the survexFile path directories
"""
@@ -665,28 +666,28 @@ def survexdir(request):
survexfiles = SurvexFile.objects.all().order_by("cave")
for f in survexfiles:
- f.matchbad = True
- if f"{f.path}".startswith(str(f.survexdirectory.path)):
- f.matchbad = False
- f.primarybad = True
- if f.primary:
- f.pathparent = Path(f.primary.path).parent
- if str(f.survexdirectory.path) == str(f.pathparent):
- f.primarybad = False
+ if f.survexdirectory:
+ f.matchbad = True
+ if f"{f.path}".startswith(str(f.survexdirectory.path)):
+ f.matchbad = False
+ f.primarybad = True
+ if f.primary:
+ f.pathparent = Path(f.primary.path).parent
+ if str(f.survexdirectory.path) == str(f.pathparent):
+ f.primarybad = False
f.pathbad = True
if Path(settings.SURVEX_DATA, f"{f.path}.svx").is_file():
f.pathbad = False
return render(request, "survexdir.html", {"survexdirs": sds, "survexfiles": survexfiles})
-def get_directories(cave):
+def get_primaries(cave):
sds = []
sfs = cave.survexfile_set.all()
for sf in sfs:
- sds.append(sf.survexdirectory)
+ sp = sf.primary # survexfile object
+ sds.append(sp)
return list(set(sds))
-
-
def survexcavesingle(request, cave_shortname):
"""parsing all the survex files of a single cave and showing that it's consistent and can find all
the files and people.
@@ -696,7 +697,7 @@ def survexcavesingle(request, cave_shortname):
if cave_shortname in Gcavelookup:
cave = Gcavelookup[cave_shortname]
# print(f"survexcavesingle {cave_shortname=} => {cave=}")
- cave.sds = get_directories(cave)
+ cave.sds = get_primaries(cave)
return render(request, "svxcaves.html", {"settings": settings, "caves": [cave]})
else:
# not a cave or an ambiguous short name, e.g. "122"
@@ -708,7 +709,7 @@ def survexcavesingle(request, cave_shortname):
if len(caves) > 0:
# print(f"many {cave_shortname=} => {caves=}")
for cave in caves:
- cave.sds = get_directories(cave)
+ cave.sds = get_primaries(cave)
# print(f"many {cave=} => {cave.sds=}")
return render(request, "svxcaves.html", {"settings": settings, "caves": caves})
else:
diff --git a/parsers/survex.py b/parsers/survex.py
index ba2622c..e802a79 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -332,9 +332,10 @@ class LoadingSurvex:
stacksvxfiles = []
svxfileslist = []
svxdirs = {}
+ svxprim = {}
uniquefile = {} # each key is a survex path, and its value is a list of parent files
expos = {}
- survexdict = {} # each key is a directory, and its value is a list of files
+ #survexdict = {} # each key is a directory, and its value is a list of files
lineno = 0
insp = ""
callcount = 0
@@ -1175,19 +1176,25 @@ class LoadingSurvex:
are doing.
"""
+ # all = SurvexDirectory.objects.all()
+ # if not all:
+ # sd0 = SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfiley)
+ # sd0 = SurvexDirectory.objects.filter(id=1)[0]
+
if not headpath:
# This is normal for .svx file in the root of the :loser: repo
# message = f" ! GetSurvexDirectory NO headpath given at {self.currentsurvexfile}"
# print("\n"+message,file=sys.stderr)
# stash_data_issue(parser="survex", message=message, url=f"/survexfile/{self.currentsurvexfile}")
- return self.svxdirs[""], self.currentsurvexfile
+ return self.currentsurvexfile
- if headpath.lower() not in self.svxdirs:
+ if headpath.lower() not in self.svxprim:
primary = self.currentsurvexfile
- self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower()
- self.svxdirs[headpath.lower()].save()
- self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory
- return self.svxdirs[headpath.lower()], self.svxdirs[headpath.lower()].primarysurvexfile
+ # self.svxdirs[headpath.lower()] = sd0 #SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower()
+ # self.svxdirs[headpath.lower()].save()
+ self.svxprim[headpath.lower()] = primary
+ #self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory
+ return self.svxprim[headpath.lower()]
def ReportNonCaveIncludes(self, headpath, includelabel, depth):
"""Ignore surface, kataser and gpx *include survex files"""
@@ -1278,17 +1285,17 @@ class LoadingSurvex:
newfile.save() # until we do this there is no internal id so no foreign key works
self.currentsurvexfile = newfile
- newdirectory, primary = self.GetSurvexDirectory(headpath)
- newdirectory.save()
- newfile.survexdirectory = newdirectory
- self.survexdict[newdirectory].append(newfile)
+ primary = self.GetSurvexDirectory(headpath)
+ #newdirectory.save()
+ #newfile.survexdirectory = newdirectory
+ #self.survexdict[newdirectory].append(newfile)
newfile.primary = primary
- if not newdirectory:
- message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})"
- print(message)
- print(message, file=sys.stderr)
- stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}")
+ # if not newdirectory:
+ # message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})"
+ # print(message)
+ # print(message, file=sys.stderr)
+ # stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}")
# REPLACE all this IdentifyCave() stuff with GCaveLookup ?
cave = IdentifyCave(headpath) # cave already exists in db
@@ -1302,23 +1309,23 @@ class LoadingSurvex:
newfile.cave = cave
# print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr)
- if not newfile.survexdirectory:
- message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} "
- print(message)
- print(message, file=sys.stderr)
- stash_data_issue(parser="survex", message=message)
+ # if not newfile.survexdirectory:
+ # message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} "
+ # print(message)
+ # print(message, file=sys.stderr)
+ # stash_data_issue(parser="survex", message=message)
if not newfile.primary:
message = f" ! .primary NOT SET in new SurvexFile {svxid} "
print(message)
print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message)
self.currentsurvexfile.save() # django insists on this although it is already saved !?
- try:
- newdirectory.save()
- except:
- print(newdirectory, file=sys.stderr)
- print(newdirectory.primarysurvexfile, file=sys.stderr)
- raise
+ # try:
+ # newdirectory.save()
+ # except:
+ # print(newdirectory, file=sys.stderr)
+ # print(newdirectory.primarysurvexfile, file=sys.stderr)
+ # raise
def ProcessIncludeLine(self, included):
@@ -2311,9 +2318,9 @@ def FindAndLoadSurvex(survexblockroot):
print("\n - Loading All Survex Blocks (LinearLoad)", file=sys.stderr)
svx_load = LoadingSurvex()
- svx_load.survexdict[survexfileroot.survexdirectory] = []
- svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
- svx_load.svxdirs[""] = survexfileroot.survexdirectory
+ #svx_load.survexdict[survexfileroot.survexdirectory] = []
+ #svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
+ #svx_load.svxdirs[""] = survexfileroot.survexdirectory
# pr2 = cProfile.Profile()
# pr2.enable()
@@ -2339,10 +2346,10 @@ def FindAndLoadSurvex(survexblockroot):
legsnumber = svx_load.legsnumber
mem1 = get_process_memory()
- print(f" - Number of SurvexDirectories: {len(svx_load.survexdict):,}")
+ print(f" - Number of SurvexDirectories: {len(svx_load.svxprim):,}")
tf = 0
- for d in svx_load.survexdict:
- tf += len(svx_load.survexdict[d])
+ # for d in svx_load.survexdict:
+ # tf += len(svx_load.survexdict[d])
print(f" - Number of SurvexFiles: {tf:,}")
print(f" - Number of Survex legs: {legsnumber:,}")
svx_load = None
@@ -2402,9 +2409,9 @@ def parse_one_file(fpath): # --------------------------------------in progress--
print(f" - block_dummy now '{block_dummy}' {type(block_dummy)} id={block_dummy.id} f:{block_dummy.survexfile}")
- svx_load.survexdict[svxfileroot.survexdirectory] = []
- svx_load.survexdict[svxfileroot.survexdirectory].append(svxfileroot)
- svx_load.svxdirs[""] = svxfileroot.survexdirectory
+ # svx_load.survexdict[svxfileroot.survexdirectory] = []
+ # svx_load.survexdict[svxfileroot.survexdirectory].append(svxfileroot)
+ #svx_load.svxdirs[""] = svxfileroot.survexdirectory
# ----------------------------------------------------------------
svx_load.LinearLoad(block_dummy, svxfileroot.path, fname)
@@ -2418,7 +2425,7 @@ def parse_one_file(fpath): # --------------------------------------in progress--
global svx_load
print(f"\n - Loading One Survex file '{fpath}'", file=sys.stderr)
svx_load = LoadingSurvex()
- svx_load.survexdict = {}
+ #svx_load.survexdict = {}
fname = Path(settings.SURVEX_DATA, (fpath + ".svx"))
diff --git a/templates/svxcaves.html b/templates/svxcaves.html
index 2f3506d..1bea274 100644
--- a/templates/svxcaves.html
+++ b/templates/svxcaves.html
@@ -1,4 +1,4 @@
-<!-- svxcavesingle.html - this text visible because this template has been included -->
+<!-- svxcaves.html - this text visible because this template has been included -->
{% extends "base.html" %}
{% block title %}List of survex files{% endblock %}
{% block content %}
@@ -13,8 +13,9 @@
<p>Wallets: <a href="/cave/scans/{{cave|safe}}">{{cave|safe}}</a>
</p>
<p>
-{% for svxdir in cave.sds %}
- <a href="#T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</a>
+{% for svxprim in cave.sds %}
+ <a href="#T_{{svxprim}}">{{svxprim}}</a>
+
{% empty %}
<p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps
because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because <br>
@@ -39,11 +40,11 @@ to go to a form to correct the online data.
Instructions for filling in this form are in this part
<a href="/handbook/survey/caveentry.html"> of the survey handbook</a>.
</p>
-{% for svxdir in cave.sds %}
- <h3 id="T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</h3>
+{% for svxprim in cave.sds %}
+ <h3 id="T_{{svxprim.path}}">{{svxprim.path}}</h3>
<table>
<tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr>
- {% for survexfile in svxdir.survexfile_set.all %}
+ {% for survexfile in svxprim.primarysurvex.all %}
<tr>
{% if survexfile.exists %}
<td rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}">
@@ -51,7 +52,7 @@ to go to a form to correct the online data.
<td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}">
{% endif %}
- {% if survexfile == svxdir.primarysurvexfile %}
+ {% if survexfile == svxprim.primarysurvexfile %}
<a href="{% url "svx" survexfile.path %}"><b>{% url "svx" survexfile.path %}</b></a>
{% else %}
<a href="{% url "svx" survexfile.path %}"><i><small>{% url "svx" survexfile.path %}</small></i></a><!-- would like to extract only the last bit. Some javascript useful ?-->
@@ -98,6 +99,6 @@ to go to a form to correct the online data.
{% endfor %} <!-- survexblock -->
{% endfor %} <!-- survexfile -->
</table>
-{% endfor %} <!-- svxdir -->
+{% endfor %} <!-- svxprim -->
{% endfor %} <!-- caves -->
{% endblock %}
diff --git a/templates/svxfile.html b/templates/svxfile.html
index db420cc..7d6dc14 100644
--- a/templates/svxfile.html
+++ b/templates/svxfile.html
@@ -82,7 +82,7 @@ LOGMESSAGES
</div>
underground survey length: {{svxlength|floatformat:2}} metres<br />
parent survex file <a href="/survexfile/{{ survexfile.cave.survex_file }}">{{survexfile.cave.survex_file}}</a> for this cave <br>
-survex directory <a href="/survexfile/{{survexfile.survexdirectory.primarysurvexfile}}">{{survexfile.survexdirectory}}</a> <br />
+survex directory <a href="/survexfile/{{survexfile.primary}}.svx">{{survexfile.primary}}</a> <br />
{% for sb in svxblocks %}
block <em>{{sb}}</em> has parent block <em>{{sb.parent}}</em><br />
{% empty %}