expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-25 08:41:51 +00:00
Code Issues 19 Projects 3 Activity

SUrvexDirectory no longer necessary

Browse Source
This commit is contained in:
Philip Sargent 2023-09-06 21:38:45 +03:00
parent 0dc0e27519
commit 1ddb8248df
4 changed files with 74 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
core/views/survex.py
View File

@ -30,13 +30,12 @@ but also displays data on a cave or caves when there is ambiguity
todo = """- survexcavesingle is not properly producing any result for Homecoming, 1626-359, 2018-dm-07 todo = """- survexcavesingle is not properly producing any result for Homecoming, 1626-359, 2018-dm-07
even though there are dozens of surveys. even though there are dozens of surveys.
- REFACTOR the very impenetrable code for scanningsubdirectories, replace with modern python pathlib
- filter out the non-public caves from display UNLESS LOGGED IN - filter out the non-public caves from display UNLESS LOGGED IN
- Never actual uses the object for the survexfile, works entirely from the filepath! Make it check and validate - Never actual uses the object for the survexfile, works entirely from the filepath! Make it check and validate
- the primary survex file in each cave directory should be in a configuration, not buried in the code... - the primary survex file in each cave directory should be in a configuration? not buried in the code
and implicit in the order of *import statements ?
- Save and re-parse an edited survexfile which already exists in the db, and update - Save and re-parse an edited survexfile which already exists in the db, and update
all its dependencies (work in progress) all its dependencies (work in progress)
@ -145,7 +144,7 @@ def get_survexfile(filename):
print("BUG - to be fixed in the survex parser - not critical..") print("BUG - to be fixed in the survex parser - not critical..")
print(f"Number of SurvexFile objects found: {len(refs)}") print(f"Number of SurvexFile objects found: {len(refs)}")
for s in refs: for s in refs:
print (s.path, s.survexdirectory, s.cave) print (s.path, s.primary, s.cave)
# print(type(survexfile), filename) # print(type(survexfile), filename)
return survexfile return survexfile
@ -369,6 +368,7 @@ def svx(request, survex_file):
# collect all the survex blocks which actually have a valid date # collect all the survex blocks which actually have a valid date
if svxfile: if svxfile:
#dirparent = Path(svxfile.primary.path).parent
has_3d = (Path(SVXPATH) / Path(survex_file + ".3d")).is_file() has_3d = (Path(SVXPATH) / Path(survex_file + ".3d")).is_file()
try: try:
svxblocks = svxfile.survexblock_set.filter(date__isnull=False).order_by('date') svxblocks = svxfile.survexblock_set.filter(date__isnull=False).order_by('date')
@ -405,6 +405,7 @@ def svx(request, survex_file):
"logmessage": logmessage, "logmessage": logmessage,
"form": form, "form": form,
"events": events, "events": events,
#"dirparent": dirparent,
} }
if outputtype == "ajax": # used by CodeMirror ajax I think if outputtype == "ajax": # used by CodeMirror ajax I think
@ -649,7 +650,7 @@ def survexcaveslist(request):
def survexdir(request): def survexdir(request):
"""report on all the SurvexDirectory objects """report on all the SurvexDirectory objects
We are trying to find out how mismatches have crept in. We are trying to find out how mismatches have crept in.
and whether the whole SUrvexDirectory class is actually redundant and whether the whole SurvexDirectory class is actually redundant
as the info it holds is always embedded in the survexFile path directories as the info it holds is always embedded in the survexFile path directories
""" """
@ -665,28 +666,28 @@ def survexdir(request):
survexfiles = SurvexFile.objects.all().order_by("cave") survexfiles = SurvexFile.objects.all().order_by("cave")
for f in survexfiles: for f in survexfiles:
f.matchbad = True if f.survexdirectory:
if f"{f.path}".startswith(str(f.survexdirectory.path)): f.matchbad = True
f.matchbad = False if f"{f.path}".startswith(str(f.survexdirectory.path)):
f.primarybad = True f.matchbad = False
if f.primary: f.primarybad = True
f.pathparent = Path(f.primary.path).parent if f.primary:
if str(f.survexdirectory.path) == str(f.pathparent): f.pathparent = Path(f.primary.path).parent
f.primarybad = False if str(f.survexdirectory.path) == str(f.pathparent):
f.primarybad = False
f.pathbad = True f.pathbad = True
if Path(settings.SURVEX_DATA, f"{f.path}.svx").is_file(): if Path(settings.SURVEX_DATA, f"{f.path}.svx").is_file():
f.pathbad = False f.pathbad = False
return render(request, "survexdir.html", {"survexdirs": sds, "survexfiles": survexfiles}) return render(request, "survexdir.html", {"survexdirs": sds, "survexfiles": survexfiles})
def get_directories(cave): def get_primaries(cave):
sds = [] sds = []
sfs = cave.survexfile_set.all() sfs = cave.survexfile_set.all()
for sf in sfs: for sf in sfs:
sds.append(sf.survexdirectory) sp = sf.primary # survexfile object
sds.append(sp)
return list(set(sds)) return list(set(sds))
def survexcavesingle(request, cave_shortname): def survexcavesingle(request, cave_shortname):
"""parsing all the survex files of a single cave and showing that it's consistent and can find all """parsing all the survex files of a single cave and showing that it's consistent and can find all
the files and people. the files and people.
@ -696,7 +697,7 @@ def survexcavesingle(request, cave_shortname):
if cave_shortname in Gcavelookup: if cave_shortname in Gcavelookup:
cave = Gcavelookup[cave_shortname] cave = Gcavelookup[cave_shortname]
# print(f"survexcavesingle {cave_shortname=} => {cave=}") # print(f"survexcavesingle {cave_shortname=} => {cave=}")
cave.sds = get_directories(cave) cave.sds = get_primaries(cave)
return render(request, "svxcaves.html", {"settings": settings, "caves": [cave]}) return render(request, "svxcaves.html", {"settings": settings, "caves": [cave]})
else: else:
# not a cave or an ambiguous short name, e.g. "122" # not a cave or an ambiguous short name, e.g. "122"
@ -708,7 +709,7 @@ def survexcavesingle(request, cave_shortname):
if len(caves) > 0: if len(caves) > 0:
# print(f"many {cave_shortname=} => {caves=}") # print(f"many {cave_shortname=} => {caves=}")
for cave in caves: for cave in caves:
cave.sds = get_directories(cave) cave.sds = get_primaries(cave)
# print(f"many {cave=} => {cave.sds=}") # print(f"many {cave=} => {cave.sds=}")
return render(request, "svxcaves.html", {"settings": settings, "caves": caves}) return render(request, "svxcaves.html", {"settings": settings, "caves": caves})
else: else:

81
parsers/survex.py
View File

@ -332,9 +332,10 @@ class LoadingSurvex:
stacksvxfiles = [] stacksvxfiles = []
svxfileslist = [] svxfileslist = []
svxdirs = {} svxdirs = {}
svxprim = {}
uniquefile = {} # each key is a survex path, and its value is a list of parent files uniquefile = {} # each key is a survex path, and its value is a list of parent files
expos = {} expos = {}
survexdict = {} # each key is a directory, and its value is a list of files #survexdict = {} # each key is a directory, and its value is a list of files
lineno = 0 lineno = 0
insp = "" insp = ""
callcount = 0 callcount = 0
@ -1175,19 +1176,25 @@ class LoadingSurvex:
are doing. are doing.
""" """
# all = SurvexDirectory.objects.all()
# if not all:
# sd0 = SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfiley)
# sd0 = SurvexDirectory.objects.filter(id=1)[0]
if not headpath: if not headpath:
# This is normal for .svx file in the root of the :loser: repo # This is normal for .svx file in the root of the :loser: repo
# message = f" ! GetSurvexDirectory NO headpath given at {self.currentsurvexfile}" # message = f" ! GetSurvexDirectory NO headpath given at {self.currentsurvexfile}"
# print("\n"+message,file=sys.stderr) # print("\n"+message,file=sys.stderr)
# stash_data_issue(parser="survex", message=message, url=f"/survexfile/{self.currentsurvexfile}") # stash_data_issue(parser="survex", message=message, url=f"/survexfile/{self.currentsurvexfile}")
return self.svxdirs[""], self.currentsurvexfile return self.currentsurvexfile
if headpath.lower() not in self.svxdirs: if headpath.lower() not in self.svxprim:
primary = self.currentsurvexfile primary = self.currentsurvexfile
self.svxdirs[headpath.lower()] = SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower() # self.svxdirs[headpath.lower()] = sd0 #SurvexDirectory(path=headpath, primarysurvexfile=primary) # NOT .lower()
self.svxdirs[headpath.lower()].save() # self.svxdirs[headpath.lower()].save()
self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory self.svxprim[headpath.lower()] = primary
return self.svxdirs[headpath.lower()], self.svxdirs[headpath.lower()].primarysurvexfile #self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory
return self.svxprim[headpath.lower()]
def ReportNonCaveIncludes(self, headpath, includelabel, depth): def ReportNonCaveIncludes(self, headpath, includelabel, depth):
"""Ignore surface, kataser and gpx *include survex files""" """Ignore surface, kataser and gpx *include survex files"""
@ -1278,17 +1285,17 @@ class LoadingSurvex:
newfile.save() # until we do this there is no internal id so no foreign key works newfile.save() # until we do this there is no internal id so no foreign key works
self.currentsurvexfile = newfile self.currentsurvexfile = newfile
newdirectory, primary = self.GetSurvexDirectory(headpath) primary = self.GetSurvexDirectory(headpath)
newdirectory.save() #newdirectory.save()
newfile.survexdirectory = newdirectory #newfile.survexdirectory = newdirectory
self.survexdict[newdirectory].append(newfile) #self.survexdict[newdirectory].append(newfile)
newfile.primary = primary newfile.primary = primary
if not newdirectory: # if not newdirectory:
message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})" # message = f" ! 'None' SurvexDirectory returned from GetSurvexDirectory({headpath})"
print(message) # print(message)
print(message, file=sys.stderr) # print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}") # stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}")
# REPLACE all this IdentifyCave() stuff with GCaveLookup ? # REPLACE all this IdentifyCave() stuff with GCaveLookup ?
cave = IdentifyCave(headpath) # cave already exists in db cave = IdentifyCave(headpath) # cave already exists in db
@ -1302,23 +1309,23 @@ class LoadingSurvex:
newfile.cave = cave newfile.cave = cave
# print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr) # print(f"\n - New directory '{newdirectory}' for cave '{cave}'",file=sys.stderr)
if not newfile.survexdirectory: # if not newfile.survexdirectory:
message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} " # message = f" ! .survexdirectory NOT SET in new SurvexFile {svxid} "
print(message) # print(message)
print(message, file=sys.stderr) # print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message) # stash_data_issue(parser="survex", message=message)
if not newfile.primary: if not newfile.primary:
message = f" ! .primary NOT SET in new SurvexFile {svxid} " message = f" ! .primary NOT SET in new SurvexFile {svxid} "
print(message) print(message)
print(message, file=sys.stderr) print(message, file=sys.stderr)
stash_data_issue(parser="survex", message=message) stash_data_issue(parser="survex", message=message)
self.currentsurvexfile.save() # django insists on this although it is already saved !? self.currentsurvexfile.save() # django insists on this although it is already saved !?
try: # try:
newdirectory.save() # newdirectory.save()
except: # except:
print(newdirectory, file=sys.stderr) # print(newdirectory, file=sys.stderr)
print(newdirectory.primarysurvexfile, file=sys.stderr) # print(newdirectory.primarysurvexfile, file=sys.stderr)
raise # raise
def ProcessIncludeLine(self, included): def ProcessIncludeLine(self, included):
@ -2311,9 +2318,9 @@ def FindAndLoadSurvex(survexblockroot):
print("\n - Loading All Survex Blocks (LinearLoad)", file=sys.stderr) print("\n - Loading All Survex Blocks (LinearLoad)", file=sys.stderr)
svx_load = LoadingSurvex() svx_load = LoadingSurvex()
svx_load.survexdict[survexfileroot.survexdirectory] = [] #svx_load.survexdict[survexfileroot.survexdirectory] = []
svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot) #svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
svx_load.svxdirs[""] = survexfileroot.survexdirectory #svx_load.svxdirs[""] = survexfileroot.survexdirectory
# pr2 = cProfile.Profile() # pr2 = cProfile.Profile()
# pr2.enable() # pr2.enable()
@ -2339,10 +2346,10 @@ def FindAndLoadSurvex(survexblockroot):
legsnumber = svx_load.legsnumber legsnumber = svx_load.legsnumber
mem1 = get_process_memory() mem1 = get_process_memory()
print(f" - Number of SurvexDirectories: {len(svx_load.survexdict):,}") print(f" - Number of SurvexDirectories: {len(svx_load.svxprim):,}")
tf = 0 tf = 0
for d in svx_load.survexdict: # for d in svx_load.survexdict:
tf += len(svx_load.survexdict[d]) # tf += len(svx_load.survexdict[d])
print(f" - Number of SurvexFiles: {tf:,}") print(f" - Number of SurvexFiles: {tf:,}")
print(f" - Number of Survex legs: {legsnumber:,}") print(f" - Number of Survex legs: {legsnumber:,}")
svx_load = None svx_load = None
@ -2402,9 +2409,9 @@ def parse_one_file(fpath): # --------------------------------------in progress--
print(f" - block_dummy now '{block_dummy}' {type(block_dummy)} id={block_dummy.id} f:{block_dummy.survexfile}") print(f" - block_dummy now '{block_dummy}' {type(block_dummy)} id={block_dummy.id} f:{block_dummy.survexfile}")
svx_load.survexdict[svxfileroot.survexdirectory] = [] # svx_load.survexdict[svxfileroot.survexdirectory] = []
svx_load.survexdict[svxfileroot.survexdirectory].append(svxfileroot) # svx_load.survexdict[svxfileroot.survexdirectory].append(svxfileroot)
svx_load.svxdirs[""] = svxfileroot.survexdirectory #svx_load.svxdirs[""] = svxfileroot.survexdirectory
# ---------------------------------------------------------------- # ----------------------------------------------------------------
svx_load.LinearLoad(block_dummy, svxfileroot.path, fname) svx_load.LinearLoad(block_dummy, svxfileroot.path, fname)
@ -2418,7 +2425,7 @@ def parse_one_file(fpath): # --------------------------------------in progress--
global svx_load global svx_load
print(f"\n - Loading One Survex file '{fpath}'", file=sys.stderr) print(f"\n - Loading One Survex file '{fpath}'", file=sys.stderr)
svx_load = LoadingSurvex() svx_load = LoadingSurvex()
svx_load.survexdict = {} #svx_load.survexdict = {}
fname = Path(settings.SURVEX_DATA, (fpath + ".svx")) fname = Path(settings.SURVEX_DATA, (fpath + ".svx"))

17
templates/svxcaves.html
View File

@ -1,4 +1,4 @@
<!-- svxcavesingle.html - this text visible because this template has been included --> <!-- svxcaves.html - this text visible because this template has been included -->
{% extends "base.html" %} {% extends "base.html" %}
{% block title %}List of survex files{% endblock %} {% block title %}List of survex files{% endblock %}
{% block content %} {% block content %}
@ -13,8 +13,9 @@
<p>Wallets: <a href="/cave/scans/{{cave|safe}}">{{cave|safe}}</a> <p>Wallets: <a href="/cave/scans/{{cave|safe}}">{{cave|safe}}</a>
</p> </p>
<p> <p>
{% for svxdir in cave.sds %} {% for svxprim in cave.sds %}
<a href="#T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</a> &nbsp; <a href="#T_{{svxprim}}">{{svxprim}}</a>
&nbsp;
{% empty %} {% empty %}
<p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps <p>If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps
because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because <br> because you followed a link from <a href="/survexfile/caves/">the master caves' survex list</a> page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because <br>
@ -39,11 +40,11 @@ to go to a form to correct the online data.
Instructions for filling in this form are in this part Instructions for filling in this form are in this part
<a href="/handbook/survey/caveentry.html"> of the survey handbook</a>. <a href="/handbook/survey/caveentry.html"> of the survey handbook</a>.
</p> </p>
{% for svxdir in cave.sds %} {% for svxprim in cave.sds %}
<h3 id="T_{{svxdir.primarysurvexfile.path}}">{{svxdir.path}}</h3> <h3 id="T_{{svxprim.path}}">{{svxprim.path}}</h3>
<table> <table>
<tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr> <tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>length</th><th>Titles</th><th>Scans</th></tr>
{% for survexfile in svxdir.survexfile_set.all %} {% for survexfile in svxprim.primarysurvex.all %}
<tr> <tr>
{% if survexfile.exists %} {% if survexfile.exists %}
<td rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}"> <td rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}">
@ -51,7 +52,7 @@ to go to a form to correct the online data.
<td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}"> <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|add:"1"}}">
{% endif %} {% endif %}
{% if survexfile == svxdir.primarysurvexfile %} {% if survexfile == svxprim.primarysurvexfile %}
<a href="{% url "svx" survexfile.path %}"><b>{% url "svx" survexfile.path %}</b></a> <a href="{% url "svx" survexfile.path %}"><b>{% url "svx" survexfile.path %}</b></a>
{% else %} {% else %}
<a href="{% url "svx" survexfile.path %}"><i><small>{% url "svx" survexfile.path %}</small></i></a><!-- would like to extract only the last bit. Some javascript useful ?--> <a href="{% url "svx" survexfile.path %}"><i><small>{% url "svx" survexfile.path %}</small></i></a><!-- would like to extract only the last bit. Some javascript useful ?-->
@ -98,6 +99,6 @@ to go to a form to correct the online data.
{% endfor %} <!-- survexblock --> {% endfor %} <!-- survexblock -->
{% endfor %} <!-- survexfile --> {% endfor %} <!-- survexfile -->
</table> </table>
{% endfor %} <!-- svxdir --> {% endfor %} <!-- svxprim -->
{% endfor %} <!-- caves --> {% endfor %} <!-- caves -->
{% endblock %} {% endblock %}

2
templates/svxfile.html
View File

@ -82,7 +82,7 @@ LOGMESSAGES
</div> </div>
underground survey length: {{svxlength|floatformat:2}} metres<br /> underground survey length: {{svxlength|floatformat:2}} metres<br />
parent survex file <a href="/survexfile/{{ survexfile.cave.survex_file }}">{{survexfile.cave.survex_file}}</a> for this cave <br> parent survex file <a href="/survexfile/{{ survexfile.cave.survex_file }}">{{survexfile.cave.survex_file}}</a> for this cave <br>
survex directory <a href="/survexfile/{{survexfile.survexdirectory.primarysurvexfile}}">{{survexfile.survexdirectory}}</a> <br /> survex directory <a href="/survexfile/{{survexfile.primary}}.svx">{{survexfile.primary}}</a> <br />
{% for sb in svxblocks %} {% for sb in svxblocks %}
block <em>{{sb}}</em> has parent block <em>{{sb.parent}}</em><br /> block <em>{{sb}}</em> has parent block <em>{{sb.parent}}</em><br />
{% empty %} {% empty %}