expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-12-15 10:37:07 +00:00
Code Issues 18 Projects 3 Activity

new fix too for misplaced images i,t,l

Browse Source
This commit is contained in:
Philip Sargent
2025-02-04 21:25:41 +00:00
parent f39e2c9bb5
commit 7b462a883b
4 changed files with 266 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
core/views/cave_kataster.py
View File

@@ -16,6 +16,7 @@ from troggle.core.utils import (
git_string,
write_and_commit,
)
from troggle.core.models.caves import Cave, Entrance
from troggle.core.views.caves import get_cave_from_slug
"""Forms to handle renaming files and editing contents when a cave
@@ -23,16 +24,111 @@ is 'katastered', ie.e moves from an informal number, such as
1623-2024-BL-10 to 1623-999
"""
def get_loser_dir(cave):
""" Copes with capitalisation mismatch between declared loser diretory and actual
def fix(request, areacode="1626"):
"""Fix bad stuff: 2023 misplaced i/l/t files for caves in
both 1623 (65 files x3) and 1626 (29 filex x 3)
"""
loser_name = f"{str(cave.unofficial_number)}"
triple = ["i", "l", "t"]
ents_html = settings.ENTRANCEDESCRIPTIONS
caves_html = settings.CAVEDESCRIPTIONS
to_move = set()
caves_in_area = Cave.objects.filter(areacode=areacode, kataster_number="")
caves_nodir=[]
ents = []
print(f"\n---------- {areacode}")
for c in caves_in_area:
c_html = caves_html / f"{c.areacode}-{c.unofficial_number}.html"
if not c_html.is_file():
raise
ents.append(c.entrances())
with open(c_html, 'r') as f:
for line in f:
search_term = f"{areacode}" + "/(l|t|i)/([^'\"]*)"
if match := re.findall(search_term, line):
# print(c, search_term, match, line)
for m in match:
dir, filename = m
to_move.add(("c", c.unofficial_number, dir,filename))
for ce in ents:
for e in ce:
print(e.entrance.slug, e.entrance.filename)
e_html = ents_html / e.entrance.filename
with open(e_html, 'r') as f:
for line in f:
search_term = f"{areacode}" + "/(l|t|i)/([^'\"]*)"
if match := re.findall(search_term, line):
# print(c, search_term, match, line)
for m in match:
dir, filename = m
to_move.add(("e", c.unofficial_number, dir,filename))
for c in caves_in_area:
cave_dir = settings.EXPOWEB / areacode / c.unofficial_number
if cave_dir.is_dir():
print(f"YES {c.unofficial_number}")
for subfile in cave_dir.iterdir():
if subfile.name not in triple:
print(f" ---- {subfile.name}")
else:
print(f" ++++ {subfile.name}")
else:
# print(f"NO {c.unofficial_number}")
caves_nodir.append(c.unofficial_number)
for item in to_move:
print(item)
caves_nodir.sort()
print("\n Caves with no description directories")
for c in caves_nodir:
print(f"{c}, ", end="")
print("")
return render(
request,
"cave_fix.html",
{"areacode": areacode, "caves_nodir": caves_nodir,
}, )
return HttpResponseRedirect("/caves")
def misplaced_html_files(cave):
filelist = ""
bad_place = settings.EXPOWEB / cave.areacode /"l"
if bad_place.is_dir():
for subfile in bad_place.iterdir():
if subfile.name.lower().startswith(cave.unofficial_number.lower()):
filelist += "&nbsp;&nbsp;&nbsp;" + subfile.name + "<br />\n"
return filelist
def get_loser_dir(cave):
""" Copes with capitalisation mismatch between declared loser diretory and actual.
Two places to look:
1. in caves-1623/<cave id>/*.svx
2. in parent of survex file specified in the cave_data html page.
return loser_dir, loser_name
e.g.
/home/philip/expo/loser/caves-1623/2024-jc-01 , caves-1623/2024-jc-01
"""
loser_name = str(cave.unofficial_number)
parent = settings.SURVEX_DATA / f"caves-{str(cave.areacode)}"
for dn in [loser_name, loser_name.lower(), loser_name.upper()]:
if ( parent / dn).is_dir():
return parent / dn, Path(f"caves-{str(cave.areacode)}") / dn
return None, None
if (settings.SURVEX_DATA / cave.survex_file).is_file():
loser_dir = (settings.SURVEX_DATA / cave.survex_file).parent
return loser_dir, Path(cave.survex_file).parent
raise FileNotFoundError("This cave has no directory for survex files in the loser repo")
def get_cd(path):
return f"$BASE_DIR/{path.relative_to(settings.REPOS_ROOT_PATH)}"
@@ -53,7 +149,7 @@ def entrances_stations(cave):
entrance_stations.append((e.entrance, e.entrance.other_station[4:]))
return entrance_stations
def kataster(request, slug):
def kataster(request, slug=None):
"""Create the page which analyses how to rename a cave and all the files from the unofficial_number
identifier, e.g. 1623-2023-mg-03 to the kataster number e.g. 1623-999
"""
@@ -71,6 +167,8 @@ def kataster(request, slug):
pt_target = "$BASE_DIR/loser/fixedpts/gps/gps*.svx"
elif cave.areacode == "1626":
pt_target = "$BASE_DIR/loser/fixedpts/*.svx"
else: # 1627, 1624
pt_target = "nowt.svx"
for ent, station in entrances_stations(cave):
new = station.replace(cave.unofficial_number,str(knum))
sed_cmd = f"s/{station}/{new}/g".replace(".","\.")
@@ -118,7 +216,10 @@ def kataster(request, slug):
loser_data = []
try:
loser_dir, loser_name = get_loser_dir(cave)
except:
return mvscript
if (loser_dir).is_dir():
print(loser_dir)
for svx in loser_dir.iterdir():
@@ -129,6 +230,7 @@ def kataster(request, slug):
def script_loser(knum):
global cavefilename, cave_data, entrance_data, loser_name, loser_data
try:
loser_dir, loser_name = get_loser_dir(cave) #/home/philip/expo/loser/caves-1623/2024-jc-01 , caves-1623/2024-jc-01
target = loser_name.parent / str(knum)
survex_name = str(loser_name.name)
@@ -140,12 +242,17 @@ def kataster(request, slug):
l_script += f'sed -i "/^*begin/s/{survex_name}/{knum}/" {loser_name}/{survex_name}.svx\n'
l_script += f'sed -i "/^*end/s/{survex_name}/{knum}/" {loser_name}/{survex_name}.svx\n'
l_script +=f"# These 'sed' edits will not do everything in all cases, but they do the basics\n\n"
except FileNotFoundError as e:
print(e)
loser_name = ""
l_script = f"\n# {e}\n\n"
area_dir = get_cd(settings.SURVEX_DATA / f"caves-{cave.areacode}")
l_script +=f"cd {area_dir}\n"
l_script +=f"mv {cave.unofficial_number} {knum}\n"
l_script +=f"cd {knum}\n"
for filename in loser_data:
if survex_name in filename:
l_script +=f"mv {filename} {filename.replace(survex_name,str(knum))}\n"
l_script +=f"# But note that git ignores .log and .3d files\n\n"
@@ -160,7 +267,8 @@ def kataster(request, slug):
knum = 9999
if not slug:
slug = "1623-2013-BL-01"
if cave := get_cave_from_slug(slug.lower()):
pass
elif cave := get_cave_from_slug(slug.upper()):
@@ -187,11 +295,11 @@ def kataster(request, slug):
print(entrance)
except PermissionError as e:
msg=f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filepath}. Ask a nerd to fix this: {e}"
msg=f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {cavefilename}. Ask a nerd to fix this: {e}"
print(msg)
raise
except Exception as e:
msg=f"CANNOT write this file {f}. Ask a nerd to fix this: {e}"
msg=f"CANNOT write this file {cavefilename}. Ask a nerd to fix this: {e}"
print(msg)
# Restart script with POST data
@@ -199,7 +307,7 @@ def kataster(request, slug):
script += "# after the edits, but in a script it does not matter so much\n"
script += "# so long as everything is consistent and tested.\n"
script += "# Except that reversing changes using git does not always restore directories exactly\n# (because of .gitignore).\n\n"
script += "# Be careful with the BASE_DIR directory: usually NOT be the same on your PC as on the server\n\n"
script += "# Be careful with the BASE_DIR directory: usually NOT the same on your PC as on the server\n\n"
warning =""
@@ -229,35 +337,23 @@ def kataster(request, slug):
script += f'grep -nirI --exclude-dir=.git --exclude-dir=gpx --exclude="*.gpx" --exclude="*.log" --exclude="*.kml" --exclude="*.pos" "{cave.unofficial_number}" $BASE_DIR/loser\n'
script += f'grep -nirI --exclude-dir=.git --exclude="*.gpx" "{cave.unofficial_number}" $BASE_DIR/expoweb\n'
misplaced = misplaced_html_files(cave)
script_rows = str(max(35,3+script.count('\n')))
return render(
request,
"cave_kataster.html",
{
"form": form, "warning": warning, "error": error,
"cave": cave, "entrances": entrances,
"cave": cave, "entrances": entrances, "misplaced": misplaced,
"cave_data": cave_data, "entrance_data": entrance_data,
"loser_name": loser_name, "loser_data": loser_data,
"knum": knum, "script": script, "rows": script_rows,
}, )
class KatasterForm(forms.Form):
# areacode = forms.CharField(label='Full name', max_length=4, widget=forms.TextInput(attrs={'tabindex': 1, 'placeholder': '1623'}))
# official_name = forms.CharField(label='CUCC name', max_length=160,widget=forms.TextInput(attrs={'tabindex': 2, 'placeholder': '2012-ns-07'}))
kataster_number= forms.IntegerField(label="New kataster no.", widget=forms.TextInput(attrs={'tabindex': 1, 'size':1, 'placeholder': '9999'}))
"""
areacode = models.CharField(max_length=4, blank=True, null=True) # could use models.IntegerChoices
entrances = models.ManyToManyField("Entrance", through="CaveAndEntrance")
filename = models.CharField(max_length=200) # if a cave is 'pending' this is not set. Otherwise it is.
kataster_code = models.CharField(max_length=20, blank=True, null=True)
kataster_number = models.CharField(max_length=10, blank=True, null=True)
kataster_status = models.TextField(blank=True, null=True)
official_name = models.CharField(max_length=160)
survex_file = models.CharField(max_length=100, blank=True, null=True) # should be a foreign key?
unofficial_number = models.CharField(max_length=60, blank=True, null=True)
url = models.CharField(max_length=300, blank=True, null=True, unique = True)
SURVEX_DATA = REPOS_ROOT_PATH / "loser"
EXPOWEB = REPOS_ROOT_PATH / "expoweb"
CAVEDESCRIPTIONS = EXPOWEB / "cave_data"

122
templates/cave_fix.html Normal file
View File

@@ -0,0 +1,122 @@
{% extends "base.html" %}
{% block title %}Cave fix misplaced images and descriptive files
<!-- cave_kataster.html - this text visible because this template has been included -->
{% endblock %}
{% block content %}
<h2>Cave fix misplaced images and descriptive files</h2>
<form action="{% url 'fix' areacode %}" method="post">
<p>
{% csrf_token %}
{{form.as_p}}
<button class="fancybutton" style="padding: 0.5em 25px; font-size: 100%;" type = "submit" >
See what the edits look like &rarr;
</button>
<div style="color:red; font-weight: bold; font-size: large">
{{warning|safe}}<p>
{{error|safe}}</div>
<h3>Rename the cave description directories in <var>expoweb<a href="/{{areacode|safe}}">{{areacode|safe}}</a></var></h3>
<p>
<ul style="list-style: disc">
<li> Edit all the '<samp>href=</samp>' and '<samp>src=</samp>' <small>URLS</small> (if they exist) inside all the <samp>cave_data</samp> and <samp>entrance_data</samp> files descriptive text to refer to the new directory
<li> Rename the directory (if it exists) inside the areacode directory, e.g. rename <samp>/{{areacode}}/{{cave.unofficial_number}}/</samp> as <samp>/{{areacode}}/<span style="color:blue">{{knum}}</span>/</samp> (if <span style="color:blue">{{knum}}</span> is the correct new kataster number). Do this last.
</ul>
In 1623 there are <strike>82</strike> 69 cave description sub-files, i.e. single pages with a larger image and caption using the t/i/l convention, are not in <br>
<samp><b>1623/[cave]/l/*.html</b></samp> but are directly in <br />
<samp><b>1623/l/[cave]*.html</b></samp> <br>
due to early versions of Martin's image insertion tool for editing cave descriptions. 21 of these <strike>do</strike> did not even have filenames beginning with [cave], the cave unofficial identifier, but are all digits. They seem to be all 2023 discoveries or caves which have had images added to their descriptions in 2023.
<p><b><samp>
{% if misplaced %}
<samp><b>1623/l/ :</samp></b><br />
{% endif %}
{{misplaced|safe}}
</samp></b><br />
If we don't edit these, things will still work, i.e. cave description images will still appear, but the names of the files no longer match the new kataster name of the cave. Fixing this - so that these files are stored together with the rest of the stuff just for that cave - could be included in in this renaming tool, but is perhaps better done as another job - which needs doing as troggle does some weird perversions to display these images. Some of these caves may never be katastered.
<p>
<h3>Move and edit the .html files in <var>expoweb/i,l,t</var></h3>
<div style="font-family: monospace; font-weight: bold;">
{{cave_data|safe}}<br />
&nbsp;&nbsp;&nbsp;<span style="color:grey">&lt;kataster_number&gt;</span><span style="color:grey">&lt;/kataster_number&gt;</span>
<br />
&nbsp;&nbsp;&nbsp;
&rarr;
<span style="color:grey">&lt;kataster_number&gt;</span><span style="color:blue">{{knum}}</span><span style="color:grey">&lt;/kataster_number&gt;</span>
<br />
&nbsp;&nbsp;&nbsp;<span style="color:grey">&lt;survex_file&gt;</span>{{cave.survex_file}}
<span style="color:grey">&lt;/survex_file&gt;</span>
<br />
&nbsp;&nbsp;&nbsp;
&rarr;
<span style="color:grey">&lt;survex_file&gt;</span><span style="color:blue">caves-{{areacode}}/{{knum}}/{{knum}}.svx</span>
<span style="color:grey">&lt;/survex_file&gt;</span>
<p>
{% for e in entrance_data %}
{{e|safe}}</br />
{% endfor %}
</div>
<ul style="list-style: disc">
<li> Edit the '<samp>kataster'number</samp>' field inside the <samp>cave_data</samp> file.
<li> Edit the '<samp>survex_file</samp>' field inside the <samp>cave_data</samp> file.
<li> Edit the '<samp>entranceslug</samp>' field inside each '<samp>entrance</samp>' field in the <samp>cave_data</samp> file.
<li> Rename all the files listed above - always do this <b>last</b> otherwise it is hard to automatically discover which edits have not been done.
</ul>
<h3>Finally</h3>
<ul style="list-style: disc">
<li><samp>cd loser<br />
git add *<br />
git commit -m 'Moving misplaced files {{cave.slug|safe}} in <span style="color:blue">{{areacode |safe}}</span>'<br />
git pull<br />
git push<br />
cd ../expoweb<br />
git add *<br />
git commit -m 'Moving misplaced files {{cave.slug|safe}} in <span style="color:blue">{{areacode |safe}}</span>'<br />
git pull<br />
git push<br />
</samp>
<li>It is then vital to do a complete databaseReset as troggle has internally indexed all those {{cave.slug}} files, the indexes are now out of date and horrible things will happen when people try to use troggle with any cave that has been altered:<br />
<samp>uv run databaseReset reset <span style="color:blue">M{{areacode}}</span></samp><br />
OR we can do an online rename of the "Cave" object in the database [code yet to be written] by pressing the button above [button not yet implemented].
<li>Look in the reset import warnings and errors report to see if anything has gone wrong:
<var><a href="http://expo.survex.com/dataissues">expo.survex.com/dataissues</a></var><br />
(in fact, look at this before you do any of this, so you can see what chnages.)
</ul>
<h3>Ideal scenario</h3>
<p>
<p>Ideally you do this renumbering on a standalone instance of troggle running on a <var> &#9874; <a href="/handbook/troggle/troglaptop.html">troggle development laptop</a></var> and you don't push any of the git commits to the server until you have got all the niggles out of the conversion, i.e. the databaseReset runs locally without any warnings or errors introduced by your renaming <em>and</em> 'cavern' runs without any new survex errors.
<h3>Copy this script and edit it to match the directory names on your PC</h3>
<textarea id="script" name="script" rows="{{rows}}" cols="110" style="font-size:small; font-weight: bold; background-color: LemonChiffon">
{{script}}
</textarea>
<br />
<!-- duplicate button at bottom of long page -->
<button class="fancybutton" style="padding: 0.5em 25px; font-size: 100%;" type = "submit" >
See what the edits look like &rarr;
</button>
</form>
<div style="color:red; font-weight: bold; font-size: large">
{{warning|safe}}<p>
{{error|safe}}</div>
{% endblock %}

13
templates/cave_kataster.html
View File

@@ -69,11 +69,16 @@ This cave needs to be "katastered". If you have the new number issued by the Aus
<li> Edit all the '<samp>href=</samp>' and '<samp>src=</samp>' <small>URLS</small> (if they exist) inside all the <samp>cave_data</samp> and <samp>entrance_data</samp> files descriptive text to refer to the new directory
<li> Rename the directory (if it exists) inside the areacode directory, e.g. rename <samp>/{{cave.areacode}}/{{cave.unofficial_number}}/</samp> as <samp>/{{cave.areacode}}/<span style="color:blue">{{knum}}</span>/</samp> (if <span style="color:blue">{{knum}}</span> is the correct new kataster number). Do this last.
</ul>
There are 82 cave description sub-files, i.e. single pages with a larger image and caption using the t/i/l convention, are not in <br>
There are <strike>82</strike> 69 cave description sub-files, i.e. single pages with a larger image and caption using the t/i/l convention, are not in <br>
<samp><b>1623/[cave]/l/*.html</b></samp> but are directly in <br />
<samp><b>1623/l/[cave]_*.html</b></samp> <br>
due to early versions of Martin's image insertion tool for editing cave descriptions. 21 of these do not even have filenames beginning with [cave], the cave unofficial identifier, but are all digits. They seem to be all 2023 discoveries or caves which have had images added to their descriptions in 2023.
<p>
<samp><b>1623/l/[cave]*.html</b></samp> <br>
due to early versions of Martin's image insertion tool for editing cave descriptions. 21 of these <strike>do</strike> did not even have filenames beginning with [cave], the cave unofficial identifier, but are all digits. They seem to be all 2023 discoveries or caves which have had images added to their descriptions in 2023.
<p><b><samp>
{% if misplaced %}
<samp><b>1623/l/ :</samp></b><br />
{% endif %}
{{misplaced|safe}}
</samp></b><br />
If we don't edit these, things will still work, i.e. cave description images will still appear, but the names of the files no longer match the new kataster name of the cave. Fixing this - so that these files are stored together with the rest of the stuff just for that cave - could be included in in this renaming tool, but is perhaps better done as another job - which needs doing as troggle does some weird perversions to display these images. Some of these caves may never be katastered.
<p>

4
urls.py
View File

@@ -23,7 +23,7 @@ from troggle.core.views.caves import (
get_entrances,
qm,
)
from troggle.core.views.cave_kataster import (kataster)
from troggle.core.views.cave_kataster import kataster, fix
from troggle.core.views.drawings import dwgallfiles, dwgfilesingle
from troggle.core.views.editor_helpers import image_selector, new_image_form
from troggle.core.views.expo import (
@@ -206,6 +206,8 @@ trogglepatterns = [
path('cave/<slug:slug>', caveslugfwd, name="caveslugfwd"),
path('cave_debug', cave_debug, name="cave_debug"),
path('kataster/<slug:slug>', kataster, name="kataster"),
path('kataster', kataster, name="kataster"),
path('fix/<slug:areacode>', fix, name="fix"),
re_path(r'^newcave/$', edit_cave, name="newcave"),
re_path(r'^cave/3d/(?P<cave_id>[^/]+).3d$', cave3d, name="cave3d"),