diff --git a/core/views/cave_kataster.py b/core/views/cave_kataster.py index bb9d68454..6ad30597a 100644 --- a/core/views/cave_kataster.py +++ b/core/views/cave_kataster.py @@ -16,6 +16,7 @@ from troggle.core.utils import ( git_string, write_and_commit, ) +from troggle.core.models.caves import Cave, Entrance from troggle.core.views.caves import get_cave_from_slug """Forms to handle renaming files and editing contents when a cave @@ -23,17 +24,112 @@ is 'katastered', ie.e moves from an informal number, such as 1623-2024-BL-10 to 1623-999 """ -def get_loser_dir(cave): - """ Copes with capitalisation mismatch between declared loser diretory and actual +def fix(request, areacode="1626"): + """Fix bad stuff: 2023 misplaced i/l/t files for caves in + both 1623 (65 files x3) and 1626 (29 filex x 3) """ - loser_name = f"{str(cave.unofficial_number)}" + triple = ["i", "l", "t"] + + ents_html = settings.ENTRANCEDESCRIPTIONS + caves_html = settings.CAVEDESCRIPTIONS + to_move = set() + + caves_in_area = Cave.objects.filter(areacode=areacode, kataster_number="") + caves_nodir=[] + ents = [] + print(f"\n---------- {areacode}") + for c in caves_in_area: + c_html = caves_html / f"{c.areacode}-{c.unofficial_number}.html" + if not c_html.is_file(): + raise + ents.append(c.entrances()) + with open(c_html, 'r') as f: + for line in f: + search_term = f"{areacode}" + "/(l|t|i)/([^'\"]*)" + if match := re.findall(search_term, line): + # print(c, search_term, match, line) + for m in match: + dir, filename = m + to_move.add(("c", c.unofficial_number, dir,filename)) + + for ce in ents: + for e in ce: + print(e.entrance.slug, e.entrance.filename) + e_html = ents_html / e.entrance.filename + with open(e_html, 'r') as f: + for line in f: + search_term = f"{areacode}" + "/(l|t|i)/([^'\"]*)" + if match := re.findall(search_term, line): + # print(c, search_term, match, line) + for m in match: + dir, filename = m + to_move.add(("e", c.unofficial_number, dir,filename)) + + for c in caves_in_area: + cave_dir = settings.EXPOWEB / areacode / c.unofficial_number + if cave_dir.is_dir(): + print(f"YES {c.unofficial_number}") + for subfile in cave_dir.iterdir(): + if subfile.name not in triple: + print(f" ---- {subfile.name}") + else: + print(f" ++++ {subfile.name}") + else: + # print(f"NO {c.unofficial_number}") + caves_nodir.append(c.unofficial_number) + + for item in to_move: + print(item) + + caves_nodir.sort() + print("\n Caves with no description directories") + for c in caves_nodir: + print(f"{c}, ", end="") + print("") + + + + return render( + request, + "cave_fix.html", + {"areacode": areacode, "caves_nodir": caves_nodir, + }, ) + return HttpResponseRedirect("/caves") + + +def misplaced_html_files(cave): + filelist = "" + bad_place = settings.EXPOWEB / cave.areacode /"l" + if bad_place.is_dir(): + for subfile in bad_place.iterdir(): + if subfile.name.lower().startswith(cave.unofficial_number.lower()): + filelist += "   " + subfile.name + "
\n" + + return filelist + +def get_loser_dir(cave): + """ Copes with capitalisation mismatch between declared loser diretory and actual. + + Two places to look: + 1. in caves-1623//*.svx + 2. in parent of survex file specified in the cave_data html page. + + return loser_dir, loser_name + e.g. + /home/philip/expo/loser/caves-1623/2024-jc-01 , caves-1623/2024-jc-01 + """ + loser_name = str(cave.unofficial_number) parent = settings.SURVEX_DATA / f"caves-{str(cave.areacode)}" for dn in [loser_name, loser_name.lower(), loser_name.upper()]: if ( parent / dn).is_dir(): return parent / dn, Path(f"caves-{str(cave.areacode)}") / dn - return None, None - + if (settings.SURVEX_DATA / cave.survex_file).is_file(): + loser_dir = (settings.SURVEX_DATA / cave.survex_file).parent + return loser_dir, Path(cave.survex_file).parent + + raise FileNotFoundError("This cave has no directory for survex files in the loser repo") + def get_cd(path): return f"$BASE_DIR/{path.relative_to(settings.REPOS_ROOT_PATH)}" @@ -53,7 +149,7 @@ def entrances_stations(cave): entrance_stations.append((e.entrance, e.entrance.other_station[4:])) return entrance_stations -def kataster(request, slug): +def kataster(request, slug=None): """Create the page which analyses how to rename a cave and all the files from the unofficial_number identifier, e.g. 1623-2023-mg-03 to the kataster number e.g. 1623-999 """ @@ -71,6 +167,8 @@ def kataster(request, slug): pt_target = "$BASE_DIR/loser/fixedpts/gps/gps*.svx" elif cave.areacode == "1626": pt_target = "$BASE_DIR/loser/fixedpts/*.svx" + else: # 1627, 1624 + pt_target = "nowt.svx" for ent, station in entrances_stations(cave): new = station.replace(cave.unofficial_number,str(knum)) sed_cmd = f"s/{station}/{new}/g".replace(".","\.") @@ -118,7 +216,10 @@ def kataster(request, slug): loser_data = [] - loser_dir, loser_name = get_loser_dir(cave) + try: + loser_dir, loser_name = get_loser_dir(cave) + except: + return mvscript if (loser_dir).is_dir(): print(loser_dir) for svx in loser_dir.iterdir(): @@ -129,24 +230,30 @@ def kataster(request, slug): def script_loser(knum): global cavefilename, cave_data, entrance_data, loser_name, loser_data - loser_dir, loser_name = get_loser_dir(cave) #/home/philip/expo/loser/caves-1623/2024-jc-01 , caves-1623/2024-jc-01 - target = loser_name.parent / str(knum) - survex_name = str(loser_name.name) - l_script = f"\ncd {get_cd(settings.SURVEX_DATA)}\n" - l_script += f'sed -i "/^*include/s/{survex_name}/{knum}/g" {loser_name.parent}/caves.svx\n' + try: + loser_dir, loser_name = get_loser_dir(cave) #/home/philip/expo/loser/caves-1623/2024-jc-01 , caves-1623/2024-jc-01 + target = loser_name.parent / str(knum) + survex_name = str(loser_name.name) + l_script = f"\ncd {get_cd(settings.SURVEX_DATA)}\n" + l_script += f'sed -i "/^*include/s/{survex_name}/{knum}/g" {loser_name.parent}/caves.svx\n' - l_script += f'sed -i "/^*equate/s/{survex_name}/{knum}/g" {loser_name}/{survex_name}.svx\n' - l_script += f'sed -i "/^*entrance/s/{survex_name}/{knum}/g" {loser_name}/{survex_name}.svx\n' - l_script += f'sed -i "/^*begin/s/{survex_name}/{knum}/" {loser_name}/{survex_name}.svx\n' - l_script += f'sed -i "/^*end/s/{survex_name}/{knum}/" {loser_name}/{survex_name}.svx\n' - l_script +=f"# These 'sed' edits will not do everything in all cases, but they do the basics\n\n" + l_script += f'sed -i "/^*equate/s/{survex_name}/{knum}/g" {loser_name}/{survex_name}.svx\n' + l_script += f'sed -i "/^*entrance/s/{survex_name}/{knum}/g" {loser_name}/{survex_name}.svx\n' + l_script += f'sed -i "/^*begin/s/{survex_name}/{knum}/" {loser_name}/{survex_name}.svx\n' + l_script += f'sed -i "/^*end/s/{survex_name}/{knum}/" {loser_name}/{survex_name}.svx\n' + l_script +=f"# These 'sed' edits will not do everything in all cases, but they do the basics\n\n" + except FileNotFoundError as e: + print(e) + loser_name = "" + l_script = f"\n# {e}\n\n" area_dir = get_cd(settings.SURVEX_DATA / f"caves-{cave.areacode}") l_script +=f"cd {area_dir}\n" l_script +=f"mv {cave.unofficial_number} {knum}\n" l_script +=f"cd {knum}\n" for filename in loser_data: - l_script +=f"mv {filename} {filename.replace(survex_name,str(knum))}\n" + if survex_name in filename: + l_script +=f"mv {filename} {filename.replace(survex_name,str(knum))}\n" l_script +=f"# But note that git ignores .log and .3d files\n\n" error = "" @@ -160,7 +267,8 @@ def kataster(request, slug): knum = 9999 - + if not slug: + slug = "1623-2013-BL-01" if cave := get_cave_from_slug(slug.lower()): pass elif cave := get_cave_from_slug(slug.upper()): @@ -187,11 +295,11 @@ def kataster(request, slug): print(entrance) except PermissionError as e: - msg=f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {filepath}. Ask a nerd to fix this: {e}" + msg=f"CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file {cavefilename}. Ask a nerd to fix this: {e}" print(msg) raise except Exception as e: - msg=f"CANNOT write this file {f}. Ask a nerd to fix this: {e}" + msg=f"CANNOT write this file {cavefilename}. Ask a nerd to fix this: {e}" print(msg) # Restart script with POST data @@ -199,7 +307,7 @@ def kataster(request, slug): script += "# after the edits, but in a script it does not matter so much\n" script += "# so long as everything is consistent and tested.\n" script += "# Except that reversing changes using git does not always restore directories exactly\n# (because of .gitignore).\n\n" - script += "# Be careful with the BASE_DIR directory: usually NOT be the same on your PC as on the server\n\n" + script += "# Be careful with the BASE_DIR directory: usually NOT the same on your PC as on the server\n\n" warning ="" @@ -229,35 +337,23 @@ def kataster(request, slug): script += f'grep -nirI --exclude-dir=.git --exclude-dir=gpx --exclude="*.gpx" --exclude="*.log" --exclude="*.kml" --exclude="*.pos" "{cave.unofficial_number}" $BASE_DIR/loser\n' script += f'grep -nirI --exclude-dir=.git --exclude="*.gpx" "{cave.unofficial_number}" $BASE_DIR/expoweb\n' + misplaced = misplaced_html_files(cave) script_rows = str(max(35,3+script.count('\n'))) return render( request, "cave_kataster.html", { "form": form, "warning": warning, "error": error, - "cave": cave, "entrances": entrances, + "cave": cave, "entrances": entrances, "misplaced": misplaced, "cave_data": cave_data, "entrance_data": entrance_data, "loser_name": loser_name, "loser_data": loser_data, "knum": knum, "script": script, "rows": script_rows, }, ) class KatasterForm(forms.Form): - # areacode = forms.CharField(label='Full name', max_length=4, widget=forms.TextInput(attrs={'tabindex': 1, 'placeholder': '1623'})) - # official_name = forms.CharField(label='CUCC name', max_length=160,widget=forms.TextInput(attrs={'tabindex': 2, 'placeholder': '2012-ns-07'})) kataster_number= forms.IntegerField(label="New kataster no.", widget=forms.TextInput(attrs={'tabindex': 1, 'size':1, 'placeholder': '9999'})) """ - areacode = models.CharField(max_length=4, blank=True, null=True) # could use models.IntegerChoices - entrances = models.ManyToManyField("Entrance", through="CaveAndEntrance") - filename = models.CharField(max_length=200) # if a cave is 'pending' this is not set. Otherwise it is. - kataster_code = models.CharField(max_length=20, blank=True, null=True) - kataster_number = models.CharField(max_length=10, blank=True, null=True) - kataster_status = models.TextField(blank=True, null=True) - official_name = models.CharField(max_length=160) - survex_file = models.CharField(max_length=100, blank=True, null=True) # should be a foreign key? - unofficial_number = models.CharField(max_length=60, blank=True, null=True) - url = models.CharField(max_length=300, blank=True, null=True, unique = True) - SURVEX_DATA = REPOS_ROOT_PATH / "loser" EXPOWEB = REPOS_ROOT_PATH / "expoweb" CAVEDESCRIPTIONS = EXPOWEB / "cave_data" diff --git a/templates/cave_fix.html b/templates/cave_fix.html new file mode 100644 index 000000000..a29eed833 --- /dev/null +++ b/templates/cave_fix.html @@ -0,0 +1,122 @@ +{% extends "base.html" %} +{% block title %}Cave fix misplaced images and descriptive files + +{% endblock %} + +{% block content %} + + + +

Cave fix misplaced images and descriptive files

+ +
+

+ +{% csrf_token %} +{{form.as_p}} + + +

+{{warning|safe}}

+{{error|safe}}

+ +

Rename the cave description directories in expoweb{{areacode|safe}}

+

+

+In 1623 there are 82 69 cave description sub-files, i.e. single pages with a larger image and caption using the t/i/l convention, are not in
+1623/[cave]/l/*.html but are directly in
+1623/l/[cave]*.html
+due to early versions of Martin's image insertion tool for editing cave descriptions. 21 of these do did not even have filenames beginning with [cave], the cave unofficial identifier, but are all digits. They seem to be all 2023 discoveries or caves which have had images added to their descriptions in 2023. +

+{% if misplaced %} +1623/l/ :
+{% endif %} +{{misplaced|safe}} +
+If we don't edit these, things will still work, i.e. cave description images will still appear, but the names of the files no longer match the new kataster name of the cave. Fixing this - so that these files are stored together with the rest of the stuff just for that cave - could be included in in this renaming tool, but is perhaps better done as another job - which needs doing as troggle does some weird perversions to display these images. Some of these caves may never be katastered. +

+ + + + +

Move and edit the .html files in expoweb/i,l,t

+
+{{cave_data|safe}}
+   <kataster_number></kataster_number> +
+    +→ +<kataster_number>{{knum}}</kataster_number> + +
+   <survex_file>{{cave.survex_file}} +</survex_file> +
+    +→ +<survex_file>caves-{{areacode}}/{{knum}}/{{knum}}.svx +</survex_file> + + +

+{% for e in entrance_data %} +{{e|safe}}
+{% endfor %} +

+ + + + + + +

Finally

+ + +

Ideal scenario

+

+

Ideally you do this renumbering on a standalone instance of troggle running on a ⚒ troggle development laptop and you don't push any of the git commits to the server until you have got all the niggles out of the conversion, i.e. the databaseReset runs locally without any warnings or errors introduced by your renaming and 'cavern' runs without any new survex errors. + +

Copy this script and edit it to match the directory names on your PC

+ + +
+ + +
+
+{{warning|safe}}

+{{error|safe}}

+{% endblock %} diff --git a/templates/cave_kataster.html b/templates/cave_kataster.html index a4e189a55..8c56c4f26 100644 --- a/templates/cave_kataster.html +++ b/templates/cave_kataster.html @@ -69,11 +69,16 @@ This cave needs to be "katastered". If you have the new number issued by the Aus
  • Edit all the 'href=' and 'src=' URLS (if they exist) inside all the cave_data and entrance_data files descriptive text to refer to the new directory
  • Rename the directory (if it exists) inside the areacode directory, e.g. rename /{{cave.areacode}}/{{cave.unofficial_number}}/ as /{{cave.areacode}}/{{knum}}/ (if {{knum}} is the correct new kataster number). Do this last. -There are 82 cave description sub-files, i.e. single pages with a larger image and caption using the t/i/l convention, are not in
    +There are 82 69 cave description sub-files, i.e. single pages with a larger image and caption using the t/i/l convention, are not in
    1623/[cave]/l/*.html but are directly in
    -1623/l/[cave]_*.html
    -due to early versions of Martin's image insertion tool for editing cave descriptions. 21 of these do not even have filenames beginning with [cave], the cave unofficial identifier, but are all digits. They seem to be all 2023 discoveries or caves which have had images added to their descriptions in 2023. -

    +1623/l/[cave]*.html
    +due to early versions of Martin's image insertion tool for editing cave descriptions. 21 of these do did not even have filenames beginning with [cave], the cave unofficial identifier, but are all digits. They seem to be all 2023 discoveries or caves which have had images added to their descriptions in 2023. +

    +{% if misplaced %} +1623/l/ :
    +{% endif %} +{{misplaced|safe}} +
    If we don't edit these, things will still work, i.e. cave description images will still appear, but the names of the files no longer match the new kataster name of the cave. Fixing this - so that these files are stored together with the rest of the stuff just for that cave - could be included in in this renaming tool, but is perhaps better done as another job - which needs doing as troggle does some weird perversions to display these images. Some of these caves may never be katastered.

    diff --git a/urls.py b/urls.py index aafcb00f2..56e0159c3 100644 --- a/urls.py +++ b/urls.py @@ -23,7 +23,7 @@ from troggle.core.views.caves import ( get_entrances, qm, ) -from troggle.core.views.cave_kataster import (kataster) +from troggle.core.views.cave_kataster import kataster, fix from troggle.core.views.drawings import dwgallfiles, dwgfilesingle from troggle.core.views.editor_helpers import image_selector, new_image_form from troggle.core.views.expo import ( @@ -206,6 +206,8 @@ trogglepatterns = [ path('cave/', caveslugfwd, name="caveslugfwd"), path('cave_debug', cave_debug, name="cave_debug"), path('kataster/', kataster, name="kataster"), + path('kataster', kataster, name="kataster"), + path('fix/', fix, name="fix"), re_path(r'^newcave/$', edit_cave, name="newcave"), re_path(r'^cave/3d/(?P[^/]+).3d$', cave3d, name="cave3d"),