From ced9a7b0241e46c8b6f0402f73d08e9ab84e95bd Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Wed, 26 Nov 2025 01:22:38 +0200 Subject: [PATCH] SHould be nearly working, but crashes on saving edited entry --- core/models/logbooks.py | 29 +++++++- core/utils.py | 7 +- core/views/logbook_edit.py | 125 +++++++++++++++++--------------- core/views/logbooks.py | 42 +++++++++-- parsers/imports.py | 2 +- parsers/logbooks.py | 51 ++++++++----- settings.py | 2 + templates/expedition.html | 2 +- templates/logbook2005style.html | 10 ++- templates/logbookentry.html | 2 +- templates/logbookform.html | 5 +- templates/logreport.html | 6 +- urls.py | 2 + 13 files changed, 187 insertions(+), 98 deletions(-) diff --git a/core/models/logbooks.py b/core/models/logbooks.py index 3152a49a4..4d35d6e13 100644 --- a/core/models/logbooks.py +++ b/core/models/logbooks.py @@ -107,7 +107,31 @@ class LogbookEntry(TroggleModel): index = index % mx return index -def writelogbook(year, filename): +READ_THIS=""" +""" +def writelogbook(year, filename, generate_on_view=False): """Writes all the database logbook entries into a new logbook.html file """ current_expedition = Expedition.objects.get(year=year) @@ -140,6 +164,9 @@ def writelogbook(year, filename): except: print(" ! Very Bad Error opening " + endpath) raise + if generate_on_view: + endmatter += READ_THIS + except: print(" ! FAIL endpath " + endpath) raise diff --git a/core/utils.py b/core/utils.py index 408198db0..0954338f5 100644 --- a/core/utils.py +++ b/core/utils.py @@ -405,8 +405,11 @@ def git_commit(cwd, message, editor, commands=[]): print(f"..{message=}\n..{editor=}") cmd_commit = [git, "commit", "-m", message, "--author", f"{editor}"] commands.append(cmd_commit) - - cp_commit = subprocess.run(cmd_commit, cwd=cwd, capture_output=True, text=True) + print(commands) + try: + cp_commit = subprocess.run(cmd_commit, cwd=cwd, capture_output=True, text=True) + except Exception as e: + print(e) # This produces return code = 1 if it commits OK, but when the local repo still needs to be pushed to origin/repo # which will be the case when running a test troggle system on a development machine diff --git a/core/views/logbook_edit.py b/core/views/logbook_edit.py index 457fdbc0c..18e3c6981 100644 --- a/core/views/logbook_edit.py +++ b/core/views/logbook_edit.py @@ -25,6 +25,7 @@ from troggle.core.utils import ( write_and_commit, wrap_text, ) +from troggle.core.views.logbooks import write_entries_json from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* @@ -301,70 +302,76 @@ def logbookedit(request, year=None, slug=None): return render(request, "errors/generic.html", {"message": message}) store_edited_entry_into_database(date, place, title, entry, others, author, tu, slug) + # Successful POST so save to fiesystem + json_entries_dir = settings.EXPOWEB / "years" / year / settings.JSON_LOG_ENTRIES + if json_entries_dir.is_dir(): # only 2025 currently, or the current expo + print(f"- Rewriting JUST this edited logbook entry to a JSON file. ") + this_entry = LogbookEntry.objects.get(slug=slug) + write_entries_json([this_entry], year, editor) + else: + print(f"- Rewriting the entire {year} logbook to disc ") + filename= "logbook.html" + try: + print(f" - Logbook for {year} to be exported and written out.") + writelogbook(year, filename) # uses a template, not the code fragment below which is just a visible hint to logged on user + except: + message = f'! - Logbook saving failed - \n!! Permissions failure ?! on attempting to save file "logbook.html"' + print(message) + return render(request, "errors/generic.html", {"message": message}) + + + # So save to database and then write out whole new logbook.html file - print(f"- Rewriting the entire {year} logbook to disc ") - filename= "logbook.html" - try: - print(f" - Logbook for {year} to be exported and written out.") - writelogbook(year, filename) # uses a template, not the code fragment below which is just a visible hint to logged on user - except: - message = f'! - Logbook saving failed - \n!! Permissions failure ?! on attempting to save file "logbook.html"' - print(message) - return render(request, "errors/generic.html", {"message": message}) - - # Code fragment illustration - not actually what gets saved to database - output = f''' + # We do author validation on the form as displayed by GET, not at the moment of POST. + # If we had JS validation then we could be more timely. + dirpath = Path(settings.EXPOWEB) / "years" / str(year) + contents_path = dirpath / filename + commit_msg = f"Online edit of logbookentry {slug}" + add_commit(contents_path, commit_msg, editor) + + # This does not change the URL in the browser, so despite a new slug being created, + # the next time this code is run it thinks a new slug needs to be created. So we should + # actually redirect to a new URL (an edit not a create) not simply return a render object. + # logbookedit/2022-08-21a + + # HOWEVER by doing a redirect rather than returning a rendered page, we lose all the + # error settings e.g dateflag and authroflag so the user gets no feedback about bad data entered. + # so we need to pass the flags explicitly in the url and then extract them from the request in the GET bit. sigh. + response = HttpResponseRedirect(f"/logbookedit/{slug}?dateflag={dateflag}&authorflag={authorflag}") + response.set_cookie('editor_id', editor, max_age=get_cookie_max_age(request)) # cookie expires after get_cookie_max_age(request) seconds + return response + + # Do the redirect instead of this: + + # Code fragment illustration - not actually what gets saved to database + # output = f''' -
{date}
-
{author}, {others}
-
{place} - {title}
+#
{date}
+#
{author}, {others}
+#
{place} - {title}
-{entry} +# {entry} -
T/U {tu} hrs
-
+#
T/U {tu} hrs
+#
-''' - # Successful POST - # So save to database and then write out whole new logbook.html file - - # We do author validation on the form as displayed by GET, not at the moment of POST. - # If we had JS validation then we could be more timely. - dirpath = Path(settings.EXPOWEB) / "years" / str(year) - contents_path = dirpath / filename - commit_msg = f"Online edit of logbookentry {slug}" - add_commit(contents_path, commit_msg, editor) - - # This does not change the URL in the browser, so despite a new slug being created, - # the next time this code is run it thinks a new slug needs to be created. So we should - # actually redirect to a new URL (an edit not a create) not simply return a render object. - # logbookedit/2022-08-21a - - # HOWEVER by doing a redirect rather than returning a rendered page, we lose all the - # error settings e.g dateflag and authroflag so the user gets no feedback about bad data entered. - # so we need to pass the flags explicitly in the url and then extract them from the request in the GET bit. sigh. - response = HttpResponseRedirect(f"/logbookedit/{slug}?dateflag={dateflag}&authorflag={authorflag}") - response.set_cookie('editor_id', editor, max_age=get_cookie_max_age(request)) # cookie expires after get_cookie_max_age(request) seconds - return response - - # Do the redirect instead of this: - # return render( - # request, - # "logbookform.html", - # { - # "form": form, - # "year": year, - # "date": date, "dateflag": dateflag, - # "author": author, "authorflag": authorflag, - # "others": others, - # "place": place, - # "title": title, - # "tu": tu, - # "entry": entry, - # "output": output, - # "slug": slug, - # }, - # ) +# ''' # return render( + # request, + # "logbookform.html", + # { + # "form": form, + # "year": year, + # "date": date, "dateflag": dateflag, + # "author": author, "authorflag": authorflag, + # "others": others, + # "place": place, + # "title": title, + # "tu": tu, + # "entry": entry, + # "output": output, + # "slug": slug, + # }, + # ) # GET here. Does not fall-through from the POST section. else: diff --git a/core/views/logbooks.py b/core/views/logbooks.py index 5c85206ce..8aed855d3 100644 --- a/core/views/logbooks.py +++ b/core/views/logbooks.py @@ -30,8 +30,6 @@ todo = """- Fix the get_person_chronology() display bug. - Fix id= value preservation on editing """ -LOGBOOK_ENTRIES = "log_entries" # directory name - def notablepersons(request): def notabilitykey(person): return person.notability() @@ -247,7 +245,7 @@ def logentrydelete(request, year): """This only gets called by a POST from the logreport page This function is dedicated to James Waite who managed to make so many duplicate logbook entries - that we needed a sopecial mechanism to delete them. + that we needed a special mechanism to delete them. """ for i in request.POST: print(f" - '{i}' {request.POST[i]}") @@ -261,6 +259,7 @@ def logentrydelete(request, year): filename= "logbook.html" try: writelogbook(year, filename) # uses a template + except: message = f'! - Logbook saving failed - \n!! Permissions failure ?! on attempting to save file "logbook.html"' print(message) @@ -368,6 +367,32 @@ def logbookentry(request, date, slug): print(msg) return render(request, "errors/generic.html", {"message": msg}) +def logbookfile(request, year): + """This was just a link to logbook.html, an ordinary HTML file handbook-style, but in Nov.2025 we + changed to have individual JSON files per entry. So this "whole logbook" is now only + re-generated when anyone want to see it. + """ + json_entries_dir = settings.EXPOWEB / "years" / year / settings.JSON_LOG_ENTRIES + exp = Expedition.objects.get(year=year) + if json_entries_dir.is_dir(): # only 2025 currently, or the current expo + # Re-generate the logbook.html because it won't have been done by + # indovidual entry edits for this year + + contents_path = settings.EXPOWEB / "years" / year / exp.logbookfile + try: + print(f" - Logbook for {year} to be exported and written out.") + writelogbook(year, exp.logbookfile, generate_on_view=True) # uses a template + commit_msg = f"Request to see whole logbook triggers re-exporting." + editor = get_editor(request) + add_commit(contents_path, commit_msg, editor) + except: + message = f'! - Logbook saving to file from database failed - \n!! Permissions failure ?! on attempting to save file {contents_path}' + print(message) + raise # got a one-off failure here, we need to know why.. + return render(request, "errors/generic.html", {"message": message}) + + return redirect(f"/years/{year}/{exp.logbookfile}") + def get_people(request, expeditionslug): exp = Expedition.objects.get(year=expeditionslug) @@ -385,11 +410,11 @@ def logbook_entries_export(request, year): entries = get_entries(year) editor = get_editor(request) - write_entries(entries, year, editor) + write_entries_json(entries, year, editor) return redirect(f"/logreport/{year}") -def write_entries(entries, year, editor): +def write_entries_json(entries, year, editor): """Exports logentries from the live database to JSON files. entries - a list, use a list of one member if writing a single entry @@ -438,13 +463,16 @@ def write_entries(entries, year, editor): expedition_data = model_to_dict(le.expedition, fields=['year', 'name']) - entrydict = model_to_dict(le, fields=('slug', 'date', 'title', 'cave', 'place', 'other_people', 'time_underground', 'text')) + entrydict = model_to_dict(le, fields=('slug', 'date', 'title', 'place', 'other_people', 'time_underground', 'text')) entrydict['author'] = author_data entrydict['trippersons'] = participants entrydict['expedition'] = expedition_data + if le.cave: + cave_data = model_to_dict(le.cave, fields=['areacode', 'kataster_number', 'unofficial_number']) + entrydict['cave'] = cave_data return entrydict - dirpath = settings.EXPOWEB / "years" / year / LOGBOOK_ENTRIES + dirpath = settings.EXPOWEB / "years" / year / settings.JSON_LOG_ENTRIES for le in entries: # filename = f"{le.slug}-{le.pk:03}.json" diff --git a/parsers/imports.py b/parsers/imports.py index d553941e4..57a73d59b 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -43,7 +43,7 @@ def import_logbooks(): with transaction.atomic(): troggle.parsers.logbooks.LoadLogbooks() -def import_logbook(year=2024): +def import_logbook(year=2025): print(f"-- Importing Logbook {year}") with transaction.atomic(): troggle.parsers.logbooks.LoadLogbook(year) diff --git a/parsers/logbooks.py b/parsers/logbooks.py index e959767b8..ba98db7d3 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -14,7 +14,7 @@ from django.template.defaultfilters import slugify from parsers.people import GetPersonExpeditionNameLookup, known_foreigner, load_people_expos from typing import Any, List, Tuple -from troggle.core.models.caves import GetCaveLookup +from troggle.core.models.caves import GetCaveLookup, Cave from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition from troggle.core.utils import alphabet_suffix, get_process_memory, unique_slug @@ -248,7 +248,7 @@ def tidy_trip_persons(trippeople, title, expedition, logtime_underground, tid): return trippersons, author, guests def tidy_trip_cave(place): - # GetCaveLookup() need to work better. None of this data is *used* though? + # GetCaveLookup() need to work better. Used in parsing logbooks: place=>cave # 'tripcave' is converted to a string doing this, which renders as the cave slug. lplace = place.lower() @@ -526,8 +526,6 @@ def parser_html(year, expedition, txt, seq=""): entry = LogbookEntryData(ldate, place, tripcave, triptitle, tripcontent, trippersons, author, guests, expedition, tu, tid) logentries.append(entry) - if str(ldate) == "2025-07-08": - print(f"PARSED from html\n",entry,"\n") return logentries @@ -629,6 +627,7 @@ def parser_blog(year, expedition, txt, sq=""): logtime_underground = 0 trippersons, author, guests = tidy_trip_persons(trippeople, triptitle, expedition, logtime_underground, tid) # print(f" - author: {author}") + tripcave = tidy_trip_cave(place) tripcontent = tidy_trip_image_urls(tripcontent, year) tid = tidy_tid(tid, triptitle, datestamp) @@ -691,7 +690,7 @@ def parse_logbook_for_expedition(expedition, blog=False): """ ldate = datetime.fromisoformat(entrydict["date"]).date() place = entrydict["place"] - tripcave = entrydict["cave"] + tripcave=None triptitle = entrydict["title"] tripcontent = entrydict["text"] @@ -699,20 +698,31 @@ def parse_logbook_for_expedition(expedition, blog=False): expedition = Expedition.objects.get(name=entrydict["expedition"]["name"]) tu = entrydict["time_underground"] tid = entrydict["slug"] - + _author_person = Person.objects.get(slug=entrydict["author"]["slug"]) - _author_nickname = entrydict["author"]["nickname"] - _author_tu = entrydict["author"]["tu"] + # author does not have tu or nickname, that info is on the same person in the participants list author = PersonExpedition.objects.get(person=_author_person, expedition=expedition) # not a tuple trippersons = [] for tp in entrydict["trippersons"]: _person = Person.objects.get(slug=tp["slug"]) _personexpo = PersonExpedition.objects.get(person=_person, expedition=expedition) + # if "nickname" not in tp: + # tp["nickname"] = "" + # if "tu" not in tp: + # tp["tu"] = "" trippersons.append((_personexpo,tp["nickname"],tp["tu"])) - + + tripcave = tidy_trip_cave(place) + if "cave" in entrydict: + _cave = Cave.objects.get(areacode=entrydict["cave"]["areacode"], + unofficial_number=entrydict["cave"]["unofficial_number"], + kataster_number=entrydict["cave"]["kataster_number"]) + if tripcave != _cave: + message = f"! MISMATCH between place and Cave: {tripcave=} {_cave=}" + print(message) + DataIssue.objects.update_or_create(parser="logbooks", message=message, url=jsonurl) + logentry = LogbookEntryData(ldate, place, tripcave, triptitle, tripcontent, trippersons, author, guests, expedition, tu, tid) - if entrydict["date"] == "2025-07-08": - print(f"PARSED from JSON\n",logentry,"\n") return logentry @@ -734,15 +744,9 @@ def parse_logbook_for_expedition(expedition, blog=False): expect = ENTRIES[year] # print(" - Logbook for: " + year) - json_entries_dir = settings.EXPOWEB / "years" / year / "log_entries" + json_entries_dir = settings.EXPOWEB / "years" / year / settings.JSON_LOG_ENTRIES + - if json_entries_dir.is_dir(): - print(f" # WARNING year {year} has JSON-encoded logbook entries. Using these instead of the archive .html file.") - logentries = load_from_json() - - logentries = [] # but don't actually use these. - # check_number_of_entries(logentries) - # return logentries if year in LOGBOOK_PARSER_SETTINGS: @@ -764,6 +768,15 @@ def parse_logbook_for_expedition(expedition, blog=False): yearfile, parsefunc = BLOG_PARSER_SETTINGS[year] print(f" - BLOG file {yearfile} using parser {parsefunc}") else: + if json_entries_dir.is_dir(): + print(f" # WARNING year {year} has JSON-encoded logbook entries. Using these instead of the archive .html file.") + logentries = load_from_json() + + check_number_of_entries(logentries) + # we know this is being called for a non-blog from the blog=False setting + # so we can just skip the rest and return. + return logentries + lb = Path(expologbase, year, logbookpath.stem + logbookpath.suffix) if not (lb.is_file()): message = f" ! Logbook file does not exist (yet): '{lb}'" diff --git a/settings.py b/settings.py index 01119d5bd..ef196f9c9 100644 --- a/settings.py +++ b/settings.py @@ -84,6 +84,8 @@ ROOT_URLCONF = "troggle.urls" # i.e. troggle/urls.py LOGOUT_REDIRECT_URL = "/statistics" # see troggle/core/views/auth.py LOGIN_REDIRECT_URL = "/controlpanel" # see troggle/core/views/auth.py +JSON_LOG_ENTRIES="log_entries" + PASSWORD_RESET_TIMEOUT = 3*60*60 # password reset sends an email. The response is valid for 3 hours #ACCOUNT_ACTIVATION_DAYS = 3 # this is only if we are using django-registration package diff --git a/templates/expedition.html b/templates/expedition.html index 9e48850e4..1fe3420bc 100644 --- a/templates/expedition.html +++ b/templates/expedition.html @@ -23,7 +23,7 @@
  • documentation index for this Expo
  • wallet completion status for this Expo
  • alias names for this Expo -
  • full logbook for this Expo +
  • full logbook for this Expo
  • logbook report for this Expo
  • new logbook entry for this Expo diff --git a/templates/logbook2005style.html b/templates/logbook2005style.html index 971c3e47f..f022d856f 100644 --- a/templates/logbook2005style.html +++ b/templates/logbook2005style.html @@ -7,7 +7,11 @@ - {%for logbook_entry in logbook_entries%}
    diff --git a/templates/logbookentry.html b/templates/logbookentry.html index 4b0b074df..2f7bfb383 100644 --- a/templates/logbookentry.html +++ b/templates/logbookentry.html @@ -9,7 +9,7 @@

    {{logbookentry.expedition.year}} calendar page -    full logbook +    full logbook     logbook report

    diff --git a/templates/logbookform.html b/templates/logbookform.html index 28a28c882..0179cb3ba 100644 --- a/templates/logbookform.html +++ b/templates/logbookform.html @@ -116,7 +116,7 @@


    {% if date %}

    Link to this entry   {% endif %} - Full logbook: Logbook {{year}} + Full logbook: Logbook {{year}}     {{year}} Logbook report @@ -128,7 +128,8 @@ {% if output %}

    -Click this triangle to see the HTML which has been put into logbook.html, and below that is the rendered logbook entry. +Click this triangle to see the HTML which has been saved (either put into logbook.html or, for the current expo, put into +an individual JSON file), and below that is the rendered logbook entry. 
     {{output}}
diff --git a/templates/logreport.html b/templates/logreport.html
index c13d94131..87b027c62 100644
--- a/templates/logreport.html
+++ b/templates/logreport.html
@@ -21,7 +21,7 @@
 
    (Hover mouse over the date to see the slug for the entry.)
 {% if logged_in %}Logged in as expoadmin
    
 Export logbook entries as individual files
-    
+This will also export any BLOG entries which have been parsed from, e.g. BLOG_PARSER_SETTINGS
 {% endif %}
 
 
@@ -83,9 +83,9 @@
 
  •  documentation index for this Expo 
 
  •  wallet completion status for this Expo 
 
  •  alias names for this Expo
-
  •  full logbook for this Expo
+
  •  full logbook for this Expo
 
  •  new logbook entry for this Expo
-{% if logged_in %}
  • export logbook entries  as individual files in /years/{{expedition.year}}/entries/
+{% if logged_in %}
  • export logbook entries  as individual files in /years/{{expedition.year}}/entries/. This will also export any BLOG entries which have been parsed from, e.g. BLOG_PARSER_SETTINGS
 {% endif %}
 
 
diff --git a/urls.py b/urls.py
index d5c287326..2ad4c1e4f 100644
--- a/urls.py
+++ b/urls.py
@@ -55,6 +55,7 @@ from troggle.core.views.logbooks import (
     logbookentry,
     logentrydelete,
     logreport,
+    logbookfile,
     notablepersons,
     people_ids,
     person,
@@ -244,6 +245,7 @@ trogglepatterns = [
     path('logbook_entries/', logbook_entries_export, name='logbook_entries_export'), 
     path('logreport/', logreport, name='logreport'), 
     path('logentrydelete/', logentrydelete, name='logentrydelete'), 
+    path('logbookfile/', logbookfile, name='logbookfile'), 
    
 #   Internal. editfile.html template uses these internally
     re_path(r'^getPeople/(?P.*)', get_people, name = "get_people"),