SHould be nearly working, but crashes on saving edited entry

core/models/logbooks.py

@@ -107,7 +107,31 @@ class LogbookEntry(TroggleModel):
             index = index % mx
         return index
 
-def writelogbook(year, filename):
+READ_THIS="""<!--
+READTHIS
+
+  ######                          #######                 
+ #     # ######   ##   #####        #    #    # #  ####  
+ #     # #       #  #  #    #       #    #    # # #      
+ ######  #####  #    # #    #       #    ###### #  ####  
+ #   #   #      ###### #    #       #    #    # #      # 
+ #    #  #      #    # #    #       #    #    # # #    # 
+ #     # ###### #    # #####        #    #    # #  ####  
+                                          
+THIS FILE IS NOT THE MASTER. 
+ANY EDITS YOU MAKE HERE WILL BE OVERWRITTEN.
+
+The MASTER record for this expo is the individual JSON files in
+expoweb/years/2025/log_entries/*.json
+In due course, when this expo is archived, the JSON files will be deleted
+and this file will beocme the master again, and this warning will be deleted.
+
+Meanwhile, use the online logbook entry editor to make changes.
+
+READTHIS
+-->                                        
+"""
+def writelogbook(year, filename, generate_on_view=False):
     """Writes all the database logbook entries into a new logbook.html file
     """
     current_expedition = Expedition.objects.get(year=year)
@@ -140,6 +164,9 @@ def writelogbook(year, filename):
             except:
                 print("   ! Very Bad Error opening " + endpath)
                 raise
+        if generate_on_view:
+            endmatter += READ_THIS
+
     except:
         print("   ! FAIL endpath " + endpath)
         raise

core/utils.py

@@ -405,8 +405,11 @@ def git_commit(cwd, message, editor, commands=[]):
     print(f"..{message=}\n..{editor=}")
     cmd_commit = [git, "commit", "-m", message, "--author", f"{editor}"]
     commands.append(cmd_commit)
-
+    print(commands)
+    try:
         cp_commit = subprocess.run(cmd_commit, cwd=cwd, capture_output=True, text=True)
+    except Exception as e:
+        print(e)
     # This produces return code = 1 if it commits OK, but when the local repo still needs to be pushed to origin/repo
     # which will be the case when running a test troggle system on a development machine
     

core/views/logbook_edit.py

@@ -25,6 +25,7 @@ from troggle.core.utils import (
     write_and_commit,
     wrap_text,
 )
+from troggle.core.views.logbooks import write_entries_json
 from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner
 
 # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
@@ -301,7 +302,13 @@ def logbookedit(request, year=None, slug=None):
                 return render(request, "errors/generic.html", {"message": message})
             store_edited_entry_into_database(date, place, title, entry, others, author, tu, slug)
 
-                
+            # Successful POST so save to fiesystem
+            json_entries_dir = settings.EXPOWEB / "years" / year / settings.JSON_LOG_ENTRIES
+            if json_entries_dir.is_dir(): # only 2025 currently, or the current expo
+                print(f"- Rewriting JUST this edited logbook entry to a JSON file. ")
+                this_entry = LogbookEntry.objects.get(slug=slug)
+                write_entries_json([this_entry], year, editor)
+            else:
                 print(f"- Rewriting the entire {year} logbook to disc ")
                 filename= "logbook.html"
                 try:
@@ -312,20 +319,7 @@ def logbookedit(request, year=None, slug=None):
                     print(message)
                     return render(request, "errors/generic.html", {"message": message})
             
-            # Code fragment illustration - not actually what gets saved to database
-            output = f'''
 
-<div class="tripdate" id="{slug}">{date}</div>
-<div class="trippeople"><u>{author}</u>, {others}</div>
-<div class="triptitle">{place} - {title}</div>
-
-{entry}
-
-<div class="timeug">T/U {tu} hrs</div>
-<hr />
-
-'''
-            # Successful POST
                 # So save to database and then write out whole new logbook.html file
                 
                 # We do author validation on the form as displayed by GET, not at the moment of POST.
@@ -348,7 +342,20 @@ def logbookedit(request, year=None, slug=None):
                 return response
                 
                 # Do the redirect instead of this:
-            # return render(
+                
+                # Code fragment illustration - not actually what gets saved to database
+                # output = f'''
+
+# <div class="tripdate" id="{slug}">{date}</div>
+# <div class="trippeople"><u>{author}</u>, {others}</div>
+# <div class="triptitle">{place} - {title}</div>
+
+# {entry}
+
+# <div class="timeug">T/U {tu} hrs</div>
+# <hr />
+
+# '''                # return render(
                     # request,
                     # "logbookform.html",
                     # {

core/views/logbooks.py

@@ -30,8 +30,6 @@ todo = """- Fix the get_person_chronology() display bug.
 - Fix id= value preservation on editing
 """
 
-LOGBOOK_ENTRIES = "log_entries" # directory name
-
 def notablepersons(request):
     def notabilitykey(person):
         return person.notability()
@@ -247,7 +245,7 @@ def logentrydelete(request, year):
     """This only gets called by a POST from the logreport page
     
     This function is dedicated to James Waite who managed to make so many duplicate  logbook entries
-    that we needed a sopecial mechanism to delete them.
+    that we needed a special mechanism to delete them.
     """
     for i in request.POST:
         print(f" - '{i}'  {request.POST[i]}")
@@ -261,6 +259,7 @@ def logentrydelete(request, year):
     filename= "logbook.html"   
     try:
         writelogbook(year, filename) # uses a template
+        
     except:
         message = f'! - Logbook saving failed - \n!! Permissions failure ?! on attempting to save file "logbook.html"'
         print(message)
@@ -368,6 +367,32 @@ def logbookentry(request, date, slug):
         print(msg)
         return render(request, "errors/generic.html", {"message": msg})
 
+def logbookfile(request, year):
+    """This was just a link to logbook.html, an ordinary HTML file handbook-style, but in Nov.2025 we 
+    changed to have individual JSON files per entry. So this "whole logbook" is now only
+    re-generated when anyone want to see it.
+    """
+    json_entries_dir = settings.EXPOWEB / "years" / year / settings.JSON_LOG_ENTRIES
+    exp = Expedition.objects.get(year=year)
+    if json_entries_dir.is_dir(): # only 2025 currently, or the current expo
+        # Re-generate the logbook.html because it won't have been done by
+        # indovidual entry edits for this year
+
+        contents_path = settings.EXPOWEB / "years" / year / exp.logbookfile
+        try:
+            print(f"  - Logbook for {year} to be exported and written out.")
+            writelogbook(year, exp.logbookfile, generate_on_view=True) # uses a template
+            commit_msg = f"Request to see whole logbook triggers re-exporting."
+            editor = get_editor(request)      
+            add_commit(contents_path, commit_msg, editor)     
+        except:
+            message = f'! - Logbook saving to file from database failed - \n!! Permissions failure ?! on attempting to save file {contents_path}'
+            print(message)
+            raise # got a one-off failure here, we need to know why..
+            return render(request, "errors/generic.html", {"message": message})
+  
+    return redirect(f"/years/{year}/{exp.logbookfile}")
+
 
 def get_people(request, expeditionslug):
     exp = Expedition.objects.get(year=expeditionslug)
@@ -385,11 +410,11 @@ def logbook_entries_export(request, year):
     
     entries = get_entries(year)
     editor = get_editor(request)
-    write_entries(entries, year, editor)
+    write_entries_json(entries, year, editor)
     return redirect(f"/logreport/{year}")
 
 
-def write_entries(entries, year, editor):  
+def write_entries_json(entries, year, editor):  
     """Exports logentries from the live database to JSON files.
     
     entries - a list, use a list of one member if writing a single entry
@@ -438,13 +463,16 @@ def write_entries(entries, year, editor):
             
         expedition_data = model_to_dict(le.expedition, fields=['year', 'name'])
 
-        entrydict = model_to_dict(le, fields=('slug', 'date', 'title', 'cave', 'place', 'other_people', 'time_underground', 'text'))
+        entrydict = model_to_dict(le, fields=('slug', 'date', 'title', 'place', 'other_people', 'time_underground', 'text'))
         entrydict['author'] = author_data
         entrydict['trippersons'] = participants
         entrydict['expedition'] = expedition_data
+        if le.cave:
+            cave_data = model_to_dict(le.cave, fields=['areacode', 'kataster_number', 'unofficial_number'])
+            entrydict['cave'] = cave_data
         return entrydict
         
-    dirpath = settings.EXPOWEB / "years" / year / LOGBOOK_ENTRIES
+    dirpath = settings.EXPOWEB / "years" / year / settings.JSON_LOG_ENTRIES
             
     for le in entries:        
         # filename = f"{le.slug}-{le.pk:03}.json"

parsers/imports.py

@@ -43,7 +43,7 @@ def import_logbooks():
     with transaction.atomic():
         troggle.parsers.logbooks.LoadLogbooks()
 
-def import_logbook(year=2024):
+def import_logbook(year=2025):
     print(f"-- Importing Logbook {year}")
     with transaction.atomic():
         troggle.parsers.logbooks.LoadLogbook(year)

parsers/logbooks.py

@@ -14,7 +14,7 @@ from django.template.defaultfilters import slugify
 
 from parsers.people import GetPersonExpeditionNameLookup, known_foreigner, load_people_expos
 from typing import Any, List, Tuple
-from troggle.core.models.caves import GetCaveLookup
+from troggle.core.models.caves import GetCaveLookup, Cave
 from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry
 from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition
 from troggle.core.utils import alphabet_suffix, get_process_memory, unique_slug
@@ -248,7 +248,7 @@ def tidy_trip_persons(trippeople, title, expedition, logtime_underground, tid):
     return trippersons, author, guests
     
 def tidy_trip_cave(place):
-    #  GetCaveLookup() need to work  better. None of this data is *used* though?
+    #  GetCaveLookup() need to work  better. Used in parsing logbooks: place=>cave
     # 'tripcave' is converted to a string doing this, which renders as the cave slug.
  
     lplace = place.lower()
@@ -526,8 +526,6 @@ def parser_html(year, expedition, txt, seq=""):
    
         entry = LogbookEntryData(ldate, place, tripcave, triptitle, tripcontent, trippersons, author, guests, expedition, tu, tid)
         logentries.append(entry)
-        if str(ldate) == "2025-07-08":
-            print(f"PARSED from html\n",entry,"\n")
     return logentries
 
 
@@ -629,6 +627,7 @@ def parser_blog(year, expedition, txt, sq=""):
         logtime_underground = 0
         trippersons, author, guests = tidy_trip_persons(trippeople, triptitle, expedition, logtime_underground, tid)
         # print(f" -  author: {author}")
+        
         tripcave = tidy_trip_cave(place)
         tripcontent = tidy_trip_image_urls(tripcontent, year)
         tid = tidy_tid(tid, triptitle, datestamp)
@@ -691,7 +690,7 @@ def parse_logbook_for_expedition(expedition, blog=False):
         """
         ldate = datetime.fromisoformat(entrydict["date"]).date()
         place = entrydict["place"]
-        tripcave = entrydict["cave"]
+        tripcave=None
         triptitle = entrydict["title"]
         tripcontent = entrydict["text"]
         
@@ -701,18 +700,29 @@ def parse_logbook_for_expedition(expedition, blog=False):
         tid = entrydict["slug"] 
                
         _author_person = Person.objects.get(slug=entrydict["author"]["slug"])
-        _author_nickname = entrydict["author"]["nickname"]
+        # author does not have tu or nickname, that info is on the same person in the participants list
-        _author_tu = entrydict["author"]["tu"]
         author = PersonExpedition.objects.get(person=_author_person, expedition=expedition) # not a tuple
         trippersons = []
         for tp in entrydict["trippersons"]:
             _person = Person.objects.get(slug=tp["slug"])
             _personexpo = PersonExpedition.objects.get(person=_person, expedition=expedition)
+            # if "nickname" not in tp:
+                # tp["nickname"] = ""
+            # if  "tu" not in tp:
+                # tp["tu"] = ""
             trippersons.append((_personexpo,tp["nickname"],tp["tu"]))
             
+        tripcave = tidy_trip_cave(place)
+        if "cave" in entrydict:
+            _cave = Cave.objects.get(areacode=entrydict["cave"]["areacode"], 
+                unofficial_number=entrydict["cave"]["unofficial_number"],
+                kataster_number=entrydict["cave"]["kataster_number"])
+            if tripcave != _cave:
+                message = f"! MISMATCH between place and Cave: {tripcave=} {_cave=}"
+                print(message)
+                DataIssue.objects.update_or_create(parser="logbooks", message=message, url=jsonurl)
+                        
         logentry = LogbookEntryData(ldate, place, tripcave, triptitle, tripcontent, trippersons, author, guests, expedition, tu, tid)
-        if entrydict["date"] == "2025-07-08":
-            print(f"PARSED from JSON\n",logentry,"\n")
         
         return logentry
         
@@ -734,15 +744,9 @@ def parse_logbook_for_expedition(expedition, blog=False):
     expect = ENTRIES[year]
     # print(" - Logbook for: " + year)
     
-    json_entries_dir = settings.EXPOWEB / "years" / year / "log_entries"
+    json_entries_dir = settings.EXPOWEB / "years" / year / settings.JSON_LOG_ENTRIES
 
-    if json_entries_dir.is_dir():
-        print(f" # WARNING year {year} has JSON-encoded logbook entries. Using these instead of the archive .html file.")
-        logentries = load_from_json()
 
-        logentries = [] # but don't actually use these.
-        # check_number_of_entries(logentries)
-        # return logentries
 
 
     if year in LOGBOOK_PARSER_SETTINGS:
@@ -764,6 +768,15 @@ def parse_logbook_for_expedition(expedition, blog=False):
             yearfile, parsefunc = BLOG_PARSER_SETTINGS[year]
         print(f" - BLOG file {yearfile} using parser {parsefunc}")
     else:
+        if json_entries_dir.is_dir():
+            print(f" # WARNING year {year} has JSON-encoded logbook entries. Using these instead of the archive .html file.")
+            logentries = load_from_json()
+            
+            check_number_of_entries(logentries)
+            # we know this is being called for a non-blog from the blog=False setting
+            # so we can just skip the rest and return.
+            return logentries
+        
         lb = Path(expologbase, year, logbookpath.stem + logbookpath.suffix) 
         if not (lb.is_file()):
             message = f"   ! Logbook file does not exist (yet): '{lb}'"

settings.py

@@ -84,6 +84,8 @@ ROOT_URLCONF = "troggle.urls"  # i.e. troggle/urls.py
 LOGOUT_REDIRECT_URL = "/statistics"   # see troggle/core/views/auth.py
 LOGIN_REDIRECT_URL = "/controlpanel"  # see troggle/core/views/auth.py
 
+JSON_LOG_ENTRIES="log_entries"
+
 PASSWORD_RESET_TIMEOUT = 3*60*60      # password reset sends an email. The response is valid for 3 hours
 #ACCOUNT_ACTIVATION_DAYS = 3 # this is only if we are using django-registration package
 

templates/expedition.html

@@ -23,7 +23,7 @@
 <li> <a href="/years/{{expedition.year}}/">documentation index</a> for this Expo 
 <li> <a href="/wallets/year/{{expedition.year}}">wallet completion status</a> for this Expo 
 <li> <a href="/aliases/{{expedition.year}}">alias names</a> for this Expo
-<li> <a href="/years/{{expedition.year}}/{{expedition.logbookfile}}">full logbook</a> for this Expo
+<li> <a href="/logbookfile/{{expedition.year}}">full logbook</a> for this Expo
 <li> <a href="/logreport/{{expedition.year}}">logbook report</a> for this Expo
 <li> <a href="/logbookedit/">new logbook entry</a> for this Expo
 </ul>

templates/logbook2005style.html

@@ -7,7 +7,11 @@
 <meta name="keywords" content="NOEDIT">
 <style>figure {font-weight: bold; font-size: small; font-family: sans-serif;font-variant-caps: small-caps;}</style>
 </head>
-<!-- Exported by troggle because one entry has been edited, and we reconstruct the whole thing. 
+<!-- Exported by troggle from the live database and we reconstruct the whole thing,
+either
+because one entry has been edited, 
+or
+this year is currently managed with JSON files and someone has asked to see this whole file.
 
 As of 10 Dec.2024 the list of participants in every trip is re-ordered to be in alphabetic
 order of name (or nickname, if used) whenever any entry is edited and the logbook.html file 
@@ -16,8 +20,10 @@ re-written. This is prevent spurious re-orderings and spurious git commit lines
 Sorry about all the crap that surrounds the image tags which has been imported along with the content
 when UK Caving blogs have been parsed.
 
-Exported on {% now 'Y-m-d H:m' %} using either the control panel webpage or when editing a logbook entry online
+Exported on {% now 'Y-m-d H:m' %} using either the control panel webpage or when editing a logbook entry online,
+or (JSON files year only) because someone has requested to see this whole file.
 See troggle/code/views/other.py and core.models/logbooks.py writelogbook(year, filename)
+The template used to generate this page is troggle/templates/logbook2005style.html
 -->
 <body>
 {%for logbook_entry in logbook_entries%} <hr />

templates/logbookentry.html

@@ -9,7 +9,7 @@
 
 <div id="relatedentries">
 <p><a href="{{ logbookentry.expedition.get_absolute_url }}">{{logbookentry.expedition.year}} calendar page</a>
-&nbsp;&nbsp;&nbsp;&nbsp;<a href="/years/{{logbookentry.expedition.year}}/{{logbookentry.expedition.logbookfile}}">full logbook</a>
+&nbsp;&nbsp;&nbsp;&nbsp;<a href="/logbookfile/{{logbookentry.expedition.year}}">full logbook</a>
 &nbsp;&nbsp;&nbsp;&nbsp;<a href="/logreport/{{logbookentry.expedition.year}}">logbook report</a>
 </p>
 

templates/logbookform.html

@@ -116,7 +116,7 @@
 
     <br /><br /><br />
 	{% if date %}<p>Link to <em><a href="/logbookentry/{{date}}/{{slug}}#">this entry</a></em> &nbsp;&nbsp;{% endif %}
-        Full logbook: <a href="/years/{{year}}/logbook.html"><em>Logbook {{year}}</em></a>
+        Full logbook: <a href="/logbookfile/{{year}}"><em>Logbook {{year}}</em></a>
 &nbsp;&nbsp;&nbsp;&nbsp;{{year}} <a href="/logreport/{{year}}">Logbook report</a>
 			
  
@@ -128,7 +128,8 @@
 
 {% if output %}
 <details><summary>
-Click this triangle to see the HTML which has been put into logbook.html, and below that is the rendered logbook entry.
+Click this triangle to see the HTML which has been saved (either put into logbook.html or, for the current expo, put into
+an individual JSON file), and below that is the rendered logbook entry.
 </summary>
 <pre>
 {{output}}

templates/logreport.html

@@ -21,7 +21,7 @@
 <p>(Hover mouse over the date to see the slug for the entry.)
 {% if logged_in %}<font color="red">Logged in as expoadmin<br />
 <a href="/logbook_entries/{{expedition.year}}">Export logbook entries as individual files</a>
-</font>
+This will also export any BLOG entries which have been parsed from, e.g. BLOG_PARSER_SETTINGS</font>
 {% endif %}
 
 
@@ -83,9 +83,9 @@
 <li> <a href="/years/{{expedition.year}}/">documentation index</a> for this Expo 
 <li> <a href="/wallets/year/{{expedition.year}}">wallet completion status</a> for this Expo 
 <li> <a href="/aliases/{{expedition.year}}">alias names</a> for this Expo
-<li> <a href="/years/{{expedition.year}}/{{expedition.logbookfile}}">full logbook</a> for this Expo
+<li> <a href="/logbookfile/{{expedition.year}}">full logbook</a> for this Expo
 <li> <a href="/logbookedit/">new logbook entry</a> for this Expo
-{% if logged_in %}<li><font color="red"><a href="/logbook_entries/{{expedition.year}}">export logbook entries</a>  as individual files in /years/{{expedition.year}}/entries/</font>
+{% if logged_in %}<li><font color="red"><a href="/logbook_entries/{{expedition.year}}">export logbook entries</a>  as individual files in /years/{{expedition.year}}/entries/. This will also export any BLOG entries which have been parsed from, e.g. BLOG_PARSER_SETTINGS</font>
 {% endif %}
 </ul>
 

urls.py

@@ -55,6 +55,7 @@ from troggle.core.views.logbooks import (
     logbookentry,
     logentrydelete,
     logreport,
+    logbookfile,
     notablepersons,
     people_ids,
     person,
@@ -244,6 +245,7 @@ trogglepatterns = [
     path('logbook_entries/<slug:year>', logbook_entries_export, name='logbook_entries_export'), 
     path('logreport/<slug:year>', logreport, name='logreport'), 
     path('logentrydelete/<slug:year>', logentrydelete, name='logentrydelete'), 
+    path('logbookfile/<slug:year>', logbookfile, name='logbookfile'), 
    
 #   Internal. editfile.html template uses these internally
     re_path(r'^getPeople/(?P<expeditionslug>.*)', get_people, name = "get_people"),