expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-21 23:01:52 +00:00
Code Issues 19 Projects 3 Activity

Edit Logbook Entry mostly working

Browse Source
This commit is contained in:
Philip Sargent 2023-08-31 18:55:20 +03:00
parent bbb821e2f9
commit c29c12ea76
7 changed files with 276 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
core/models/logbooks.py
View File

@ -1,8 +1,11 @@
import re
from pathlib import Path
from urllib.parse import urljoin
from django.db import models
from django.urls import reverse
from django.template import loader
import settings
from troggle.core.models.troggle import Expedition, TroggleModel
@ -50,7 +53,7 @@ class LogbookEntry(TroggleModel):
max_length=100, blank=True, null=True, help_text="Only use this if you haven't chosen a cave"
)
text = models.TextField()
slug = models.SlugField(max_length=50)
slug = models.SlugField(max_length=50) # this is tripid
time_underground = models.FloatField(null=True, help_text="In decimal hours")
class Meta:
@ -93,7 +96,47 @@ class LogbookEntry(TroggleModel):
index = index % mx
return index
def writelogbook(year, filename):
current_expedition = Expedition.objects.get(year=year)
logbook_entries = LogbookEntry.objects.filter(expedition=current_expedition).order_by(
"slug"
) # now that slug, aka tripid, is in our standard date form, this will preserve ordering.
print(f"Logbook exported has {len(logbook_entries)} entries in it.")
extension = "html"
template = "logbook2005style.html"
t = loader.get_template(template)
logbookfile = t.render({"logbook_entries": logbook_entries})
endpath = Path(settings.EXPOWEB, "years", year, "endmatter.html")
endmatter = ""
if endpath.is_file():
try:
with open(endpath, "r") as end:
endmatter = end.read()
except:
print(" ! Very Bad Error opening " + endpath)
frontpath = Path(settings.EXPOWEB, "years", year, "frontmatter.html")
if frontpath.is_file():
try:
with open(frontpath, "r") as front:
frontmatter = front.read()
except:
print(" ! Very Bad Error opening " + frontpath)
logbookfile = re.sub(r"<body>", "<body>\n" + frontmatter + endmatter, logbookfile)
else:
logbookfile = re.sub(r"<body>", f"<body>\n<h1>Expo {year}</h1>\n" + endmatter, logbookfile)
dir = Path(settings.EXPOWEB) / "years" / year
filepath = Path(dir, filename)
with (open(filepath, "w")) as lb:
lb.writelines(logbookfile)
# print(f'Logbook exported to {filepath}')
class PersonLogEntry(TroggleModel):
"""Single Person going on a trip, which may or may not be written up.
It could account for different T/U for people in same logbook entry.

62
core/views/other.py
View File

@ -7,7 +7,7 @@ from django.shortcuts import render
from django.template import loader
from troggle.core.models.caves import Cave
from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
from troggle.core.models.logbooks import LogbookEntry, writelogbook # , PersonLogEntry
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import Expedition
@ -169,69 +169,31 @@ def controlpanel(request):
)
def exportlogbook(request, year=None, extension=None):
def exportlogbook(request, year=None):
"""Constructs, from the database, a complete HTML formatted logbook
for the current year. Formats available are HTML2005. Other formats
have been retired.
for the current year. Format available is now just HTML2005.
Other formats have been retired.
There are no images stored in the database. However links to images work in the HTML text of a logbook entry.
This function is the recipient of the POST action os the export form in the control panel
This function is the recipient of the POST action as the export form in the control panel
"""
def lbeKey(lbe):
"""This function goes into a lexicographic sort function"""
return str(lbe.date)
"""This function goes into a lexicographic sort function - but where?!"""
return str(lbe.slug) # now that slugs are tripid such as 2023-07-30b
if not request.method == "POST":
return render(request, "controlPanel.html", {"expeditions": Expedition.objects.all(), "jobs_completed": ""})
else:
print(f"Logbook export {request.POST}")
# print(f"Logbook export {request.POST}")
year = request.POST["year"]
current_expedition = Expedition.objects.get(year=year)
logbook_entries = LogbookEntry.objects.filter(expedition=current_expedition).order_by(
"date"
) # need to be sorted by date!
filename = "logbook-new-format.html"
print(f"Logbook has {len(logbook_entries)} entries in it.")
extension = "html"
response = HttpResponse(content_type="text/html")
style = "2005"
filename = "logbook-new-format." + extension
template = "logbook" + style + "style." + extension
response["Content-Disposition"] = "attachment; filename=" + filename
t = loader.get_template(template)
logbookfile = t.render({"logbook_entries": logbook_entries})
endpath = Path(settings.EXPOWEB, "years", year, "endmatter.html")
endmatter = ""
if endpath.is_file():
try:
with open(endpath, "r") as end:
endmatter = end.read()
except:
print(" ! Very Bad Error opening " + endpath)
frontpath = Path(settings.EXPOWEB, "years", year, "frontmatter.html")
if frontpath.is_file():
try:
with open(frontpath, "r") as front:
frontmatter = front.read()
except:
print(" ! Very Bad Error opening " + frontpath)
logbookfile = re.sub(r"<body>", "<body>\n" + frontmatter + endmatter, logbookfile)
else:
logbookfile = re.sub(r"<body>", f"<body>\n<h1>Expo {year}</h1>\n" + endmatter, logbookfile)
dir = Path(settings.EXPOWEB) / "years" / year
filepath = Path(dir, filename)
with (open(filepath, "w")) as lb:
lb.writelines(logbookfile)
# print(f'Logbook exported to {filepath}')
writelogbook(year, filename)
#response = HttpResponse(content_type="text/html")
#response["Content-Disposition"] = "attachment; filename=" + filename
completed = f'Logbook exported to <a href="/years/{filename}">{filename}</a>'
return render(

213
core/views/uploads.py
View File

@ -1,5 +1,6 @@
import subprocess
import hashlib
import string
from datetime import datetime
from pathlib import Path
@ -8,8 +9,11 @@ from django.core.files.storage import FileSystemStorage
from django.shortcuts import render, redirect
import settings
from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry
from troggle.core.models.caves import GetCaveLookup
from troggle.core.models.logbooks import LogbookEntry, writelogbook, PersonLogEntry
from troggle.core.models.survex import DrawingFile
from troggle.core.models.troggle import DataIssue, Expedition, PersonExpedition
from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
@ -21,11 +25,6 @@ and that core/forms.py contains Django class-based forms for caves and entrances
"""
todo = """
- munge the URL of images in the logbook entry so that they work from both the " /logbookedit/" page,
the logbook /years/2023/ page and the logbook fragment page /logbookentry/<date>/
Note that this munging has already been done when the entry is imported into the database, so
when doing an online edit it has already been fixed.
- Ideally we should validate uploaded file as being a valid file type, not a dubious script or hack
Validate image files using a magic recogniser in walletedit()
https://pypi.org/project/reportlab/ or
@ -48,6 +47,89 @@ todo = """
"""
sha = hashlib.new('sha256')
def unique_slug(text, n):
"""This gives each logbook entry a unique id based on the date+content, so the order of entries on a particular day
does not matter. This is a change (August 2023) from previous process.
2 hex digits would seem adequate for each expo day, but we might get a collision.
The hash is based on the content after substitution of <p> so should be stable. Which means these ids
can be used elsewhere in the troggle system as permanent slugs.
When SAVING an edited entry (as opposed to a new one) we will have a different hash so we will have to
delete the original database object
"""
sha.update(text.encode('utf-8'))
return sha.hexdigest()[0:n]
def create_new_lbe_slug(date):
onthisdate = LogbookEntry.objects.filter(date=date)
n = len(onthisdate)
# print(f" Already entries on this date: {n}\n {onthisdate}")
alphabet = list(string.ascii_lowercase)
tid = f"{date}{alphabet[n]}"
print(tid)
return tid
def store_edited_entry_into_database(date, place, title, text, others, author, tu, slug):
"""saves a single logbook entry and related personlogentry items
Rather similar to similarly named function in parsers/logbooks but circular reference prevents us using it directly,
and they need refactoring anyway.
"""
year = slug[0:4]
expedition = Expedition.objects.get(year=year)
cave = GetCaveLookup().get(place.lower())
# print(f"{place} {cave=}")
if LogbookEntry.objects.filter(slug=slug).exists():
# oops.
message = " ! - DUPLICATE SLUG for logbook entry " + tripdate + " - " + slug
DataIssue.objects.create(parser="logbooks", message=message)
slug = slug + "_" + unique_slug(text,2)
nonLookupAttribs = {
"place": place,
"text": text,
"expedition": expedition,
"time_underground": tu,
"cave_slug": str(cave),
}
lookupAttribs = {"slug": slug, "date": date, "title": title}
lbo = LogbookEntry.objects.create(**nonLookupAttribs, **lookupAttribs)
pt_list = []
# These entities have to be PersonExpedition objects
team = others.split(",")
team.append(author)
for name in team:
name = name.strip()
if name[0] != "*": # a name prefix of "*" is special, just a string.
try:
personyear = GetPersonExpeditionNameLookup(expedition).get(name.lower())
if not personyear:
if known_foreigner(name):
message = f" ! - Known foreigner: '{name}' in entry {slug=}"
print(message)
else:
message = f" ! - No name match for: '{name}' in entry {slug=}"
print(message)
DataIssue.objects.create(parser="logbooks", message=message)
else:
lookupAttribs = {"personexpedition": personyear, "nickname_used": name, "logbook_entry": lbo} # lbo is primary key
nonLookupAttribs = {"time_underground": tu, "is_logbook_entry_author": (name==author)}
pt_list.append(PersonLogEntry(**nonLookupAttribs, **lookupAttribs))
except:
# This should not happen. We do not raise exceptions in that function
message = f" ! - EXCEPTION: '{name}' in entry {slug=}"
print(message)
DataIssue.objects.create(parser="logbooks", message=message)
raise
PersonLogEntry.objects.bulk_create(pt_list)
class FilesForm(forms.Form): # not a model-form, just a form-form
uploadfiles = forms.FileField()
@ -70,9 +152,7 @@ def logbookedit(request, year=None, slug=None):
"""Edit a logbook entry
This is daft: we have the parsed identity of the person and we render it to text as 'nickname_used'
(or, previously, 'fullname'), to be re-parsed on re-importing.
And there is no guarantee that this will be the same thing.
Someone can put in a nickname which is invalid (e.g. 2 Sophies on expo). When is this checked?
And there is no guarantee that this will be the same thing. Oh well.
"""
def clean_tu(tu):
if tu =="":
@ -82,28 +162,14 @@ def logbookedit(request, year=None, slug=None):
except:
return 0
return tu
def unique_id(text, n):
"""This gives each logbook entry a unique id based on the date+content, so the order of entries on a particular day
does not matter. This is a change (August 2023) from previous process.
Otherwise we could get 2023-07-20a and 2023-07-20b swapped on exporting and re-importing logbooks
because the database does not record precedence.
2 hex digits would seem adequate for each expo day, but we might get a collision.
The hash is based on the content after substitution of <p> so should be stable. Which means these ids
can be used elsewhere in the troggle system as permanent slugs.
When SAVING an edited entry (as opposed to a new one) we will have a different hash so we will have to
delete the original database object
"""
sha.update(text.encode('utf-8'))
return sha.hexdigest()[0:n]
if not year:
if not slug:
year = 2023
year = 2023 # we need a CURRENT_EXPO() function, we use this in a lot of places..
else:
year = slug[0:4]
print(year)
author = ""
if request.method == "POST":
form = LogbookEditForm(request.POST)
@ -112,11 +178,13 @@ def logbookedit(request, year=None, slug=None):
print(message)
return render(request, "errors/generic.html", {"message": message})
else:
# if there is no slug then this is a completely new lbe and we need to enter it into the db
# otherwise it is an update
# validation all to be done yet..
date = request.POST["date"].strip()
author = request.POST["author"].strip() # TODO check against personexpedition
others = request.POST["others"].strip() # TODO check each against personexpedition
place = request.POST["place"].strip().replace('-','=') # no hyphens !
place = request.POST["place"].strip().replace(' - ',' = ') # no hyphens !
title = request.POST["title"].strip()
entry = request.POST["text"].strip()
entry = entry.replace('\r','') # remove HTML-standard CR inserted
@ -135,16 +203,30 @@ def logbookedit(request, year=None, slug=None):
dateflag = True
date = odate.isoformat()
newslug = f"{date}_{unique_id(entry,2)}"
if slug:
if slug != newslug:
print(f"! Entry id changed! from {slug} to {newslug}")
if not slug:
# Creating a new logbook entry with all the gubbins
slug = create_new_lbe_slug(date)
else:
# OK we could patch the object in place, but if the people on the trip have changed this
# would get very messy. So we delete it and recreate it and all its links
print(f"- Deleting the LogBookEntry {slug}")
LogbookEntry.objects.filter(slug=slug).delete()
print(f"- Creating the LogBookEntry {slug}")
store_edited_entry_into_database(date, place, title, entry, others, author, tu, slug)
print(f"- Rewriting the entire {year} logbook to disc ")
filename= "logbook.html"
try:
writelogbook(year, filename) # uses a template, not the code fragment below
except:
message = f'! - Logbook saving failed - \n!! Permissions failure ?! on attempting to save file "logbook.html"'
print(message)
return render(request, "errors/generic.html", {"message": message})
# OK this could be done by rendering a template, but for such a small bit of HTML, it is easier to have
# it all in one place: here
# Code fragment illustration - not actually what gets saved to database
output = f'''
<div class="tripdate" id="{newslug}">{date}</div>
<div class="tripdate" id="{slug}">{date}</div>
<div class="trippeople"><u>{author}</u>, {others}</div>
<div class="triptitle">{place} - {title}</div>
@ -154,6 +236,65 @@ def logbookedit(request, year=None, slug=None):
<hr />
'''
# Successful POST
# So save to database and then write out whole new logbook.html file
#TO DO author and team validation, and check that 'place' is not deleted and that *bloke not forgotten
git = settings.GIT
dirpath = Path(settings.EXPOWEB) / "years" / year
lbe_add = subprocess.run(
[git, "add", filename], cwd=dirpath, capture_output=True, text=True
)
msgdata = (
lbe_add.stderr
+ "\n"
+ lbe_add.stdout
+ "\nreturn code: "
+ str(lbe_add.returncode)
)
message = f'! - FORM Logbook Edit {slug} - Success: git ADD on server for this file {filename}.' + msgdata
print(message)
if lbe_add.returncode != 0:
msgdata = (
"Ask a nerd to fix this.\n\n"
+ lbe_add.stderr
+ "\n\n"
+ lbe_add.stdout
+ "\n\nreturn code: "
+ str(lbe_add.returncode)
)
message = (
f"! - FORM Logbook Edit - CANNOT git ADD on server for this file {filename}. {slug} edits saved but not added to git.\n"
+ msgdata
)
print(message)
return render(request, "errors/generic.html", {"message": message})
lbe_commit = subprocess.run(
[git, "commit", "-m", f"Logbook edited {slug}"],
cwd=dirpath,
capture_output=True,
text=True,
)
message = f'! - FORM Logbook Edit - {filename}. {slug} edits saved, added to git, and COMMITTED.\n' + msgdata
print(message)
#This produces return code = 1 if it commits OK
if lbe_commit.returncode != 0:
msgdata = (
"Ask a nerd to fix this.\n\n"
+ lbe_commit.stderr
+ "\n"
+ lbe_commit.stdout
+ "\nreturn code: "
+ str(lbe_commit.returncode)
)
message = (
f"! - FORM Logbook Edit -Error code with git on server for {filename}. {slug} edits saved, added to git, but NOT committed.\n"
+ msgdata
)
print(message)
return render(request, "errors/generic.html", {"message": message})
return render(
request,
"logbookform.html",
@ -212,10 +353,10 @@ def logbookedit(request, year=None, slug=None):
"tu": tu,
"entry": text,
"textrows": rows,
#"output": output,
},
},
)
else:
else: # no slug
# NEW logbook entry
return render(
request,
"logbookform.html",

46
parsers/logbooks.py
View File

@ -1,6 +1,7 @@
import os
import re
import sys
import string
import time
from datetime import date, datetime
@ -15,6 +16,7 @@ from troggle.core.models.caves import GetCaveLookup
from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry
from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.utils import get_process_memory
from troggle.core.views.uploads import unique_slug
"""
Parses and imports logbooks in all their wonderful confusion
@ -106,11 +108,31 @@ ENTRIES = {
logentries = [] # the entire logbook for one year is a single object: a list of entries
noncaveplaces = ["travel", "Journey", "Loser Plateau", "UNKNOWN", "plateau", "base camp", "basecamp", "top camp", "topcamp"]
tripsdate = {}
alphabet = []
def set_trip_id(year, seq):
def set_trip_seq_id(year, seq):
'''We have not parsed the trip date yet, so this is a sequence numer
'''
tid = f"{year}_s{seq:02d}"
return tid
def reset_trip_id(date):
'''Now we have the date, we can set the tripid (the lbe slug) to be in our standard form
of <date><letter>, i.e. '2003-07-30b'
BUT this gets re-set every time the logbook is imported,
so they are not persistent as we would much prefer.
'''
global alphabet
already =tripsdate.get(date, 0) # returns zero if none found
tripsdate[date] = already +1
if not alphabet:
alphabet = list(string.ascii_lowercase)
tid = f"{date}{alphabet[already]}"
# print(tid)
return tid
rx_tripperson = re.compile(r"(?i)<u>(.*?)</u>$")
rx_round_bracket = re.compile(r"[\(\[].*?[\)\]]")
@ -246,9 +268,14 @@ def store_entry_into_database(date, place, tripcave, title, text, trippersons, a
"expedition": expedition,
"time_underground": logtime_underground,
"cave_slug": str(tripcave),
"slug": tid,
}
lookupAttribs = {"date": date, "title": title}
lookupAttribs = {"slug": tid, "date": date, "title": title}
if LogbookEntry.objects.filter(slug=tid).exists():
# oops.
message = " ! - DUPLICATE SLUG for logbook entry " + tripdate + " - " + slug
DataIssue.objects.create(parser="logbooks", message=message)
slug = slug + "_" + unique_slug(text,2)
lbo = LogbookEntry.objects.create(**nonLookupAttribs, **lookupAttribs)
pt_list = []
@ -332,7 +359,7 @@ def parser_html(year, expedition, txt, seq=""):
logbook_entry_count = 0
for trippara in tripparas:
logbook_entry_count += 1
tid = set_trip_id(year, logbook_entry_count)
tid = set_trip_seq_id(year, logbook_entry_count)
# print(f' - new tid:{tid} lbe count: {logbook_entry_count}')
s = re.match(
@ -376,6 +403,9 @@ def parser_html(year, expedition, txt, seq=""):
continue
ldate = parser_date(tripdate.strip(), year)
# Now we have a date, we can reset tripid
tid = reset_trip_id(ldate)
triptitles = triptitle.split(" - ")
if len(triptitles) >= 2:
place = triptitles[0]
@ -385,7 +415,7 @@ def parser_html(year, expedition, txt, seq=""):
tripcontent = re.sub(r"<p>", "<br /><br />", tripcontent).strip()
triptitle = triptitle.strip()
# triptitle must be unique for a given date. We fix this here.
# triptitle must be unique for a given date. We fix this here. [Why?!]
check = (ldate, triptitle)
if check in dupl:
dupl[check] += 1
@ -458,7 +488,7 @@ def parser_blog(year, expedition, txt, sq=""):
# print(f"{i} - {len(tripstuff)} - {tripstuff[1]}")
triphead = tripheads[i]
logbook_entry_count += 1
tid = set_trip_id(year, logbook_entry_count) + "_blog" + sq
tid = set_trip_seq_id(year, logbook_entry_count) + "_blog" + sq
# print(f" - tid: {tid}")
# data-author="tcacrossley"
@ -580,7 +610,7 @@ def parse_logbook_for_expedition(expedition, blog=False):
if logbook_parseable:
# --------------------
parser = globals()[parsefunc]
print(f" - {year} parsing with {parsefunc} - {lb}")
# print(f" - {year} parsing with {parsefunc} - {lb}")
print(" .", end="")
logentries = parser(year, expedition, txt, sq) # this launches the right parser
# --------------------
@ -595,7 +625,7 @@ def parse_logbook_for_expedition(expedition, blog=False):
def LoadLogbook(year):
"""One off logbook for testing purposes, and also reloadable on '/expedition/2022?reload'
"""One off logbook for testing purposes, and also reloadable on '/expedition/2023?reload'
This is inside an atomic transaction"""
expo = Expedition.objects.get(year=year)

3
templates/logbook2005style.html
View File

@ -14,7 +14,8 @@ maintain half a dozen parser functions.
Sorry about all the crap that surrounds the image tags which has been imported along with the content
when UK Caving blogs have been parsed.
Exported on {% now 'Y-m-d H:m' %} using control panel webpage and exportlogbook() in troggle/code/views/other.py
Exported on {% now 'Y-m-d H:m' %} using either the control panel webpage or when editing a logbook entry online
See troggle/code/views/other.py and core.models/logbooks.py writelogbook(year, filename)
-->
<body>
{%for logbook_entry in logbook_entries%}

4
templates/logbookform.html
View File

@ -79,9 +79,9 @@
title="Time underground (hours)"
{% if tu %}value="{{tu}}"{% else %}placeholder="0.1" {% endif %}
/>
<br /><br /> <span style="color: red">This DOES NOT SAVE ANYTHING yet</span>
<br /><br />
<button class="fancybutton2" style="padding: 0.5em 25px; margin-left: 110px" type = "submit" value = "save" >
Do logbook entry
Update logbook entry
</button>
</form>

3
urls.py
View File

@ -132,8 +132,7 @@ trogglepatterns = [
re_path(r'^api/QMs_json$', QMs_jsonListView.as_view()),
# Logbook entries
re_path(r'^logbookentry/(?P<date>.*)/(?P<slug>.*)/?$', logbookentry,name="logbookentry"),
re_path(r'^logbook(?P<year>\d\d\d\d)\.(?P<extension>.*)/?$', exportlogbook, name='exportlogbook'), # e.g. /logbook2019.html # working but old CSS in
re_path(r'^logbookentry/(?P<date>.*)/(?P<slug>.*)/?$', logbookentry,name="logbookentry"),
re_path(r'^logbook$', exportlogbook, name='exportlogbook'),
# Internal. editfile.html template uses these internally