diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index a65774d..fd1e7eb 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -21,13 +21,20 @@ Parses and imports logbooks in all their wonderful confusion
https://expo.survex.com/handbook/computing/logbooks-parsing.html
"""
todo = """
-- refactor everything with some urgency, esp. parse_logbook_for_expedition()
+- Most of the time is during the database writing (13s out of 14s).
+
+- Move a lot of non-db code from store_entry_into_database()
+into parse_logbook_for_expedition()
+
+- call GetTripPersons at parsing time, not db writing time
+- this is a slow and uncertain function too: cave = getCaveByReference(caveRef)
+
+- if I am certain that we are creating from scratch, don't use save_carefully() to
+create the Django objects. And I am, because I delete the outdated stuff.
- pre-compile all the heavily used regular expressions !
-- break out the code that hits the database from that which parses the logbook
-so that the file-reading and parsing can be parallelized, while writing to the
-database remains serialized (sqlite is single-user).
+- refactor to get rid of the global 'logentries', very ugly indeed.
- profile the code to find bad repetitive things, of which there are many.
@@ -35,13 +42,14 @@ database remains serialized (sqlite is single-user).
- far too many uses of Django field dereferencing to get values, which is SLOW
+- replace explicit 1970 date with a constant EPOCH
+
- rewrite to use generators rather than storing everything intermediate in lists - to reduce memory impact.
- We should ensure logbook.html is utf-8 and stop this crap:
file_in = open(logbookfile,'rb')
txt = file_in.read().decode("latin1")
-- this is a slow and uncertain function: cave = getCaveByReference(caveRef)
- use Fixtures https://docs.djangoproject.com/en/4.1/ref/django-admin/#django-admin-loaddata to cache
data for old logbooks? Not worth it..
@@ -162,17 +170,7 @@ def GetTripPersons(trippeople, expedition, logtime_underground, tid=None):
# print(f" - {tid} [{author.person}] '{res[0][0].person}'...")
return res, author
-
-def store_entry_into_database(date, place, title, text, trippeople, expedition, logtime_underground, tid=None):
- """saves a single logbook entry and related persontrips
- Does NOT save the expeditionday_id - all NULLs. why? Because we are deprecating expeditionday !
-
- troggle.log shows that we are creating lots of duplicates, which is no no problem with SQL as they just overwrite but we are saving the same thing too many times..
-
- Until 18 Dec.2022, this was overwriting logbook entries for the same date with the same title, because
- lookupAttribs={'date':date, 'title':title}
- """
-
+def tidy_time_underground(logtime_underground):
# Nasty hack, must tidy this up..
if logtime_underground:
try:
@@ -187,7 +185,9 @@ def store_entry_into_database(date, place, title, text, trippeople, expedition,
logtime_underground = 0
else:
logtime_underground = 0
+ return logtime_underground
+def tidy_trip_persons(trippeople, expedition, logtime_underground, tid):
try:
trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground, tid=tid)
# print(f" - {author} - {logtime_underground}")
@@ -195,14 +195,25 @@ def store_entry_into_database(date, place, title, text, trippeople, expedition,
message = f" ! - {expedition.year} Skipping logentry: {title} - GetTripPersons FAIL"
DataIssue.objects.create(parser="logbooks", message=message)
print(message)
- raise
+ # raise
return
if not author:
message = f" ! - {expedition.year} Warning: logentry: {title} - no expo member author for entry '{tid}'"
DataIssue.objects.create(parser="logbooks", message=message)
print(message)
- # return
+
+ return trippersons, author
+
+def store_entry_into_database(date, place, title, text, trippersons, author, expedition, logtime_underground, tid=None):
+ """saves a single logbook entry and related persontrips
+ Does NOT save the expeditionday_id - all NULLs. why? Because we are deprecating expeditionday !
+
+ troggle.log shows that we are creating lots of duplicates, which is no no problem with SQL as they just overwrite but we are saving the same thing too many times..
+
+ Until 18 Dec.2022, this was overwriting logbook entries for the same date with the same title, because
+ lookupAttribs={'date':date, 'title':title}
+ """
# This needs attention. The slug field is derived from 'title'
# both GetCaveLookup() and GetTripCave() need to work together better. None of this data is *used* though?
@@ -257,9 +268,6 @@ def store_entry_into_database(date, place, title, text, trippeople, expedition,
# this creates the PersonTrip instance.
save_carefully(PersonTrip, lookupAttribs, nonLookupAttribs)
-
-
-
def parser_date(tripdate, year):
"""Interprets dates in the expo logbooks and returns a correct datetime.date object"""
dummydate = date(1970, 1, 1) # replace with _EPOCH
@@ -395,7 +403,10 @@ def parser_html(year, expedition, txt, seq=""):
else:
dupl[check] = 1
- entrytuple = (ldate, tripcave, triptitle, ltriptext, trippeople, expedition, tu, tripid1)
+ tu = tidy_time_underground(tu)
+
+ trippersons, author = tidy_trip_persons(trippeople, expedition, tu, tid)
+ entrytuple = (ldate, tripcave, triptitle, ltriptext, trippersons, author, expedition, tu, tripid1)
logentries.append(entrytuple)
@@ -439,7 +450,7 @@ def parser_blog(year, expedition, txt, sq=""):
print(f"{len(tripheads)} - {len(tripparas)}")
location = "Plateau" # best guess, fix manually later
- tu = 0
+ tu = 0 # no logged time underground in a blog entry
logbook_entry_count = 0
for i in range(0, len(tripparas)):
tripstuff = tripparas[i]
@@ -493,7 +504,8 @@ def parser_blog(year, expedition, txt, sq=""):
tripcontent = re.sub(r"<hr\s*>", "", tripcontent)
tripcontent = f"\n\n<!-- Content parsed from UK Caving Blog -->\nBlog Author: {trippeople}" + tripcontent
- entrytuple = (tripdate, location, triptitle, tripcontent, trippeople, expedition, tu, tid)
+ trippersons, author = tidy_trip_persons(trippeople, expedition, logtime_underground, tid)
+ entrytuple = (tripdate, location, triptitle, tripcontent, trippersons, author, expedition, tu, tid)
logentries.append(entrytuple)
def clean_all_logbooks():
@@ -564,7 +576,6 @@ def parse_logbook_for_expedition(expedition, blog=False):
print(f" ! Very Bad Error opening {lb}")
if logbook_parseable:
-
# --------------------
parser = globals()[parsefunc]
print(f" - {year} parsing with {parsefunc} - {lb}")
@@ -572,8 +583,8 @@ def parse_logbook_for_expedition(expedition, blog=False):
# --------------------
# move database storage into separate step
# for entrytuple in logentries:
- # date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1 = entrytuple
- # store_entry_into_database(date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1)
+ # date, tripcave, triptitle, text, trippersons, author, expedition, logtime_underground, tripid1 = entrytuple
+ # store_entry_into_database(date, tripcave, triptitle, text, trippersons, author, expedition, logtime_underground, tripid1)
if len(logentries) == expect:
# print(f"OK {year} {len(logentries):5d} is {expect}\n")
@@ -603,10 +614,10 @@ def LoadLogbook(year):
f" - Not a year with extant blog entries to import: '{year}' not in BLOG_PARSER_SETTINGS {BLOG_PARSER_SETTINGS}"
)
for entrytuple in logentries:
- date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1 = entrytuple
+ date, tripcave, triptitle, text, trippersons, author, expedition, logtime_underground, tripid1 = entrytuple
if expo == expedition:
#print(f" - {triptitle}")
- store_entry_into_database(date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1)
+ store_entry_into_database(date, tripcave, triptitle, text, trippersons, author, expedition, logtime_underground, tripid1)
expedition.save() # to save logbook name property
def LoadLogbooks():
@@ -692,8 +703,8 @@ def LoadLogbooks():
# - LogBookEntry (text, who when etc.)
# - PersonTrip (who was on that specific trip mentione din the logbook entry)
for entrytuple in allentries:
- date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1 = entrytuple
- store_entry_into_database(date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1)
+ date, tripcave, triptitle, text, trippersons, author, expedition, logtime_underground, tripid1 = entrytuple
+ store_entry_into_database(date, tripcave, triptitle, text, trippersons, author, expedition, logtime_underground, tripid1)
for expo in expos:
expedition.save() # to save logbook name property