expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-12-14 21:57:09 +00:00
Code Issues 18 Projects 3 Activity

Nicknames preserved, date checked

Browse Source
This commit is contained in:
Philip Sargent
2023-08-26 18:39:29 +03:00
parent 93397a774f
commit d08a6aa204
6 changed files with 77 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
parsers/logbooks.py
View File

@@ -22,6 +22,9 @@ Parses and imports logbooks in all their wonderful confusion
https://expo.survex.com/handbook/computing/logbooks-parsing.html
"""
todo = """
- make id= for each entry persistent and unchanging, and check cross-references in other logbooks and other HTML frahments
e.g. cave descriptions
- Most of the time is during the database writing (6s out of 8s).
- profile the code to find bad repetitive things, of which there are many.
@@ -37,8 +40,7 @@ todo = """
file_in = open(logbookfile,'rb')
txt = file_in.read().decode("latin1")
- use Fixtures https://docs.djangoproject.com/en/dev/ref/django-admin/#django-admin-loaddata to cache
data for old logbooks? Not worth it..
"""
MAX_LOGBOOK_ENTRY_TITLE_LENGTH = 200
BLOG_PARSER_SETTINGS = { # no default, must be explicit
@@ -127,7 +129,8 @@ def GetTripPersons(trippeople, expedition, logtime_underground, tid=None):
if tripperson[0] != "*": # a name prefix of "*" is special
tripperson = re.sub(rx_round_bracket, "", tripperson).strip()
# Whacky aliases all handled in GetPersonExpeditionNameLookup()
# Whacky aliases all resolved in GetPersonExpeditionNameLookup()
nickname_used = tripperson
try:
personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower())
if not personyear:
@@ -138,9 +141,9 @@ def GetTripPersons(trippeople, expedition, logtime_underground, tid=None):
message = f" ! - {expedition.year} No name match for: '{tripperson}' in entry {tid=} for this year."
print(message)
DataIssue.objects.create(parser="logbooks", message=message)
res.append((personyear, logtime_underground))
res.append((personyear, nickname_used, logtime_underground))
except:
message = f" ! - {expedition.year} EXCEPTION: '{tripperson}' in entry {tid=} for this year."
message = f" ! - {expedition.year} EXCEPTION: '{tripperson}' ({nickname_used}) in entry {tid=} for this year."
print(message)
DataIssue.objects.create(parser="logbooks", message=message)
raise
@@ -179,7 +182,7 @@ def tidy_time_underground(logtime_underground):
def tidy_trip_persons(trippeople, title, expedition, logtime_underground, tid):
try:
trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground, tid=tid)
# print(f" - {author} - {logtime_underground}")
# trippersons is a list of tuples (personyear, nickname_used, logtime_underground)
except:
message = f" ! - {expedition.year} Skipping logentry: {title} - GetTripPersons FAIL"
DataIssue.objects.create(parser="logbooks", message=message)
@@ -247,14 +250,16 @@ def store_entry_into_database(date, place, tripcave, title, text, trippersons, a
lbo = LogbookEntry.objects.create(**nonLookupAttribs, **lookupAttribs)
pt_list = []
for tripperson, time_underground in trippersons:
lookupAttribs = {"personexpedition": tripperson, "logbook_entry": lbo} # lbo is primary key
for tripperson, nickname_used, time_underground in trippersons:
lookupAttribs = {"personexpedition": tripperson, "nickname_used": nickname_used, "logbook_entry": lbo} # lbo is primary key
nonLookupAttribs = {"time_underground": time_underground, "is_logbook_entry_author": (tripperson == author)}
pt_list.append(PersonLogEntry(**nonLookupAttribs, **lookupAttribs))
PersonLogEntry.objects.bulk_create(pt_list)
def parser_date(tripdate, year):
"""Interprets dates in the expo logbooks and returns a correct datetime.date object"""
"""Interprets dates in the expo logbooks and returns a correct datetime.date object
Does NOT actually check that it is a truly valid date..
"""
dummydate = date(1970, 1, 1) # replace with _EPOCH
month = 1
day = 1