diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 864f1b2..300adf3 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -20,9 +20,6 @@ from parsers.people import GetPersonExpeditionNameLookup ''' Parses and imports logbooks in all their wonderful confusion -The Objectstore stuff is an initial attempt to see how we can migrate away from the Django database. -An idea which no longer seems sensible given that we rely on the database to do the multiuser bit. - # When we edit logbook entries, allow a "?" after any piece of data to say we've frigged it and # it can be checked up later from the hard-copy if necessary; or it's not possible to determin (name, trip place, etc) ''' @@ -49,14 +46,9 @@ todo=''' - this is a slow and uncertain function: cave = getCaveByReference(caveRef) -- the object store will need additional functions to replicate the persontrip calculation - and storage. For the moment we leave all that to be done in the django db - Concurrent synchronisation would be nice.. - DB lock currently prevents multiple threads for loading logbooks. But asyncio might work..? -- Put the object store 'trips' and the 'logdataissues' into TROG global object - ''' logentries = [] # the entire logbook for one year is a single object: a list of entries @@ -263,49 +255,8 @@ def Parselogwikitxt(year, expedition, txt): entrytuple = (ldate, tripcave, tripsplace, triptext, trippeople, expedition, tu, "wiki", tripid) logentries.append(entrytuple) - - # EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, - # expedition=expedition, logtime_underground=0, tid=tid) - # EnterLogIntoObjStore(year, ldate, tripcave, tripplace, triptext, trippeople, - # tu, "wiki", tripid, logbook_entry_count, tid=tid) - - -def EnterLogIntoObjStore(year, date, tripcave, triptitle, text, trippeople, tu, tripid1, seq, tid=None): - '''Called once for each logbook entry as the logbook is parsed - ''' - # This will need additional functions to replicate the persontrip calculation and storage. For the - # moment we leave all that to be done in the django db - global trips # should be a singleton TROG eventually - global logdataissues - - - if tid in trips: - tyear, tdate, *trest = trips[tid] - msg = f" ! DUPLICATE tid: '{tid}' on date:{tdate} " - print(msg) - DataIssue.objects.create(parser='logbooks', message=msg) - tid = set_trip_id(str(date),seq) - #print(" - De-dup ",seq, tid) - logdataissues[tid]=msg - - if not tid: - tid = set_trip_id(str(date),seq) - trips[tid] = (year, date, tripcave, triptitle, text, trippeople, tu) - - ## copy a lot of checking functionality here from EnterLogIntoDbase() - # GetTripPersons is a db query, so this will need to be put in ObjStore before this will work.. - # or design a different way to do it. - #trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground) - # if not author: - # print(" ! - Skipping logentry: " + title + " - no RECOGNISED author for entry") - # message = " ! - Skipping logentry: %s - no author for entry in year '%s'" % (title, expedition.year) - # DataIssue.objects.create(parser='logbooks', message=message) - # logdataissues[tid+"author"]=message - return - - - + # 2002, 2004, 2005, 2007, 2010 - now # 2006 wiki text is incomplete, but the html all there. So using this parser now. def Parseloghtmltxt(year, expedition, txt): @@ -364,12 +315,6 @@ def Parseloghtmltxt(year, expedition, txt): trippeople, expedition, tu, "html", tripid1) logentries.append(entrytuple) - # EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, - # trippeople=trippeople, expedition=expedition, logtime_underground=0, - # entry_type="html", tid=tid) - - # EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, - # "html", tripid1, logbook_entry_count, tid=tid) # main parser for 1991 - 2001. simpler because the data has been hacked so much to fit it # trying it out for years 1982 - 1990 too. Some logbook editing required by hand.. place @@ -466,24 +411,6 @@ def Parseloghtml01(year, expedition, txt): entrytuple = (ldate, tripcave, triptitle, ltriptext, trippeople, expedition, tu, "html01", tid) logentries.append(entrytuple) - # try: - # EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, - # trippeople=trippeople, expedition=expedition, logtime_underground=0, - # entry_type="html", tid=tid) - # except: - # message = " ! - Enter log entry into database FAIL exception in: " + tid - # DataIssue.objects.create(parser='logbooks', message=message) - # logdataissues[tid]=message - # print(message) - - # try: - # EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, - # "html01", tid, logbook_entry_count, tid=tid) - # except: - # message = " ! - Enter log entry into ObjectStore FAIL exception in: " + tid - # DataIssue.objects.create(parser='logbooks', message=message) - # logdataissues[tid]=message - # print(message) except: message = f" ! - Skipping logentry {year} due to exception in: {tid}" @@ -543,14 +470,6 @@ def Parseloghtml03(year, expedition, txt): trippeople, expedition, tu, "html03", tid) logentries.append(entrytuple) - # EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, - # text = ltriptext, trippeople=trippeople, expedition=expedition, - # logtime_underground=0, entry_type="html", tid=tid) - - # EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, - # "html03", tid, logbook_entry_count, tid=tid) - - def SetDatesFromLogbookEntries(expedition): """ Sets the date_from and date_to field for an expedition based on persontrips. @@ -709,8 +628,6 @@ def LoadLogbookForExpedition(expedition): date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, entry_type, tripid1 = entrytuple EnterLogIntoDbase(date, tripcave, triptitle, text, trippeople, expedition, 0, tripid1) - EnterLogIntoObjStore(year, date, tripcave, triptitle, text, trippeople, logtime_underground, - tripid1, i) i +=1 SetDatesFromLogbookEntries(expedition)