forked from expo/troggle
Removed misguided ObjStore as we do need multiuser db
This commit is contained in:
Philip Sargent
parent
4514eda311
commit
4260b0f092
@ -20,9 +20,6 @@ from parsers.people import GetPersonExpeditionNameLookup
|
||||
'''
|
||||
Parses and imports logbooks in all their wonderful confusion
|
||||
|
||||
The Objectstore stuff is an initial attempt to see how we can migrate away from the Django database.
|
||||
An idea which no longer seems sensible given that we rely on the database to do the multiuser bit.
|
||||
|
||||
# When we edit logbook entries, allow a "?" after any piece of data to say we've frigged it and
|
||||
# it can be checked up later from the hard-copy if necessary; or it's not possible to determin (name, trip place, etc)
|
||||
'''
|
||||
@ -49,14 +46,9 @@ todo='''
|
||||
|
||||
- this is a slow and uncertain function: cave = getCaveByReference(caveRef)
|
||||
|
||||
- the object store will need additional functions to replicate the persontrip calculation
|
||||
and storage. For the moment we leave all that to be done in the django db
|
||||
Concurrent synchronisation would be nice..
|
||||
|
||||
- DB lock currently prevents multiple threads for loading logbooks. But asyncio might work..?
|
||||
|
||||
- Put the object store 'trips' and the 'logdataissues' into TROG global object
|
||||
|
||||
'''
|
||||
|
||||
logentries = [] # the entire logbook for one year is a single object: a list of entries
|
||||
@ -263,49 +255,8 @@ def Parselogwikitxt(year, expedition, txt):
|
||||
entrytuple = (ldate, tripcave, tripsplace, triptext,
|
||||
trippeople, expedition, tu, "wiki", tripid)
|
||||
logentries.append(entrytuple)
|
||||
|
||||
# EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople,
|
||||
# expedition=expedition, logtime_underground=0, tid=tid)
|
||||
|
||||
# EnterLogIntoObjStore(year, ldate, tripcave, tripplace, triptext, trippeople,
|
||||
# tu, "wiki", tripid, logbook_entry_count, tid=tid)
|
||||
|
||||
|
||||
def EnterLogIntoObjStore(year, date, tripcave, triptitle, text, trippeople, tu, tripid1, seq, tid=None):
|
||||
'''Called once for each logbook entry as the logbook is parsed
|
||||
'''
|
||||
# This will need additional functions to replicate the persontrip calculation and storage. For the
|
||||
# moment we leave all that to be done in the django db
|
||||
global trips # should be a singleton TROG eventually
|
||||
global logdataissues
|
||||
|
||||
|
||||
if tid in trips:
|
||||
tyear, tdate, *trest = trips[tid]
|
||||
msg = f" ! DUPLICATE tid: '{tid}' on date:{tdate} "
|
||||
print(msg)
|
||||
DataIssue.objects.create(parser='logbooks', message=msg)
|
||||
tid = set_trip_id(str(date),seq)
|
||||
#print(" - De-dup ",seq, tid)
|
||||
logdataissues[tid]=msg
|
||||
|
||||
if not tid:
|
||||
tid = set_trip_id(str(date),seq)
|
||||
trips[tid] = (year, date, tripcave, triptitle, text, trippeople, tu)
|
||||
|
||||
## copy a lot of checking functionality here from EnterLogIntoDbase()
|
||||
# GetTripPersons is a db query, so this will need to be put in ObjStore before this will work..
|
||||
# or design a different way to do it.
|
||||
#trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground)
|
||||
# if not author:
|
||||
# print(" ! - Skipping logentry: " + title + " - no RECOGNISED author for entry")
|
||||
# message = " ! - Skipping logentry: %s - no author for entry in year '%s'" % (title, expedition.year)
|
||||
# DataIssue.objects.create(parser='logbooks', message=message)
|
||||
# logdataissues[tid+"author"]=message
|
||||
return
|
||||
|
||||
|
||||
|
||||
|
||||
# 2002, 2004, 2005, 2007, 2010 - now
|
||||
# 2006 wiki text is incomplete, but the html all there. So using this parser now.
|
||||
def Parseloghtmltxt(year, expedition, txt):
|
||||
@ -364,12 +315,6 @@ def Parseloghtmltxt(year, expedition, txt):
|
||||
trippeople, expedition, tu, "html", tripid1)
|
||||
logentries.append(entrytuple)
|
||||
|
||||
# EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext,
|
||||
# trippeople=trippeople, expedition=expedition, logtime_underground=0,
|
||||
# entry_type="html", tid=tid)
|
||||
|
||||
# EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
|
||||
# "html", tripid1, logbook_entry_count, tid=tid)
|
||||
|
||||
# main parser for 1991 - 2001. simpler because the data has been hacked so much to fit it
|
||||
# trying it out for years 1982 - 1990 too. Some logbook editing required by hand.. place
|
||||
@ -466,24 +411,6 @@ def Parseloghtml01(year, expedition, txt):
|
||||
entrytuple = (ldate, tripcave, triptitle, ltriptext,
|
||||
trippeople, expedition, tu, "html01", tid)
|
||||
logentries.append(entrytuple)
|
||||
# try:
|
||||
# EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext,
|
||||
# trippeople=trippeople, expedition=expedition, logtime_underground=0,
|
||||
# entry_type="html", tid=tid)
|
||||
# except:
|
||||
# message = " ! - Enter log entry into database FAIL exception in: " + tid
|
||||
# DataIssue.objects.create(parser='logbooks', message=message)
|
||||
# logdataissues[tid]=message
|
||||
# print(message)
|
||||
|
||||
# try:
|
||||
# EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
|
||||
# "html01", tid, logbook_entry_count, tid=tid)
|
||||
# except:
|
||||
# message = " ! - Enter log entry into ObjectStore FAIL exception in: " + tid
|
||||
# DataIssue.objects.create(parser='logbooks', message=message)
|
||||
# logdataissues[tid]=message
|
||||
# print(message)
|
||||
|
||||
except:
|
||||
message = f" ! - Skipping logentry {year} due to exception in: {tid}"
|
||||
@ -543,14 +470,6 @@ def Parseloghtml03(year, expedition, txt):
|
||||
trippeople, expedition, tu, "html03", tid)
|
||||
logentries.append(entrytuple)
|
||||
|
||||
# EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle,
|
||||
# text = ltriptext, trippeople=trippeople, expedition=expedition,
|
||||
# logtime_underground=0, entry_type="html", tid=tid)
|
||||
|
||||
# EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
|
||||
# "html03", tid, logbook_entry_count, tid=tid)
|
||||
|
||||
|
||||
def SetDatesFromLogbookEntries(expedition):
|
||||
"""
|
||||
Sets the date_from and date_to field for an expedition based on persontrips.
|
||||
@ -709,8 +628,6 @@ def LoadLogbookForExpedition(expedition):
|
||||
date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, entry_type, tripid1 = entrytuple
|
||||
EnterLogIntoDbase(date, tripcave, triptitle, text, trippeople, expedition, 0,
|
||||
tripid1)
|
||||
EnterLogIntoObjStore(year, date, tripcave, triptitle, text, trippeople, logtime_underground,
|
||||
tripid1, i)
|
||||
i +=1
|
||||
SetDatesFromLogbookEntries(expedition)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user