)? # second date
\s*(?: )?
\s* ", "", ltriptext).strip()
entrytuple = (ldate, tripcave, triptitle, ltriptext,
trippeople, expedition, tu, "html", tripid1)
logentries.append(entrytuple)
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext,
trippeople=trippeople, expedition=expedition, logtime_underground=0,
entry_type="html", tid=tid)
EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
"html", tripid1, logbook_entry_count, tid=tid)
# main parser for 1991 - 2001. simpler because the data has been hacked so much to fit it
# trying it out for years 1982 - 1990 too. Some logbook editing required by hand.. place
def Parseloghtml01(year, expedition, txt):
global logentries
global logdataissues
errorcount = 0
tripparas = re.findall(r" |
", "\n\n", ltriptext).strip()
ltriptext = re.sub(r"", "_", ltriptext)
ltriptext = re.sub(r"", "''", ltriptext)
ltriptext = re.sub(r"", "'''", ltriptext)
if ltriptext == "":
message = " ! - Zero content for logbook entry!: " + tid
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
entrytuple = (ldate, tripcave, triptitle, ltriptext,
trippeople, expedition, tu, "html01", tid)
logentries.append(entrytuple)
try:
EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext,
trippeople=trippeople, expedition=expedition, logtime_underground=0,
entry_type="html", tid=tid)
except:
message = " ! - Enter log entry into database FAIL exception in: " + tid
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
try:
EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
"html01", tid, logbook_entry_count, tid=tid)
except:
message = " ! - Enter log entry into ObjectStore FAIL exception in: " + tid
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
except:
message = f" ! - Skipping logentry {year} due to exception in: {tid}"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
errorcount += 1
if errorcount >5 :
message = f" !!- TOO MANY ERRORS - aborting at '{tid}' logbook: {year}"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
return
# parser for 2003
def Parseloghtml03(year, expedition, txt):
global logentries
global logdataissues
tripparas = re.findall(r"
", "\n\n", ltriptext).strip()
ltriptext = re.sub(r"[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext)
entrytuple = (ldate, tripcave, triptitle, ltriptext,
trippeople, expedition, tu, "html03", tid)
logentries.append(entrytuple)
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle,
text = ltriptext, trippeople=trippeople, expedition=expedition,
logtime_underground=0, entry_type="html", tid=tid)
EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
"html03", tid, logbook_entry_count, tid=tid)
def SetDatesFromLogbookEntries(expedition):
"""
Sets the date_from and date_to field for an expedition based on persontrips.
Then sets the expedition date_from and date_to based on the personexpeditions.
"""
# Probably a faster way to do this. This uses a lot of db queries, but we have all this
# in memory..
for personexpedition in expedition.personexpedition_set.all():
persontrips = personexpedition.persontrip_set.order_by('logbook_entry__date')
# sequencing is difficult to do
lprevpersontrip = None
for persontrip in persontrips:
persontrip.persontrip_prev = lprevpersontrip
if lprevpersontrip:
lprevpersontrip.persontrip_next = persontrip
lprevpersontrip.save()
persontrip.persontrip_next = None
lprevpersontrip = persontrip
persontrip.save() # also saved in EnterLogIntoDbase. MERGE these to speed up import.
def LoadLogbookForExpedition(expedition, expect):
""" Parses all logbook entries for one expedition
If a cache is found it uses it. If not found, or fails sanity checks, parses source file.
"""
# absolutely horrid. REFACTOR THIS (all my fault..)
global logentries
global logdataissues
logbook_parseable = False
logbook_cached = False
yearlinks = settings.LOGBOOK_PARSER_SETTINGS
expologbase = os.path.join(settings.EXPOWEB, "years")
logentries=[]
def validcache(year,n):
if year != expedition:
print(" ! year != expedition ",year, expedition )
return False
if len(logentries) != n:
print(" ! len(logentries) != n ",len(logentries), n )
return False
if n != expect:
print(" ! n != expect ",n, expect )
return False
return True
def cleanerrors(year):
global logdataissues
dataissues = DataIssue.objects.filter(parser='logbooks')
for di in dataissues:
ph = year
if re.search(ph, di.message) is not None:
#print(f' - CLEANING dataissue {di.message}')
di.delete()
#print(f' - CLEAN {year} {len(logdataissues)} {type(logdataissues)} data issues for this year')
dellist = []
for key, value in logdataissues.items():
#print(f' - CLEANING logdataissues [{key}]: {value}')
if key.startswith(year):
#print(f' - CLEANING logdataissues [{key:12}]: {value} ')
dellist.append(key)
for i in dellist:
del logdataissues[i]
cleanerrors(expedition.year)
if expedition.year in yearlinks:
logbookfile = os.path.join(expologbase, yearlinks[expedition.year][0])
expedition.logbookfile = yearlinks[expedition.year][0]
parsefunc = yearlinks[expedition.year][1]
else:
logbookfile = os.path.join(expologbase, expedition.year, settings.DEFAULT_LOGBOOK_FILE)
expedition.logbookfile = settings.DEFAULT_LOGBOOK_FILE
parsefunc = settings.DEFAULT_LOGBOOK_PARSER
cache_filename = Path(logbookfile + ".cache")
if not cache_filename.is_file():
print(" - Cache file does not exist \"" + str(cache_filename) +"\"")
expedition.save()
now = time.time()
bad_cache = True # emporarily disable reading the cache - buggy
try:
cache_t = os.path.getmtime(cache_filename)
if os.path.getmtime(logbookfile) - cache_t > 2: # at least 2 secs later
bad_cache= True
if now - cache_t > 30*24*60*60:
bad_cache= True
if bad_cache:
print(" - ! Cache is either stale or more than 30 days old. Deleting it.")
os.remove(cache_filename)
logentries=[]
print(" ! Removed stale or corrupt cache file")
raise
print(" - Reading cache: " + str(cache_filename), end='')
try:
with open(cache_filename, "rb") as f:
year,n,logentries = pickle.load(f)
if validcache(year,n):
print(" -- Loaded ", len(logentries), " log entries")
logbook_cached = True
else:
print(" !- Told to expect ", expect, " but ", len(logentries), " found in cache")
raise
except:
print(" ! Failed to load corrupt cache. (Or I was told to ignore it). Deleting it.")
os.remove(cache_filename)
logentries=[]
raise
except :
print(" - Cache de-pickle failure \"" + str(cache_filename) +"\"")
try:
file_in = open(logbookfile,'rb')
txt = file_in.read().decode("latin1")
file_in.close()
logbook_parseable = True
print((" - Using: " + parsefunc + " to parse " + logbookfile))
except (IOError):
logbook_parseable = False
print((" ! Couldn't open logbook " + logbookfile))
if logbook_parseable:
parser = globals()[parsefunc]
parser(expedition.year, expedition, txt) # this launches the parser
SetDatesFromLogbookEntries(expedition)
if len(logentries) >0:
print(" - Cacheing " , len(logentries), " log entries")
with open(cache_filename, "wb") as fc: # we much check that permission are g+w ! or expo can't delete the cache
logbk=(expedition,len(logentries),logentries)
pickle.dump(logbk, fc, protocol=4)
else:
print(" ! NO TRIP entries found in logbook, check the syntax.")
if logbook_cached: # working on this bit...
i=0
for entrytuple in range(len(logentries)):
date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, entry_type, tripid1 = logentries[i]
#print(" - entry tuple " , i, " tid", tripid1)
EnterLogIntoDbase(date, tripcave, triptitle, text, trippeople, expedition, 0,
entry_type, tripid1)
EnterLogIntoObjStore(expedition.year, date, tripcave, triptitle, text, trippeople, logtime_underground,
entry_type, tripid1, i)
i +=1
SetDatesFromLogbookEntries(expedition)
return len(logentries)
def LoadLogbooks():
""" This is the master function for parsing all logbooks into the Troggle database.
Parser settings appropriate for each year are set in settings.py LOGBOOK_PARSER_SETTINGS.
This should be rewritten to use coroutines to load all logbooks from disc in parallel.
"""
global logdataissues
logdataissues = {}
DataIssue.objects.filter(parser='logbooks').delete()
expos = Expedition.objects.all()
if len(expos) <= 1:
print(" ! No expeditions found. Load 'people' first.\n")
noexpo = ["1986", "2020", "2021",] #no expo
lostlogbook = ["1976", "1977", "1978", "1979", "1980", "1981"]
sqlfail = ["1987", "1988", "1989"] # breaks mysql with db constraint fail - debug locally first]
nologbook = noexpo + lostlogbook + sqlfail
entries = { "2019": 20, "2018": 74, "2017": 60, "2016": 81, "2015": 79,
"2014": 65, "2013": 51, "2012": 75, "2011": 68, "2010": 22, "2009": 52,
"2008": 49, "2007": 111, "2006": 60, "2005": 55, "2004": 76, "2003": 40, "2002": 31,
"2001": 48, "2000": 54, "1999": 79, "1998": 43, "1997": 53, "1996": 94, "1995": 41,
"1994": 32, "1993": 41, "1992": 61, "1991": 38, "1990": 87, "1989": 1,"1988": 1,"1987": 1,
"1985": 22,"1984": 32,"1983": 52,"1982": 42,}
# Logbooks log.htm exist for 1983, 84, 85, 87, 88, 89 but have no full-working parser, or need hand-editing.
try:
os.remove("loadlogbk.log")
except OSError:
pass
nlbe={}
expd ={}
with open("loadlogbk.log", "a") as log:
for expo in expos:
TROG['pagecache']['expedition'][expo.year] = None # clear cache
if expo.year in sqlfail:
print(" - Logbook for: " + expo.year + " NO parsing attempted - known sql failures")
message = f" ! - Not even attempting to parse logbook for {expo.year} until code fixed"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[f"sqlfail {expo.year}"]=message
if expo.year not in nologbook:
print((" - Logbook for: " + expo.year))
if expo.year in entries:
numentries = LoadLogbookForExpedition(expo, entries[expo.year]) # this actually loads the logbook for one year
log.write("{} {:5d} should be {}\n".format(expo.year, numentries, entries[expo.year]))
nlbe[expo.year]=numentries
expd[expo.year]= 0
else:
print(" - No Logbook yet for: " + expo.year) # catch case when preparing for next expo
print("** total trips in ObjStore:", len(trips))
#for i in logdataissues:
# print("{:15s}: {}".format(i, logdataissues[i]))
for lbe in trips:
year, date, tripcave, triptitle, text, trippeople, tu, formattype = trips[lbe]
expd[year] += 1
yt=0
for y in expd:
print("{} {}".format(y, expd[y]), nlbe[y])
yt += expd[y]
print("total {} log entries in all expeditions".format(yt))
try:
shelvfilenm = 'logbktrips.shelve' # ".db" automatically apended after python 3.8
with shelve.open(shelvfilenm, writeback=True) as odb:
for lbe in trips:
odb[lbe]=trips[lbe]
odb.sync()
odb.close()
except:
message = f" ! - Failed store cached logbooks in '{shelvfilenm}.db' - Delete old file and try again"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues["Shelve Fail"]=message
print(message)
# dateRegex = re.compile(r'(.*?)
', re.S)
# reportRegex = re.compile(r'