)? # second date
\s*(?: )?
\s* )? # second date
\s*(?: )?
\s* ", "", ltriptext).strip()
entrytuple = (ldate, tripcave, triptitle, ltriptext,
trippeople, expedition, tu, tripid1)
logentries.append(entrytuple)
# main parser for 1991 - 2001. simpler because the data has been hacked so much to fit it
# trying it out for years 1982 - 1990 too. Some logbook editing required by hand.. place
def parser_html_01(year, expedition, txt):
global logentries
global logdataissues
errorcount = 0
tripparas = re.findall(r" | ", "\n\n", ltriptext).strip()
ltriptext = re.sub(r"[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext)
entrytuple = (ldate, tripcave, triptitle, ltriptext,
trippeople, expedition, tu, tid)
logentries.append(entrytuple)
def LoadLogbookForExpedition(expedition):
""" Parses all logbook entries for one expedition
"""
# absolutely horrid. REFACTOR THIS (all my fault..)
global logentries
global logdataissues
global entries
logbook_parseable = False
yearlinks = LOGBOOK_PARSER_SETTINGS
expologbase = os.path.join(settings.EXPOWEB, "years")
logentries=[]
year = expedition.year
expect = entries[year]
# print(" - Logbook for: " + year)
def cleanerrors(year):
global logdataissues
dataissues = DataIssue.objects.filter(parser='logbooks')
for di in dataissues:
ph = year
if re.search(ph, di.message) is not None:
#print(f' - CLEANING dataissue {di.message}')
di.delete()
#print(f' - CLEAN {year} {len(logdataissues)} {type(logdataissues)} data issues for this year')
dellist = []
for key, value in logdataissues.items():
#print(f' - CLEANING logdataissues [{key}]: {value}')
if key.startswith(year):
#print(f' - CLEANING logdataissues [{key:12}]: {value} ')
dellist.append(key)
for i in dellist:
del logdataissues[i]
cleanerrors(year)
if year in yearlinks:
logbookpath = Path(expologbase) / year / yearlinks[year][0]
expedition.logbookfile = yearlinks[year][0]
parsefunc = yearlinks[year][1]
else:
logbookpath = Path(expologbase) / year / DEFAULT_LOGBOOK_FILE
expedition.logbookfile = DEFAULT_LOGBOOK_FILE
parsefunc = DEFAULT_LOGBOOK_PARSER
expedition.save()
try:
file_in = open(logbookpath,'rb')
txt = file_in.read().decode("utf-8")
file_in.close()
logbook_parseable = True
except (IOError):
logbook_parseable = False
print(" ! Couldn't open logbook as UTF-8 " + logbookpath)
except:
logbook_parseable = False
print(" ! Very Bad Error opening " + logbookpath)
if logbook_parseable:
parser = globals()[parsefunc]
print(f' - {year} parsing with {parsefunc}')
parser(year, expedition, txt) # this launches the right parser for this year
i=0
for entrytuple in logentries:
# date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1 = entrytuple
try:
date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, tripid1 = entrytuple
except ValueError: # cope with removal of entry_type but still in cache files. Remove in Dec. 2022.
date, tripcave, triptitle, text, trippeople, expedition, logtime_underground, entry_type, tripid1 = entrytuple
print(f' - Exception entry_type "{entry_type}" {tripid1}')
EnterLogIntoDbase(date, tripcave, triptitle, text, trippeople, expedition, 0,
tripid1)
i +=1
if len(logentries) == expect:
# print(f"OK {year} {len(logentries):5d} is {expect}\n")
pass
else:
print(f"Mismatch {year} {len(logentries):5d} is not {expect}\n")
return len(logentries)
def LoadLogbooks():
""" This is the master function for parsing all logbooks into the Troggle database.
This should be rewritten to use coroutines to load all logbooks from disc in parallel,
but must be serialised to write to database as sqlite is single-user.
"""
global logdataissues
global entries
logdataissues = {}
DataIssue.objects.filter(parser='logbooks').delete()
expos = Expedition.objects.all()
if len(expos) <= 1:
message = f" ! - No expeditions found. Load 'people' first"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[f"sqlfail 0000"]=message
print(message)
noexpo = ["1986", "2020", "2021",] #no expo
lostlogbook = ["1976", "1977", "1978", "1979", "1980", "1981"]
sqlfail = ["1987", "1988", "1989"] # breaks mysql with db constraint fail - debug locally first]
nologbook = noexpo + lostlogbook + sqlfail
nlbe={}
expd ={}
actuals = []
for expo in expos: # pointless as we explicitly know the years in this code.
year = expo.year
TROG['pagecache']['expedition'][year] = None # clear cache
if year in sqlfail:
print(" - Logbook for: " + year + " NO parsing attempted - known sql failures")
message = f" ! - Not even attempting to parse logbook for {year} until code fixed"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[f"sqlfail {year}"]=message
print(message)
if year not in nologbook:
if year in entries:
actuals.append(expo)
else:
print(" - No Logbook yet for: " + year) # catch case when preparing for next expo
for ex in actuals:
nlbe[ex] = LoadLogbookForExpedition(ex) # this actually loads the logbook for one expo
# tried to use map with concurrent threads - but sqlite database is not concurrent, so failed with database lock
# yt = 0
# for r in map(LoadLogbookForExpedition, actuals):
# yt = r
yt = 0
for e in nlbe:
yt += nlbe[e]
print(f"total {yt:,} log entries parsed in all expeditions")
if yt != len(trips):
print(f"** total trips in ObjStore:{len(trips):,}")
# dateRegex = re.compile(r'
", "\n\n", ltriptext).strip()
ltriptext = re.sub(r"", "_", ltriptext)
ltriptext = re.sub(r"", "''", ltriptext)
ltriptext = re.sub(r"", "'''", ltriptext)
if ltriptext == "":
message = " ! - Zero content for logbook entry!: " + tid
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
entrytuple = (ldate, tripcave, triptitle, ltriptext,
trippeople, expedition, tu, tid)
logentries.append(entrytuple)
except:
message = f" ! - Skipping logentry {year} due to exception in: {tid}"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
errorcount += 1
if errorcount >5 :
message = f" !!- TOO MANY ERRORS - aborting at '{tid}' logbook: {year}"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
return
# parser for 2003
def parser_html_03(year, expedition, txt):
global logentries
global logdataissues
tripparas = re.findall(r"(.*?)
', re.S)
# reportRegex = re.compile(r'