expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-12-14 00:37:12 +00:00
Code Issues 18 Projects 3 Activity

delete old forms, templates. fix logdataissues

Browse Source
This commit is contained in:
Philip Sargent
2021-04-23 11:43:25 +01:00
parent dbd186e299
commit 343d6cf350
8 changed files with 70 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
parsers/logbooks.py
View File

@@ -58,10 +58,12 @@ trips ={}
#
# the logbook loading section
#
def GetTripPersons(trippeople, expedition, logtime_underground):
def GetTripPersons(trippeople, expedition, logtime_underground, tid="!"):
res = [ ]
author = None
round_bracket_regex = re.compile(r"[\(\[].*?[\)\]]")
if tid =="!":
tid = expedition.year + "." + tripperson
for tripperson in re.split(r",|\+|&|&(?!\w+;)| and ", trippeople):
tripperson = tripperson.strip()
mul = re.match(r"<u>(.*?)</u>$(?i)", tripperson)
@@ -71,10 +73,10 @@ def GetTripPersons(trippeople, expedition, logtime_underground):
tripperson = re.sub(round_bracket_regex, "", tripperson).strip()
personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower())
if not personyear:
message = "No name match for: ||'%s'|| in year '%s'" % (tripperson, expedition.year)
message = f" ! - {expedition.year} No name match for: '{tripperson}' "
print(message)
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[expedition.year + "~" + tripperson]=message
logdataissues[tid]=message
res.append((personyear, logtime_underground))
if mul:
author = personyear
@@ -84,37 +86,18 @@ def GetTripPersons(trippeople, expedition, logtime_underground):
author = res[-1][0]
return res, author
def GetTripCave(place):
try:
katastNumRes=[]
katastNumRes=list(Cave.objects.filter(kataster_number=int(place)))
except ValueError:
message = " ! - ValueError on finding place " + str(place) + " entered. " + tripdate + " - " + year
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues["author"]=message
officialNameRes=list(Cave.objects.filter(official_name=place))
tripCaveRes=officialNameRes+katastNumRes
if len(tripCaveRes)==1:
return tripCaveRes[0]
elif len(tripCaveRes)>1:
message = " ! - Ambiguous place " + str(place) + " entered. " + tripdate + " - " + year + " " + str(tripCaveRes)
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues["author"]=message
return tripCaveRes[0]
else:
print((" " , place))
message = " ! - No cave found for place:" + str(place) + tripdate + " - " + year
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues["author"]=message
return None
def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki"):
def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki", tid="!"):
""" saves a logbook entry and related persontrips
Does NOT save the expeditionday_id - all NULLs. why?
"""
trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground)
try:
trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground, tid=tid)
except:
message = " ! - Skipping logentry: %s - GetTripPersons FAIL in year '%s'" % (title, expedition.year)
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues["title"]=message
return
if not author:
print(" ! - Skipping logentry: " + title + " - no author for entry")
message = " ! - Skipping logentry: %s - no author for entry in year '%s'" % (title, expedition.year)
@@ -264,7 +247,8 @@ def Parseloghtmltxt(year, expedition, txt):
logbook_entry_count = 0
for trippara in tripparas:
logbook_entry_count += 1
tid= "n{}-s{:02d}".format(year,logbook_entry_count)
s = re.match(r'''(?x)(?:\s*<div\sclass="tripdate"\sid=".*?">.*?</div>\s*<p>)? # second date
\s*(?:<a\s+id="(.*?)"\s*/>\s*</a>)?
\s*<div\s+class="tripdate"\s*(?:id="(.*?)")?>(.*?)</div>(?:<p>)?
@@ -298,7 +282,7 @@ def Parseloghtmltxt(year, expedition, txt):
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext,
trippeople=trippeople, expedition=expedition, logtime_underground=0,
entry_type="html")
entry_type="html", tid=tid)
EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
"html", tripid1, logbook_entry_count)
@@ -314,13 +298,13 @@ def Parseloghtml01(year, expedition, txt):
logbook_entry_count = 0
for trippara in tripparas:
logbook_entry_count += 1
tid= f"{year}.s{logbook_entry_count:02d}"
try:
tripentry = year + "." + str(logbook_entry_count)
s = re.match(r"(?s)\s*(?:<p>)?(.*?)</?p>(.*)$(?i)", trippara)
if not s:
message = " ! - Skipping logentry on failure to parse header: " + tripentry + trippara[:300] + "..."
message = " ! - Skipping logentry on failure to parse header: " + tid + trippara[:300] + "..."
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tripentry]=message
logdataissues[tid]=message
print(message)
break
tripheader, triptext = s.group(1), s.group(2)
@@ -356,23 +340,35 @@ def Parseloghtml01(year, expedition, txt):
entrytuple = (ldate, tripcave, triptitle, ltriptext,
trippeople, expedition, tu, "html01", tripid)
logentries.append(entrytuple)
try:
EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext,
trippeople=trippeople, expedition=expedition, logtime_underground=0,
entry_type="html")
EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext,
trippeople=trippeople, expedition=expedition, logtime_underground=0,
entry_type="html")
EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
"html01", tripid, logbook_entry_count)
except:
message = " ! - Enter log entry into database FAIL exception in: " + tid
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
try:
EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
"html01", tripid, logbook_entry_count)
except:
message = " ! - Enter log entry into ObjectStore FAIL exception in: " + tid
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
except:
message = " ! - Skipping logentry due to exception in: " + tripentry
message = " ! - Skipping logentry due to exception in: " + tid
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tripentry]=message
logdataissues[tid]=message
print(message)
errorcount += 1
if errorcount >5 :
message = " !!- TOO MANY ERRORS - aborting logbook: " + year
message = f" !!- TOO MANY ERRORS - aborting at '{tid}' logbook: {year}"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tripentry]=message
logdataissues[tid]=message
print(message)
return
@@ -385,12 +381,13 @@ def Parseloghtml03(year, expedition, txt):
logbook_entry_count = 0
for trippara in tripparas:
logbook_entry_count += 1
tid= f"{year}.s{logbook_entry_count:02d}"
s = re.match(r"(?s)\s*<p>(.*?)</p>(.*)$", trippara)
if not ( s ) :
message = " ! - Skipping logentry on failure to parse Parseloghtml03: {} {} {}...".format(tripentry,s,trippara[:300])
message = " ! - Skipping logentry on failure to parse Parseloghtml03: {} {} {}...".format(tid,s,trippara[:300])
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tripentry]=message
logdataissues[tid]=message
print(message)
break
@@ -415,7 +412,6 @@ def Parseloghtml03(year, expedition, txt):
ltriptext = re.sub(r"<p>", "\n\n", ltriptext).strip()
ltriptext = re.sub(r"[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext)
tid= "n{}-s{:02d}".format(str(ldate),logbook_entry_count)
entrytuple = (ldate, tripcave, triptitle, ltriptext,
trippeople, expedition, tu, "html03", tid)
@@ -477,21 +473,23 @@ def LoadLogbookForExpedition(expedition, expect):
def cleanerrors(year):
global logdataissues
print(f' - CLEAN {year} {len(logdataissues)} data issues in total')
dataissues = DataIssue.objects.filter(parser='logbooks')
for di in dataissues:
ph = "t" + year + "-"
ph = year
if re.search(ph, di.message) is not None:
print(f' - CLEANING dataissue {di.message}')
di.delete()
for te, content in logdataissues:
print(f' - CLEAN {year} {len(logdataissues)} {type(logdataissues)} data issues for this year')
dellist = []
for key, value in logdataissues.items():
# tripentry = year + "." + str(logbook_entry_count)
print(f' - CLEAN {te}')
if te.startswith(year + "."):
print(f' - CLEANING logdataissue {te}')
logdataissues.pop(te)
print(f' - CLEAN [{key}]')
if key.startswith(year + "."):
print(f' - CLEANING logdataissues [{key:12}]: value ')
dellist.append(key)
for i in dellist:
del logdataissues[i]
cleanerrors(expedition.year)