xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

WORKING Dj2.2.24 & 2.2.25 py3.7

Browse Source
This commit is contained in:
Philip Sargent
2022-03-02 21:15:24 +00:00
parent af50d4912d
commit 601fc2cffc
12 changed files with 97 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
parsers/logbooks.py
View File

@@ -659,9 +659,10 @@ def LoadLogbooks():
expos = Expedition.objects.all()
if len(expos) <= 1:
print(" ! No expeditions found. Load 'people' first.\n")
nologbook = ["1976", "1977", "1978", "1979", "1980", "1981",
"1987", "1988", "1989", # breaks mysql with db constraint fail - debug locally first
"1986", "2020", "2021",] #no expo
noexpo = ["1986", "2020", "2021",] #no expo
lostlogbook = ["1976", "1977", "1978", "1979", "1980", "1981"]
sqlfail = ["1987", "1988", "1989"] # breaks mysql with db constraint fail - debug locally first]
nologbook = noexpo + lostlogbook + sqlfail
entries = { "2019": 20, "2018": 74, "2017": 60, "2016": 81, "2015": 79,
"2014": 65, "2013": 51, "2012": 75, "2011": 68, "2010": 22, "2009": 52,
"2008": 49, "2007": 111, "2006": 60, "2005": 55, "2004": 76, "2003": 40, "2002": 31,
@@ -678,6 +679,12 @@ def LoadLogbooks():
with open("loadlogbk.log", "a") as log:
for expo in expos:
TROG['pagecache']['expedition'][expo.year] = None # clear cache
if expo.year in sqlfail:
print(" - Logbook for: " + expo.year + " NO parsing attempted - known sql failures")
message = f" ! - Not even attempting to parse logbook for {expo.year} until code fixed"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[f"sqlfail {expo.year}"]=message
if expo.year not in nologbook:
print((" - Logbook for: " + expo.year))
if expo.year in entries:
@@ -686,7 +693,7 @@ def LoadLogbooks():
nlbe[expo.year]=numentries
expd[expo.year]= 0
else:
print((" - No Logbook yet for: " + expo.year)) # catch case when preparing for next expo
print(" - No Logbook yet for: " + expo.year) # catch case when preparing for next expo
print("** total trips in ObjStore:", len(trips))
#for i in logdataissues:
# print("{:15s}: {}".format(i, logdataissues[i]))
@@ -700,11 +707,19 @@ def LoadLogbooks():
yt += expd[y]
print("total {} log entries in all expeditions".format(yt))
with shelve.open('logbktrips.shelve',writeback=True) as odb:
for lbe in trips:
odb[lbe]=trips[lbe]
odb.sync()
odb.close()
try:
shelvfilenm = 'logbktrips.shelve' # ".db" automatically apended after python 3.8
with shelve.open(shelvfilenm, writeback=True) as odb:
for lbe in trips:
odb[lbe]=trips[lbe]
odb.sync()
odb.close()
except:
message = f" ! - Failed store cached logbooks in '{shelvfilenm}.db' - Delete old file and try again"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues["Shelve Fail"]=message
print(message)
# dateRegex = re.compile(r'<span\s+class="date">(\d\d\d\d)-(\d\d)-(\d\d)</span>', re.S)
# expeditionYearRegex = re.compile(r'<span\s+class="expeditionyear">(.*?)</span>', re.S)