", "\n\n", ltriptext).strip()
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
# main parser for pre-2001. simpler because the data has been hacked so much to fit it
def Parseloghtml01(year, expedition, txt):
tripparas = re.findall("
|
", "\n\n", ltriptext).strip()
#ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!]", "NONASCII", ltriptext)
ltriptext = re.sub("", "_", ltriptext)
ltriptext = re.sub("", "''", ltriptext)
ltriptext = re.sub("", "'''", ltriptext)
#print ldate, trippeople.strip()
# could includ the tripid (url link for cross referencing)
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
def Parseloghtml03(year, expedition, txt):
tripparas = re.findall("
", "\n\n", ltriptext).strip() ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext) EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0) yearlinks = [ ("2008", "2008/2008logbook.txt", Parselogwikitxt), ("2007", "2007/2007logbook.txt", Parselogwikitxt), ("2006", "2006/logbook/logbook_06.txt", Parselogwikitxt), ("2005", "2005/logbook.html", Parseloghtmltxt), ("2004", "2004/logbook.html", Parseloghtmltxt), ("2003", "2003/logbook.html", Parseloghtml03), ("2002", "2002/logbook.html", Parseloghtmltxt), ("2001", "2001/log.htm", Parseloghtml01), ("2000", "2000/log.htm", Parseloghtml01), ("1999", "1999/log.htm", Parseloghtml01), ("1998", "1998/log.htm", Parseloghtml01), ("1997", "1997/log.htm", Parseloghtml01), ("1996", "1996/log.htm", Parseloghtml01), ] def SetDatesFromLogbookEntries(expedition): for personexpedition in expedition.personexpedition_set.all(): persontrips = personexpedition.persontrip_set.order_by('date') personexpedition.date_from = min([persontrip.date for persontrip in persontrips] or [None]) personexpedition.date_to = max([persontrip.date for persontrip in persontrips] or [None]) personexpedition.save() # The below is all unnecessary, just use the built in get_previous_by_date and get_next_by_date # lprevpersontrip = None # for persontrip in persontrips: # persontrip.persontrip_prev = lprevpersontrip # if lprevpersontrip: # lprevpersontrip.persontrip_next = persontrip # lprevpersontrip.save() # persontrip.persontrip_next = None # lprevpersontrip = persontrip # persontrip.save() # from trips rather than logbook entries, which may include events outside the expedition expedition.date_from = min([personexpedition.date_from for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_from] or [None]) expedition.date_to = max([personexpedition.date_to for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_to] or [None]) expedition.save() # The below has been replaced with the methods get_next_by_id and get_previous_by_id # # order by appearance in the logbook (done by id) # lprevlogbookentry = None # for logbookentry in expedition.logbookentry_set.order_by('id'): # logbookentry.logbookentry_prev = lprevlogbookentry # if lprevlogbookentry: # lprevlogbookentry.logbookentry_next = logbookentry # lprevlogbookentry.save() # logbookentry.logbookentry_next = None # logbookentry.save() # lprevlogbookentry = logbookentry # This combined date / number key is a weird way of doing things. Use the primary key instead. If we are going to use the date for looking up entries, we should set it up to allow multiple results. # # order by date for setting the references # lprevlogbookentry = None # for logbookentry in expedition.logbookentry_set.order_by('date'): # if lprevlogbookentry and lprevlogbookentry.date == logbookentry.date: # mcount = re.search("_(\d+)$", lprevlogbookentry.href) # mc = mcount and (int(mcount.group(1)) + 1) or 1 # logbookentry.href = "%s_%d" % (logbookentry.date, mc) # else: # logbookentry.href = "%s" % logbookentry.date # logbookentry.save() # lprevlogbookentry = logbookentry def LoadLogbookForExpedition(expedition): print "deleting logbooks for", expedition expedition.logbookentry_set.all().delete() models.PersonTrip.objects.filter(person_expedition__expedition=expedition).delete() expowebbase = os.path.join(settings.EXPOWEB, "years") year = str(expedition.year) for lyear, lloc, parsefunc in yearlinks: if lyear == year: break fin = open(os.path.join(expowebbase, lloc)) txt = fin.read() fin.close() parsefunc(year, expedition, txt) SetDatesFromLogbookEntries(expedition) return "TOLOAD: " + year + " " + str(expedition.personexpedition_set.all()[1].logbookentry_set.count()) + " " + str(models.PersonTrip.objects.filter(person_expedition__expedition=expedition).count()) def LoadLogbooks(): models.LogbookEntry.objects.all().delete() expowebbase = os.path.join(settings.EXPOWEB, "years") #yearlinks = [ ("2001", "2001/log.htm", Parseloghtml01), ] #overwrite #yearlinks = [ ("1996", "1996/log.htm", Parseloghtml01),] # overwrite for year, lloc, parsefunc in yearlinks: expedition = models.Expedition.objects.filter(year = year)[0] fin = open(os.path.join(expowebbase, lloc)) txt = fin.read() fin.close() parsefunc(year, expedition, txt) SetDatesFromLogbookEntries(expedition)