", "\n\n", ltriptext).strip()
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
# main parser for pre-2001. simpler because the data has been hacked so much to fit it
def Parseloghtml01(year, expedition, txt):
tripparas = re.findall("
|
", "\n\n", ltriptext).strip()
#ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!]", "NONASCII", ltriptext)
ltriptext = re.sub("", "_", ltriptext)
ltriptext = re.sub("", "''", ltriptext)
ltriptext = re.sub("", "'''", ltriptext)
#print ldate, trippeople.strip()
# could include the tripid (url link for cross referencing)
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
def Parseloghtml03(year, expedition, txt):
tripparas = re.findall("
", "\n\n", ltriptext).strip() ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext) EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0) yearlinks = [ ("2008", "2008/2008logbook.txt", Parselogwikitxt), ("2007", "2007/2007logbook.txt", Parselogwikitxt), ("2006", "2006/logbook/logbook_06.txt", Parselogwikitxt), ("2005", "2005/logbook.html", Parseloghtmltxt), ("2004", "2004/logbook.html", Parseloghtmltxt), ("2003", "2003/logbook.html", Parseloghtml03), ("2002", "2002/logbook.html", Parseloghtmltxt), ("2001", "2001/log.htm", Parseloghtml01), ("2000", "2000/log.htm", Parseloghtml01), ("1999", "1999/log.htm", Parseloghtml01), ("1998", "1998/log.htm", Parseloghtml01), ("1997", "1997/log.htm", Parseloghtml01), ("1996", "1996/log.htm", Parseloghtml01), ("1995", "1995/log.htm", Parseloghtml01), ("1994", "1994/log.htm", Parseloghtml01), ("1993", "1993/log.htm", Parseloghtml01), ] def SetDatesFromLogbookEntries(expedition): for personexpedition in expedition.personexpedition_set.all(): persontrips = personexpedition.persontrip_set.order_by('date') personexpedition.date_from = min([persontrip.date for persontrip in persontrips] or [None]) personexpedition.date_to = max([persontrip.date for persontrip in persontrips] or [None]) personexpedition.save() # The below is all unnecessary, just use the built in get_previous_by_date and get_next_by_date # it might be if it f***ing worked! but it doesn't does it! lprevpersontrip = None for persontrip in persontrips: persontrip.persontrip_prev = lprevpersontrip if lprevpersontrip: lprevpersontrip.persontrip_next = persontrip lprevpersontrip.save() persontrip.persontrip_next = None lprevpersontrip = persontrip persontrip.save() # from trips rather than logbook entries, which may include events outside the expedition (so what?) expedition.date_from = min([personexpedition.date_from for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_from] or [None]) expedition.date_to = max([personexpedition.date_to for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_to] or [None]) expedition.save() # The below has been replaced with the methods get_next_by_id and get_previous_by_id # (this might be true, but it's sort of in the order of the logbook. # [Maybe we'd want to sort it by date and display it in that order, in which case it's tough luck! - JGT] # # order by appearance in the logbook (done by id) # lprevlogbookentry = None # for logbookentry in expedition.logbookentry_set.order_by('id'): # logbookentry.logbookentry_prev = lprevlogbookentry # if lprevlogbookentry: # lprevlogbookentry.logbookentry_next = logbookentry # lprevlogbookentry.save() # logbookentry.logbookentry_next = None # logbookentry.save() # lprevlogbookentry = logbookentry # This combined date / number key is a weird way of doing things. Use the primary key instead. # If we are going to use the date for looking up entries, we should set it up to allow multiple results. # [Not weird. Very common. A date is meaningful and almost gets there. Simply needs small amount of disambiguation to make it a primary key -- JGT] # # order by date for setting the references # lprevlogbookentry = None # for logbookentry in expedition.logbookentry_set.order_by('date'): # if lprevlogbookentry and lprevlogbookentry.date == logbookentry.date: # mcount = re.search("_(\d+)$", lprevlogbookentry.href) # mc = mcount and (int(mcount.group(1)) + 1) or 1 # logbookentry.href = "%s_%d" % (logbookentry.date, mc) # else: # logbookentry.href = "%s" % logbookentry.date # logbookentry.save() # lprevlogbookentry = logbookentry def LoadLogbookForExpedition(expedition): print "deleting logbooks for", expedition expedition.logbookentry_set.all().delete() models.PersonTrip.objects.filter(person_expedition__expedition=expedition).delete() expowebbase = os.path.join(settings.EXPOWEB, "years") year = str(expedition.year) for lyear, lloc, parsefunc in yearlinks: if lyear == year: break fin = open(os.path.join(expowebbase, lloc)) txt = fin.read() fin.close() parsefunc(year, expedition, txt) SetDatesFromLogbookEntries(expedition) return "TOLOAD: " + year + " " + str(expedition.personexpedition_set.all()[1].logbookentry_set.count()) + " " + str(models.PersonTrip.objects.filter(person_expedition__expedition=expedition).count()) def LoadLogbooks(): models.LogbookEntry.objects.all().delete() expowebbase = os.path.join(settings.EXPOWEB, "years") #yearlinks = [ ("2001", "2001/log.htm", Parseloghtml01), ] #overwrite #yearlinks = [ ("1993", "1993/log.htm", Parseloghtml01),] # overwrite for year, lloc, parsefunc in yearlinks: expedition = models.Expedition.objects.filter(year = year)[0] fin = open(os.path.join(expowebbase, lloc)) txt = fin.read() fin.close() parsefunc(year, expedition, txt) SetDatesFromLogbookEntries(expedition)