", "\n\n", ltriptext).strip()
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
# main parser for pre-2001. simpler because the data has been hacked so much to fit it
def Parseloghtml01(year, expedition, txt):
tripparas = re.findall("
|
", "\n\n", ltriptext).strip()
#ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!]", "NONASCII", ltriptext)
ltriptext = re.sub("", "_", ltriptext)
ltriptext = re.sub("", "''", ltriptext)
ltriptext = re.sub("", "'''", ltriptext)
#print ldate, trippeople.strip()
# could includ the tripid (url link for cross referencing)
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
def Parseloghtml03(year, expedition, txt):
tripparas = re.findall("
", "\n\n", ltriptext).strip() ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext) EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0) yearlinks = [ ("2008", "2008/2008logbook.txt", Parselogwikitxt), #("2007", "2007/2007logbook.txt", Parselogwikitxt), ("2006", "2006/logbook/logbook_06.txt", Parselogwikitxt), ("2005", "2005/logbook.html", Parseloghtmltxt), ("2004", "2004/logbook.html", Parseloghtmltxt), ("2003", "2003/logbook.html", Parseloghtml03), ("2002", "2002/logbook.html", Parseloghtmltxt), ("2001", "2001/log.htm", Parseloghtml01), ("2000", "2000/log.htm", Parseloghtml01), ("1999", "1999/log.htm", Parseloghtml01), ("1998", "1998/log.htm", Parseloghtml01), ("1997", "1997/log.htm", Parseloghtml01), ("1996", "1996/log.htm", Parseloghtml01), ("1995", "1995/log.htm", Parseloghtml01), ("1994", "1994/log.htm", Parseloghtml01), ("1993", "1993/log.htm", Parseloghtml01), ] def SetDatesFromLogbookEntries(expedition): for personexpedition in expedition.personexpedition_set.all(): persontrips = personexpedition.persontrip_set.order_by('date') personexpedition.date_from = min([persontrip.date for persontrip in persontrips] or [None]) personexpedition.date_to = max([persontrip.date for persontrip in persontrips] or [None]) personexpedition.save() # The below is all unnecessary, just use the built in get_previous_by_date and get_next_by_date # lprevpersontrip = None # for persontrip in persontrips: # persontrip.persontrip_prev = lprevpersontrip # if lprevpersontrip: # lprevpersontrip.persontrip_next = persontrip # lprevpersontrip.save() # persontrip.persontrip_next = None # lprevpersontrip = persontrip # persontrip.save() # from trips rather than logbook entries, which may include events outside the expedition expedition.date_from = min([personexpedition.date_from for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_from] or [None]) expedition.date_to = max([personexpedition.date_to for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_to] or [None]) expedition.save() # The below has been replaced with the methods get_next_by_id and get_previous_by_id # # order by appearance in the logbook (done by id) # lprevlogbookentry = None # for logbookentry in expedition.logbookentry_set.order_by('id'): # logbookentry.logbookentry_prev = lprevlogbookentry # if lprevlogbookentry: # lprevlogbookentry.logbookentry_next = logbookentry # lprevlogbookentry.save() # logbookentry.logbookentry_next = None # logbookentry.save() # lprevlogbookentry = logbookentry # This combined date / number key is a weird way of doing things. Use the primary key instead. If we are going to use the date for looking up entries, we should set it up to allow multiple results. # order by date for setting the references # lprevlogbookentry = None # for logbookentry in expedition.logbookentry_set.order_by('date'): # if lprevlogbookentry and lprevlogbookentry.date == logbookentry.date: # mcount = re.search("_(\d+)$", lprevlogbookentry.href) # mc = mcount and (int(mcount.group(1)) + 1) or 1 # logbookentry.href = "%s_%d" % (logbookentry.date, mc) # else: # logbookentry.href = "%s" % logbookentry.date # logbookentry.save() # lprevlogbookentry = logbookentry def LoadLogbookForExpedition(expedition): """ Parses all logbook entries for one expedition """ #We're checking for stuff that's changed in admin before deleting it now. #print "deleting logbooks for", expedition #expedition.logbookentry_set.all().delete() #models.PersonTrip.objects.filter(person_expedition__expedition=expedition).delete() expowebbase = os.path.join(settings.EXPOWEB, "years") year = str(expedition.year) for lyear, lloc, parsefunc in yearlinks: if lyear == year: break fin = open(os.path.join(expowebbase, lloc)) txt = fin.read() fin.close() parsefunc(year, expedition, txt) SetDatesFromLogbookEntries(expedition) return "TOLOAD: " + year + " " + str(expedition.personexpedition_set.all()[1].logbookentry_set.count()) + " " + str(models.PersonTrip.objects.filter(person_expedition__expedition=expedition).count()) def LoadLogbooks(): """ This is the master function for parsing all logbooks into the Troggle database. Requires yearlinks, which is a list of tuples for each expedition with expedition year, logbook path, and parsing function. """ #Deletion has been moved to a seperate function to enable the non-destructive importing #models.LogbookEntry.objects.all().delete() expowebbase = os.path.join(settings.EXPOWEB, "years") #yearlinks = [ ("2001", "2001/log.htm", Parseloghtml01), ] #overwrite #yearlinks = [ ("1996", "1996/log.htm", Parseloghtml01),] # overwrite for year, lloc, parsefunc in yearlinks: expedition = models.Expedition.objects.filter(year = year)[0] fin = open(os.path.join(expowebbase, lloc)) txt = fin.read() fin.close() parsefunc(year, expedition, txt) SetDatesFromLogbookEntries(expedition)