)? # second date
\s*(?: )?
\s* ", "\n\n", ltriptext).strip()
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
# main parser for pre-2001. simpler because the data has been hacked so much to fit it
def Parseloghtml01(year, expedition, txt):
tripparas = re.findall(" |
", "\n\n", ltriptext).strip()
#ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!]", "NONASCII", ltriptext)
ltriptext = re.sub("", "_", ltriptext)
ltriptext = re.sub("", "''", ltriptext)
ltriptext = re.sub("", "'''", ltriptext)
print ldate, trippeople.strip()
# could includ the tripid (url link for cross referencing)
EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
def Parseloghtml03(year, expedition, txt):
tripparas = re.findall("
", "\n\n", ltriptext).strip() ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext) EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0) yearlinks = [ ("2009", "2009/2009logbook.txt", Parselogwikitxt), ("2008", "2008/2008logbook.txt", Parselogwikitxt), ("2007", "2007/logbook.html", Parseloghtmltxt), ("2006", "2006/logbook/logbook_06.txt", Parselogwikitxt), ("2005", "2005/logbook.html", Parseloghtmltxt), ("2004", "2004/logbook.html", Parseloghtmltxt), ("2003", "2003/logbook.html", Parseloghtml03), ("2002", "2002/logbook.html", Parseloghtmltxt), ("2001", "2001/log.htm", Parseloghtml01), ("2000", "2000/log.htm", Parseloghtml01), ("1999", "1999/log.htm", Parseloghtml01), ("1998", "1998/log.htm", Parseloghtml01), ("1997", "1997/log.htm", Parseloghtml01), ("1996", "1996/log.htm", Parseloghtml01), ("1995", "1995/log.htm", Parseloghtml01), ("1994", "1994/log.htm", Parseloghtml01), ("1993", "1993/log.htm", Parseloghtml01), ("1992", "1992/log.htm", Parseloghtml01), ("1991", "1991/log.htm", Parseloghtml01), ] def SetDatesFromLogbookEntries(expedition): """ Sets the date_from and date_to field for an expedition based on persontrips. Then sets the expedition date_from and date_to based on the personexpeditions. """ for personexpedition in expedition.personexpedition_set.all(): persontrips = personexpedition.persontrip_set.order_by('logbook_entry__date') personexpedition.date_from = min([persontrip.logbook_entry.date for persontrip in persontrips] or [None]) personexpedition.date_to = max([persontrip.logbook_entry.date for persontrip in persontrips] or [None]) personexpedition.save() # sequencing is difficult to do lprevpersontrip = None for persontrip in persontrips: persontrip.persontrip_prev = lprevpersontrip if lprevpersontrip: lprevpersontrip.persontrip_next = persontrip lprevpersontrip.save() persontrip.persontrip_next = None lprevpersontrip = persontrip persontrip.save() # from trips rather than logbook entries, which may include events outside the expedition expedition.date_from = min([personexpedition.date_from for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_from] or [None]) expedition.date_to = max([personexpedition.date_to for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_to] or [None]) expedition.save() # The below has been replaced with the methods get_next_by_id and get_previous_by_id # # order by appearance in the logbook (done by id) # lprevlogbookentry = None # for logbookentry in expedition.logbookentry_set.order_by('id'): # logbookentry.logbookentry_prev = lprevlogbookentry # if lprevlogbookentry: # lprevlogbookentry.logbookentry_next = logbookentry # lprevlogbookentry.save() # logbookentry.logbookentry_next = None # logbookentry.save() # lprevlogbookentry = logbookentry # This combined date / number key is a weird way of doing things. Use the primary key instead. If we are going to use the date for looking up entries, we should set it up to allow multiple results. # order by date for setting the references # lprevlogbookentry = None # for logbookentry in expedition.logbookentry_set.order_by('date'): # if lprevlogbookentry and lprevlogbookentry.date == logbookentry.date: # mcount = re.search("_(\d+)$", lprevlogbookentry.href) # mc = mcount and (int(mcount.group(1)) + 1) or 1 # logbookentry.href = "%s_%d" % (logbookentry.date, mc) # else: # logbookentry.href = "%s" % logbookentry.date # logbookentry.save() # lprevlogbookentry = logbookentry def LoadLogbookForExpedition(expedition): """ Parses all logbook entries for one expedition """ #We're checking for stuff that's changed in admin before deleting it now. #print "deleting logbooks for", expedition #expedition.logbookentry_set.all().delete() #models.PersonTrip.objects.filter(person_expedition__expedition=expedition).delete() expowebbase = os.path.join(settings.EXPOWEB, "years") year = str(expedition.year) for lyear, lloc, parsefunc in yearlinks: if lyear == year: break fin = open(os.path.join(expowebbase, lloc)) print "opennning", lloc txt = fin.read().decode("latin1") fin.close() parsefunc(year, expedition, txt) SetDatesFromLogbookEntries(expedition) return "TOLOAD: " + year + " " + str(expedition.personexpedition_set.all()[1].logbookentry_set.count()) + " " + str(models.PersonTrip.objects.filter(person_expedition__expedition=expedition).count()) def LoadLogbooks(): """ This is the master function for parsing all logbooks into the Troggle database. Requires yearlinks, which is a list of tuples for each expedition with expedition year, logbook path, and parsing function. """ #Deletion has been moved to a seperate function to enable the non-destructive importing #models.LogbookEntry.objects.all().delete() expowebbase = os.path.join(settings.EXPOWEB, "years") #yearlinks = [ ("2001", "2001/log.htm", Parseloghtml01), ] #overwrite #yearlinks = [ ("1996", "1996/log.htm", Parseloghtml01),] # overwrite for year, lloc, parsefunc in yearlinks: expedition = models.Expedition.objects.filter(year = year)[0] fin = open(os.path.join(expowebbase, lloc)) txt = fin.read().decode("latin1") fin.close() parsefunc(year, expedition, txt) SetDatesFromLogbookEntries(expedition)