mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2024-11-22 07:11:52 +00:00
[svn] Julian playing with the logbooks and expoyears
Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8180 by julian @ 1/18/2009 3:59 PM
This commit is contained in:
substantialnoninfringinguser
parent
b950ee70f7
commit
f229ff35f9
@ -18,7 +18,10 @@ user.is_superuser = True
|
||||
user.save()
|
||||
|
||||
import parsers.cavetab
|
||||
import parsers.people
|
||||
parsers.people.LoadPersonsExpos()
|
||||
import parsers.logbooks
|
||||
parsers.logbooks.LoadLogbooks()
|
||||
import parsers.QMs
|
||||
import parsers.survex
|
||||
import parsers.surveys
|
||||
@ -12,8 +12,8 @@ from models_survex import *
|
||||
class Expedition(models.Model):
|
||||
year = models.CharField(max_length=20, unique=True)
|
||||
name = models.CharField(max_length=100)
|
||||
start_date = models.DateField(blank=True,null=True)
|
||||
end_date = models.DateField(blank=True,null=True)
|
||||
date_from = models.DateField(blank=True,null=True)
|
||||
date_to = models.DateField(blank=True,null=True)
|
||||
|
||||
def __unicode__(self):
|
||||
return self.year
|
||||
@ -47,14 +47,24 @@ class Person(models.Model):
|
||||
blurb = models.TextField(blank=True,null=True)
|
||||
class Meta:
|
||||
verbose_name_plural = "People"
|
||||
|
||||
def __unicode__(self):
|
||||
return "%s %s" % (self.first_name, self.last_name)
|
||||
if self.last_name:
|
||||
return "%s %s" % (self.first_name, self.last_name)
|
||||
return self.first_name
|
||||
|
||||
# this should be a member entry
|
||||
def href(self):
|
||||
if self.last_name:
|
||||
return "%s_%s" % (self.first_name.lower(), self.last_name.lower())
|
||||
return self.first_name.lower()
|
||||
|
||||
|
||||
class PersonExpedition(models.Model):
|
||||
expedition = models.ForeignKey(Expedition)
|
||||
person = models.ForeignKey(Person)
|
||||
from_date = models.DateField(blank=True,null=True)
|
||||
to_date = models.DateField(blank=True,null=True)
|
||||
date_from = models.DateField(blank=True,null=True)
|
||||
date_to = models.DateField(blank=True,null=True)
|
||||
is_guest = models.BooleanField(default=False)
|
||||
nickname = models.CharField(max_length=100,blank=True,null=True)
|
||||
|
||||
@ -86,8 +96,13 @@ class PersonExpedition(models.Model):
|
||||
|
||||
def __unicode__(self):
|
||||
return "%s: (%s)" % (self.person, self.expedition)
|
||||
|
||||
|
||||
|
||||
def name(self):
|
||||
if self.nickname:
|
||||
return "%s (%s) %s" % (self.person.first_name, self.nickname, self.person.last_name)
|
||||
if self.person.last_name:
|
||||
return "%s %s" % (self.person.first_name, self.person.last_name)
|
||||
return self.person.first_name
|
||||
|
||||
|
||||
#class LogbookSentanceRating(models.Model):
|
||||
@ -187,13 +202,18 @@ class Cave(models.Model):
|
||||
|
||||
class LogbookEntry(models.Model):
|
||||
date = models.DateField()
|
||||
expedition = models.ForeignKey(Expedition,blank=True,null=True) # yes this is double-
|
||||
author = models.ForeignKey(PersonExpedition,blank=True,null=True) # the person who writes it up doesn't have to have been on the trip
|
||||
title = models.CharField(max_length=200)
|
||||
cave = models.ForeignKey(Cave,blank=True,null=True)
|
||||
place = models.CharField(max_length=100,blank=True,null=True)
|
||||
text = models.TextField()
|
||||
|
||||
logbookentry_next = models.ForeignKey('LogbookEntry', related_name='pnext', blank=True,null=True)
|
||||
logbookentry_prev = models.ForeignKey('LogbookEntry', related_name='pprev', blank=True,null=True)
|
||||
|
||||
class Meta:
|
||||
verbose_name_plural = "Logbook Entries"
|
||||
verbose_name_plural = "Logbook Entries"
|
||||
# several PersonTrips point in to this object
|
||||
|
||||
def __unicode__(self):
|
||||
@ -206,9 +226,12 @@ class PersonTrip(models.Model):
|
||||
# possibly a trip has a plurality of triplets pointing into it
|
||||
place = models.CharField(max_length=100)
|
||||
date = models.DateField()
|
||||
time_underground = models.CharField(max_length=100)
|
||||
time_underground = models.FloatField()
|
||||
logbook_entry = models.ForeignKey(LogbookEntry)
|
||||
is_logbook_entry_author = models.BooleanField()
|
||||
|
||||
persontrip_next = models.ForeignKey('PersonTrip', related_name='pnext', blank=True,null=True)
|
||||
persontrip_prev = models.ForeignKey('PersonTrip', related_name='pprev', blank=True,null=True)
|
||||
|
||||
def __unicode__(self):
|
||||
return "%s %s (%s)" % (self.person_expedition, self.place, self.date)
|
||||
|
||||
@ -15,6 +15,8 @@ def weighted_choice(lst):
|
||||
def randomLogbookSentence():
|
||||
randSent={}
|
||||
|
||||
# needs to handle empty logbooks without crashing
|
||||
|
||||
#Choose a random logbook entry
|
||||
randSent['entry']=LogbookEntry.objects.order_by('?')[0]
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
from django.shortcuts import render_to_response
|
||||
from troggle.expo.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry
|
||||
import troggle.settings as settings
|
||||
from troggle.parsers.logbooks import LoadLogbookForExpedition
|
||||
import search
|
||||
import re
|
||||
|
||||
@ -8,17 +9,33 @@ def personindex(request):
|
||||
persons = Person.objects.all()
|
||||
return render_to_response('personindex.html', {'persons': persons, 'settings': settings})
|
||||
|
||||
def person(request, person_id, first_name, last_name):
|
||||
if first_name == '' or last_name == '':
|
||||
person = Person.objects.filter(id = person_id)[0]
|
||||
else:
|
||||
person = Person.objects.filter(first_name = first_name, last_name = last_name)[0]
|
||||
def expedition(request, expeditionname):
|
||||
year = int(expeditionname)
|
||||
expedition = Expedition.objects.get(year=year)
|
||||
expedition_next = Expedition.objects.filter(year=year+1) and Expedition.objects.get(year=year+1) or None
|
||||
expedition_prev = Expedition.objects.filter(year=year-1) and Expedition.objects.get(year=year-1) or None
|
||||
message = "No message"
|
||||
if "reload" in request.GET:
|
||||
message = LoadLogbookForExpedition(expedition)
|
||||
|
||||
logbookentries = expedition.logbookentry_set.order_by('date')
|
||||
return render_to_response('expedition.html', {'expedition': expedition, 'expedition_next':expedition_next, 'expedition_prev':expedition_prev, 'logbookentries':logbookentries, 'message':message, 'settings': settings})
|
||||
|
||||
def person(request, name):
|
||||
persons = Person.objects.all()
|
||||
for person in persons:
|
||||
if person.href() == name:
|
||||
break
|
||||
person = None
|
||||
return render_to_response('person.html', {'person': person, 'settings': settings})
|
||||
|
||||
def logbookentry(request, logbookentry_id):
|
||||
logbookentry = LogbookEntry.objects.filter(id = logbookentry_id)[0]
|
||||
|
||||
|
||||
return render_to_response('logbookentry.html', {'logbookentry': logbookentry, 'settings': settings})
|
||||
|
||||
|
||||
def logbookSearch(request, extra):
|
||||
query_string = ''
|
||||
found_entries = None
|
||||
|
||||
@ -3,9 +3,12 @@ from troggle.expo.models import Cave, Expedition, Person, LogbookEntry, PersonEx
|
||||
import troggle.settings as settings
|
||||
from django import forms
|
||||
from django.db.models import Q
|
||||
from troggle.parsers.people import LoadPersonsExpos
|
||||
import re
|
||||
import randSent
|
||||
|
||||
from django.core.urlresolvers import reverse
|
||||
|
||||
def stats(request):
|
||||
statsDict={}
|
||||
statsDict['expoCount'] = int(Expedition.objects.count())
|
||||
@ -15,8 +18,13 @@ def stats(request):
|
||||
return render_to_response('statistics.html', statsDict)
|
||||
|
||||
def frontPage(request):
|
||||
|
||||
return render_to_response('index.html', {'randSent':randSent.randomLogbookSentence(),'settings':settings})
|
||||
message = "no test message" #reverse('personn', kwargs={"name":"hkjhjh"})
|
||||
if "reload" in request.GET:
|
||||
message = LoadPersonsExpos()
|
||||
message = "Reloaded personexpos"
|
||||
#'randSent':randSent.randomLogbookSentence(),
|
||||
expeditions = Expedition.objects.all()
|
||||
return render_to_response('index.html', {'expeditions':expeditions, 'settings':settings, "message":message})
|
||||
|
||||
def calendar(request,year):
|
||||
week=['S','S','M','T','W','T','F']
|
||||
@ -27,4 +35,3 @@ def calendar(request,year):
|
||||
dictToPass=locals()
|
||||
dictToPass.update({'settings':settings})
|
||||
return render_to_response('calendar.html', dictToPass)
|
||||
|
||||
@ -1,22 +1,42 @@
|
||||
html, body, div, span, applet, object, iframe,
|
||||
h1, h2, h3, h4, h5, h6, p, blockquote, pre,
|
||||
a, abbr, acronym, address, big, cite, code,
|
||||
del, dfn, em, font, img, ins, kbd, q, s, samp,
|
||||
small, strike, strong, sub, sup, tt, var,
|
||||
dl, dt, dd, ol, ul, li,
|
||||
fieldset, form, label, legend,
|
||||
table, caption, tbody, tfoot, thead, tr, th, td
|
||||
{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
border: 0;
|
||||
outline: 0;
|
||||
font-weight: inherit;
|
||||
font-style: inherit;
|
||||
font-size: 100%;
|
||||
font-family: inherit;
|
||||
vertical-align: baseline;
|
||||
}
|
||||
|
||||
|
||||
.caption { font-size: 8pt; margin-bottom: 0pt; }
|
||||
.centre { text-align: center; }
|
||||
.plus2pt { font-size: 160%; }
|
||||
|
||||
body, td, center, ul, p, input
|
||||
ul
|
||||
{
|
||||
color: rgb(0, 0, 0);
|
||||
font-family: sans-serif;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
body
|
||||
{
|
||||
background-color: #FFFFFF;
|
||||
background-color: white;
|
||||
color: black;
|
||||
font: 100% Verdana, Arial, Helvetica, sans-serif;
|
||||
Zbackground: #666666;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
text-align: center;
|
||||
color: #000000;
|
||||
margin-left: auto;
|
||||
margin-right: auto;
|
||||
}
|
||||
|
||||
div#content
|
||||
@ -27,6 +47,7 @@ div#content
|
||||
|
||||
div#footer
|
||||
{
|
||||
clear:both;
|
||||
background-color:black;
|
||||
color:white;
|
||||
text-align:center;
|
||||
@ -34,6 +55,76 @@ div#footer
|
||||
margin-right:auto;
|
||||
}
|
||||
|
||||
div.logbookentry
|
||||
{
|
||||
text-align:left;
|
||||
}
|
||||
div.logbookentry ul.cavers
|
||||
{
|
||||
float:left;
|
||||
padding-left:20px;
|
||||
padding-right:10px;
|
||||
margin-top:0px;
|
||||
}
|
||||
td.author
|
||||
{
|
||||
background-color:yellow;
|
||||
}
|
||||
|
||||
div.logbookentry p
|
||||
{
|
||||
margin:10px;
|
||||
}
|
||||
|
||||
div#content div#col2
|
||||
{
|
||||
float:right;
|
||||
width:33%;
|
||||
background-color:#feeeed;
|
||||
}
|
||||
|
||||
div#content h2
|
||||
{
|
||||
text-align:center;
|
||||
font-size:200%;
|
||||
padding-bottom:30px;
|
||||
}
|
||||
|
||||
|
||||
table.prevnextexpeditions
|
||||
{
|
||||
width:100%;
|
||||
}
|
||||
|
||||
table.prevnextexpeditions td
|
||||
{
|
||||
padding: 2px;
|
||||
}
|
||||
|
||||
table.expeditionpersonlist
|
||||
{
|
||||
width:100%;
|
||||
}
|
||||
|
||||
table.expeditionpersonlist td
|
||||
{
|
||||
padding: 2px;
|
||||
}
|
||||
|
||||
div#content div#col1
|
||||
{
|
||||
width:66%;
|
||||
}
|
||||
table.expeditionlogbooks td
|
||||
{
|
||||
padding: 2px;
|
||||
}
|
||||
|
||||
ul#expeditionlist
|
||||
{
|
||||
width: 300px
|
||||
}
|
||||
|
||||
|
||||
#expoHeader {
|
||||
width:100%;
|
||||
|
||||
@ -7,120 +7,11 @@ import re
|
||||
import datetime
|
||||
import os
|
||||
|
||||
persontab = open(os.path.join(settings.EXPOWEB, "noinfo", "folk.csv"))
|
||||
personreader = csv.reader(persontab)
|
||||
headers = personreader.next()
|
||||
header = dict(zip(headers, range(len(headers))))
|
||||
|
||||
def LoadExpos():
|
||||
models.Expedition.objects.all().delete()
|
||||
years = headers[5:]
|
||||
years.append("2008")
|
||||
for year in years:
|
||||
y = models.Expedition(year = year, name = "CUCC expo%s" % year)
|
||||
y.save()
|
||||
print "lll", years
|
||||
|
||||
def LoadPersons():
|
||||
models.Person.objects.all().delete()
|
||||
models.PersonExpedition.objects.all().delete()
|
||||
expoers2008 = """Edvin Deadman,Kathryn Hopkins,Djuke Veldhuis,Becka Lawson,Julian Todd,Natalie Uomini,Aaron Curtis,Tony Rooke,Ollie Stevens,Frank Tully,Martin Jahnke,Mark Shinwell,Jess Stirrups,Nial Peters,Serena Povia,Olly Madge,Steve Jones,Pete Harley,Eeva Makiranta,Keith Curtis""".split(",")
|
||||
expomissing = set(expoers2008)
|
||||
|
||||
for person in personreader:
|
||||
name = person[header["Name"]]
|
||||
name = re.sub("<.*?>", "", name)
|
||||
mname = re.match("(\w+)(?:\s((?:van |ten )?\w+))?(?:\s\(([^)]*)\))?", name)
|
||||
|
||||
if mname.group(3):
|
||||
nickname = mname.group(3)
|
||||
else:
|
||||
nickname = ""
|
||||
|
||||
firstname, lastname = mname.group(1), mname.group(2) or ""
|
||||
|
||||
print firstname, lastname, "NNN", nickname
|
||||
#assert lastname == person[header[""]], person
|
||||
|
||||
pObject = models.Person(first_name = firstname,
|
||||
last_name = lastname,
|
||||
is_vfho = person[header["VfHO member"]],
|
||||
)
|
||||
|
||||
is_guest = person[header["Guest"]] == "1" # this is really a per-expo catagory; not a permanent state
|
||||
pObject.save()
|
||||
parseMugShotAndBlurb(firstname, lastname, person, header, pObject)
|
||||
|
||||
for year, attended in zip(headers, person)[5:]:
|
||||
yo = models.Expedition.objects.filter(year = year)[0]
|
||||
if attended == "1" or attended == "-1":
|
||||
pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname=nickname, is_guest=is_guest)
|
||||
pyo.save()
|
||||
|
||||
# error
|
||||
elif (firstname, lastname) == ("Mike", "Richardson") and year == "2001":
|
||||
print "Mike Richardson(2001) error"
|
||||
pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname=nickname, is_guest=is_guest)
|
||||
pyo.save()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# this fills in those peopl for whom 2008 was their first expo
|
||||
for name in expomissing:
|
||||
firstname, lastname = name.split()
|
||||
is_guest = name in ["Eeva Makiranta", "Keith Curtis"]
|
||||
print "2008:", name
|
||||
pObject = models.Person(first_name = firstname,
|
||||
last_name = lastname,
|
||||
is_vfho = False,
|
||||
mug_shot = "")
|
||||
pObject.save()
|
||||
yo = models.Expedition.objects.filter(year = "2008")[0]
|
||||
pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname="", is_guest=is_guest)
|
||||
pyo.save()
|
||||
|
||||
# Julian: the below code was causing errors and it seems like a duplication of the above. Hope I haven't broken anything by commenting it. -Aaron
|
||||
#
|
||||
# if name in expoers2008:
|
||||
# print "2008:", name
|
||||
# expomissing.discard(name) # I got an error which I think was caused by this -- python complained that a set changed size during iteration.
|
||||
# yo = models.Expedition.objects.filter(year = "2008")[0]
|
||||
# pyo = models.PersonExpedition(person = pObject, expedition = yo, is_guest=is_guest)
|
||||
# pyo.save()
|
||||
|
||||
def parseMugShotAndBlurb(firstname, lastname, person, header, pObject):
|
||||
#create mugshot Photo instance
|
||||
mugShotPath = os.path.join(settings.EXPOWEB, "folk", person[header["Mugshot"]])
|
||||
if mugShotPath[-3:]=='jpg': #if person just has an image, add it
|
||||
mugShotObj = models.Photo(
|
||||
caption="Mugshot for "+firstname+" "+lastname,
|
||||
is_mugshot=True,
|
||||
file=mugShotPath,
|
||||
)
|
||||
mugShotObj.save()
|
||||
mugShotObj.contains_person.add(pObject)
|
||||
mugShotObj.save()
|
||||
elif mugShotPath[-3:]=='htm': #if person has an html page, find the image(s) and add it. Also, add the text from the html page to the "blurb" field in his model instance.
|
||||
personPageOld=open(mugShotPath,'r').read()
|
||||
pObject.blurb=re.search('<body>.*<hr',personPageOld,re.DOTALL).group() #this needs to be refined, take care of the HTML and make sure it doesn't match beyond the blurb
|
||||
for photoFilename in re.findall('i/.*?jpg',personPageOld,re.DOTALL):
|
||||
mugShotPath=settings.EXPOWEB+"folk/"+photoFilename
|
||||
mugShotObj = models.Photo(
|
||||
caption="Mugshot for "+firstname+" "+lastname,
|
||||
is_mugshot=True,
|
||||
file=mugShotPath,
|
||||
)
|
||||
mugShotObj.save()
|
||||
mugShotObj.contains_person.add(pObject)
|
||||
mugShotObj.save()
|
||||
pObject.save()
|
||||
|
||||
#
|
||||
# the logbook loading section
|
||||
#
|
||||
def GetTripPersons(trippeople, expedition):
|
||||
def GetTripPersons(trippeople, expedition, logtime_underground):
|
||||
res = [ ]
|
||||
author = None
|
||||
for tripperson in re.split(",|\+|&|&(?!\w+;)| and ", trippeople):
|
||||
@ -133,11 +24,11 @@ def GetTripPersons(trippeople, expedition):
|
||||
personyear = expedition.GetPersonExpedition(tripperson)
|
||||
if not personyear:
|
||||
print "NoMatchFor: '%s'" % tripperson
|
||||
res.append(personyear)
|
||||
res.append((personyear, logtime_underground))
|
||||
if mul:
|
||||
author = personyear
|
||||
if not author:
|
||||
author = res[-1]
|
||||
author = res[-1][0]
|
||||
return res, author
|
||||
|
||||
def GetTripCave(place): #need to be fuzzier about matching here. Already a very slow function...
|
||||
@ -167,20 +58,22 @@ def GetTripCave(place): #need to be fuzzier about matching h
|
||||
print "No cave found for place " , place
|
||||
return
|
||||
|
||||
def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, tu):
|
||||
trippersons, author = GetTripPersons(trippeople, expedition)
|
||||
|
||||
def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground):
|
||||
trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground)
|
||||
tripCave = GetTripCave(place)
|
||||
|
||||
lbo = models.LogbookEntry(date=date, place=place, title=title[:50], text=text, author=author)
|
||||
lbo = models.LogbookEntry(date=date, place=place, title=title[:50], text=text, author=author, expedition=expedition)
|
||||
if tripCave:
|
||||
lbo.cave=tripCave
|
||||
lbo.save()
|
||||
print "ttt", date, place
|
||||
for tripperson in trippersons:
|
||||
pto = models.PersonTrip(person_expedition = tripperson, place=place, date=date, time_underground=(tu or ""),
|
||||
for tripperson, time_underground in trippersons:
|
||||
pto = models.PersonTrip(person_expedition = tripperson, place=place, date=date, time_underground=time_underground,
|
||||
logbook_entry=lbo, is_logbook_entry_author=(tripperson == author))
|
||||
pto.save()
|
||||
|
||||
|
||||
def ParseDate(tripdate, year):
|
||||
mdatestandard = re.match("(\d\d\d\d)-(\d\d)-(\d\d)", tripdate)
|
||||
mdategoof = re.match("(\d\d?)/0?(\d)/(20|19)?(\d\d)", tripdate)
|
||||
@ -216,7 +109,7 @@ def Parselogwikitxt(year, expedition, txt):
|
||||
|
||||
ldate = ParseDate(tripdate.strip(), year)
|
||||
#print "\n", tripcave, "--- ppp", trippeople, len(triptext)
|
||||
EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, tu=tu)
|
||||
EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
|
||||
|
||||
# 2002, 2004, 2005
|
||||
def Parseloghtmltxt(year, expedition, txt):
|
||||
@ -246,7 +139,7 @@ def Parseloghtmltxt(year, expedition, txt):
|
||||
ltriptext = re.sub("</p>", "", triptext)
|
||||
ltriptext = re.sub("\s*?\n\s*", " ", ltriptext)
|
||||
ltriptext = re.sub("<p>", "\n\n", ltriptext).strip()
|
||||
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, tu=tu)
|
||||
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
|
||||
|
||||
|
||||
# main parser for pre-2001. simpler because the data has been hacked so much to fit it
|
||||
@ -283,7 +176,8 @@ def Parseloghtml01(year, expedition, txt):
|
||||
|
||||
#print ldate, trippeople.strip()
|
||||
# could includ the tripid (url link for cross referencing)
|
||||
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, tu=tu)
|
||||
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
|
||||
|
||||
|
||||
def Parseloghtml03(year, expedition, txt):
|
||||
tripparas = re.findall("<hr\s*/>([\s\S]*?)(?=<hr)", txt)
|
||||
@ -312,25 +206,67 @@ def Parseloghtml03(year, expedition, txt):
|
||||
ltriptext = re.sub("\s*?\n\s*", " ", ltriptext)
|
||||
ltriptext = re.sub("<p>", "\n\n", ltriptext).strip()
|
||||
ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext)
|
||||
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, tu=tu)
|
||||
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
|
||||
|
||||
yearlinks = [
|
||||
("2008", "2008/2008logbook.txt", Parselogwikitxt),
|
||||
("2007", "2007/2007logbook.txt", Parselogwikitxt),
|
||||
("2006", "2006/logbook/logbook_06.txt", Parselogwikitxt),
|
||||
("2005", "2005/logbook.html", Parseloghtmltxt),
|
||||
("2004", "2004/logbook.html", Parseloghtmltxt),
|
||||
("2003", "2003/logbook.html", Parseloghtml03),
|
||||
("2002", "2002/logbook.html", Parseloghtmltxt),
|
||||
("2001", "2001/log.htm", Parseloghtml01),
|
||||
("2000", "2000/log.htm", Parseloghtml01),
|
||||
("1999", "1999/log.htm", Parseloghtml01),
|
||||
("1998", "1998/log.htm", Parseloghtml01),
|
||||
("1997", "1997/log.htm", Parseloghtml01),
|
||||
#("1996", "1996/log.htm", Parseloghtml01),
|
||||
]
|
||||
|
||||
def SetDatesFromLogbookEntries(expedition):
|
||||
for personexpedition in expedition.personexpedition_set.all():
|
||||
persontrips = personexpedition.persontrip_set.order_by('date')
|
||||
personexpedition.date_from = min([persontrip.date for persontrip in persontrips] or [None])
|
||||
personexpedition.date_to = max([persontrip.date for persontrip in persontrips] or [None])
|
||||
personexpedition.save()
|
||||
|
||||
lprevpersontrip = None
|
||||
for persontrip in persontrips:
|
||||
persontrip.persontrip_prev = lprevpersontrip
|
||||
if lprevpersontrip:
|
||||
lprevpersontrip.persontrip_next = persontrip
|
||||
lprevpersontrip.save()
|
||||
persontrip.persontrip_next = None
|
||||
lprevpersontrip = persontrip
|
||||
persontrip.save()
|
||||
|
||||
# from trips rather than logbook entries, which may include events outside the expedition
|
||||
expedition.date_from = min([personexpedition.date_from for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_from] or [None])
|
||||
expedition.date_to = max([personexpedition.date_to for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_to] or [None])
|
||||
expedition.save()
|
||||
|
||||
|
||||
|
||||
def LoadLogbookForExpedition(expedition):
|
||||
expedition.logbookentry_set.all().delete()
|
||||
models.PersonTrip.objects.filter(person_expedition__expedition=expedition).delete()
|
||||
expowebbase = os.path.join(settings.EXPOWEB, "years")
|
||||
year = str(expedition.year)
|
||||
for lyear, lloc, parsefunc in yearlinks:
|
||||
if lyear == year:
|
||||
break
|
||||
fin = open(os.path.join(expowebbase, lloc))
|
||||
txt = fin.read()
|
||||
fin.close()
|
||||
parsefunc(year, expedition, txt)
|
||||
SetDatesFromLogbookEntries(expedition)
|
||||
return "TOLOAD: " + year + " " + str(expedition.personexpedition_set.all()[1].logbookentry_set.count()) + " " + str(models.PersonTrip.objects.filter(person_expedition__expedition=expedition).count())
|
||||
|
||||
|
||||
def LoadLogbooks():
|
||||
models.LogbookEntry.objects.all().delete()
|
||||
expowebbase = os.path.join(settings.EXPOWEB, "years")
|
||||
yearlinks = [
|
||||
("2008", "2008/2008logbook.txt", Parselogwikitxt),
|
||||
("2007", "2007/2007logbook.txt", Parselogwikitxt),
|
||||
("2006", "2006/logbook/logbook_06.txt", Parselogwikitxt),
|
||||
("2005", "2005/logbook.html", Parseloghtmltxt),
|
||||
("2004", "2004/logbook.html", Parseloghtmltxt),
|
||||
("2003", "2003/logbook.html", Parseloghtml03),
|
||||
("2002", "2002/logbook.html", Parseloghtmltxt),
|
||||
("2001", "2001/log.htm", Parseloghtml01),
|
||||
("2000", "2000/log.htm", Parseloghtml01),
|
||||
("1999", "1999/log.htm", Parseloghtml01),
|
||||
("1998", "1998/log.htm", Parseloghtml01),
|
||||
("1997", "1997/log.htm", Parseloghtml01),
|
||||
]
|
||||
#yearlinks = [ ("2001", "2001/log.htm", Parseloghtml01), ] #overwrite
|
||||
|
||||
for year, lloc, parsefunc in yearlinks:
|
||||
@ -339,11 +275,6 @@ def LoadLogbooks():
|
||||
txt = fin.read()
|
||||
fin.close()
|
||||
parsefunc(year, expedition, txt)
|
||||
|
||||
SetDatesFromLogbookEntries(expedition)
|
||||
|
||||
# command line run through the loading stages
|
||||
# you can comment out these in turn to control what gets reloaded
|
||||
LoadExpos()
|
||||
LoadPersons()
|
||||
LoadLogbooks()
|
||||
|
||||
|
||||
119
parsers/people.py
Normal file
119
parsers/people.py
Normal file
@ -0,0 +1,119 @@
|
||||
#.-*- coding: utf-8 -*-
|
||||
|
||||
import settings
|
||||
import expo.models as models
|
||||
import csv
|
||||
import re
|
||||
import datetime
|
||||
import os
|
||||
|
||||
# Julian: the below code was causing errors and it seems like a duplication of the above. Hope I haven't broken anything by commenting it. -Aaron
|
||||
#
|
||||
# if name in expoers2008:
|
||||
# print "2008:", name
|
||||
# expomissing.discard(name) # I got an error which I think was caused by this -- python complained that a set changed size during iteration.
|
||||
# yo = models.Expedition.objects.filter(year = "2008")[0]
|
||||
# pyo = models.PersonExpedition(person = pObject, expedition = yo, is_guest=is_guest)
|
||||
# pyo.save()
|
||||
|
||||
def parseMugShotAndBlurb(firstname, lastname, person, header, pObject):
|
||||
#create mugshot Photo instance
|
||||
mugShotPath = os.path.join(settings.EXPOWEB, "folk", person[header["Mugshot"]])
|
||||
if mugShotPath[-3:]=='jpg': #if person just has an image, add it
|
||||
mugShotObj = models.Photo(
|
||||
caption="Mugshot for "+firstname+" "+lastname,
|
||||
is_mugshot=True,
|
||||
file=mugShotPath,
|
||||
)
|
||||
mugShotObj.save()
|
||||
mugShotObj.contains_person.add(pObject)
|
||||
mugShotObj.save()
|
||||
elif mugShotPath[-3:]=='htm': #if person has an html page, find the image(s) and add it. Also, add the text from the html page to the "blurb" field in his model instance.
|
||||
personPageOld=open(mugShotPath,'r').read()
|
||||
pObject.blurb=re.search('<body>.*<hr',personPageOld,re.DOTALL).group() #this needs to be refined, take care of the HTML and make sure it doesn't match beyond the blurb
|
||||
for photoFilename in re.findall('i/.*?jpg',personPageOld,re.DOTALL):
|
||||
mugShotPath=settings.EXPOWEB+"folk/"+photoFilename
|
||||
mugShotObj = models.Photo(
|
||||
caption="Mugshot for "+firstname+" "+lastname,
|
||||
is_mugshot=True,
|
||||
file=mugShotPath,
|
||||
)
|
||||
mugShotObj.save()
|
||||
mugShotObj.contains_person.add(pObject)
|
||||
mugShotObj.save()
|
||||
pObject.save()
|
||||
|
||||
|
||||
|
||||
def LoadPersonsExpos():
|
||||
|
||||
persontab = open(os.path.join(settings.EXPOWEB, "noinfo", "folk.csv"))
|
||||
personreader = csv.reader(persontab)
|
||||
headers = personreader.next()
|
||||
header = dict(zip(headers, range(len(headers))))
|
||||
|
||||
models.Expedition.objects.all().delete()
|
||||
years = headers[5:]
|
||||
years.append("2008")
|
||||
for year in years:
|
||||
y = models.Expedition(year = year, name = "CUCC expo%s" % year)
|
||||
y.save()
|
||||
print "lll", years
|
||||
|
||||
|
||||
models.Person.objects.all().delete()
|
||||
models.PersonExpedition.objects.all().delete()
|
||||
expoers2008 = """Edvin Deadman,Kathryn Hopkins,Djuke Veldhuis,Becka Lawson,Julian Todd,Natalie Uomini,Aaron Curtis,Tony Rooke,Ollie Stevens,Frank Tully,Martin Jahnke,Mark Shinwell,Jess Stirrups,Nial Peters,Serena Povia,Olly Madge,Steve Jones,Pete Harley,Eeva Makiranta,Keith Curtis""".split(",")
|
||||
expomissing = set(expoers2008)
|
||||
|
||||
for person in personreader:
|
||||
name = person[header["Name"]]
|
||||
name = re.sub("<.*?>", "", name)
|
||||
mname = re.match("(\w+)(?:\s((?:van |ten )?\w+))?(?:\s\(([^)]*)\))?", name)
|
||||
|
||||
if mname.group(3):
|
||||
nickname = mname.group(3)
|
||||
else:
|
||||
nickname = ""
|
||||
|
||||
firstname, lastname = mname.group(1), mname.group(2) or ""
|
||||
|
||||
print firstname, lastname, "NNN", nickname
|
||||
#assert lastname == person[header[""]], person
|
||||
|
||||
pObject = models.Person(first_name = firstname,
|
||||
last_name = lastname,
|
||||
is_vfho = person[header["VfHO member"]],
|
||||
)
|
||||
|
||||
is_guest = person[header["Guest"]] == "1" # this is really a per-expo catagory; not a permanent state
|
||||
pObject.save()
|
||||
parseMugShotAndBlurb(firstname, lastname, person, header, pObject)
|
||||
|
||||
for year, attended in zip(headers, person)[5:]:
|
||||
yo = models.Expedition.objects.filter(year = year)[0]
|
||||
if attended == "1" or attended == "-1":
|
||||
pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname=nickname, is_guest=is_guest)
|
||||
pyo.save()
|
||||
|
||||
# error
|
||||
elif (firstname, lastname) == ("Mike", "Richardson") and year == "2001":
|
||||
print "Mike Richardson(2001) error"
|
||||
pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname=nickname, is_guest=is_guest)
|
||||
pyo.save()
|
||||
|
||||
|
||||
# this fills in those people for whom 2008 was their first expo
|
||||
for name in expomissing:
|
||||
firstname, lastname = name.split()
|
||||
is_guest = name in ["Eeva Makiranta", "Keith Curtis"]
|
||||
print "2008:", name
|
||||
pObject = models.Person(first_name = firstname,
|
||||
last_name = lastname,
|
||||
is_vfho = False,
|
||||
mug_shot = "")
|
||||
pObject.save()
|
||||
yo = models.Expedition.objects.filter(year = "2008")[0]
|
||||
pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname="", is_guest=is_guest)
|
||||
pyo.save()
|
||||
|
||||
47
templates/expedition.html
Normal file
47
templates/expedition.html
Normal file
@ -0,0 +1,47 @@
|
||||
{% extends "base.html" %}
|
||||
{% load wiki_markup %}
|
||||
|
||||
{% block title %}Expedition {{expedition.name}}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h2>{{expedition.name}}: {{expedition.date_from}} - {{expedition.date_to}}</h2>
|
||||
|
||||
<div id="col2">
|
||||
<table class="prevnextexpeditions">
|
||||
<tr>
|
||||
<td>{% if expedition_prev %}< < <a href="{% url expedition expedition_prev.year %}">{{expedition_prev.year}}</a>{% endif %}</td>
|
||||
<td>{% if expedition_next %}> > <a href="{% url expedition expedition_next.year %}">{{expedition_next.year}}</a>{% endif %}</td>
|
||||
</tr>
|
||||
</ul>
|
||||
|
||||
<table class="expeditionpersonlist">
|
||||
<tr><th>Caver</th><th>From</th><th>To</th></tr>
|
||||
{% for personexpedition in expedition.personexpedition_set.all %}
|
||||
<tr>
|
||||
<td><a href="{% url person personexpedition.person.href%}">{{personexpedition.person}}</a></td>
|
||||
<td>{{personexpedition.date_from}}</td>
|
||||
<td>{{personexpedition.date_to}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div id="col1">
|
||||
<h3>Logbook entries</h3>
|
||||
<form action="" method="GET"><input type="submit" name="reload" value="Reload"></form>
|
||||
<p>debug message: {{message}}</p>
|
||||
|
||||
<table class="expeditionlogbooks">
|
||||
<tr><th>Date</th><th>Title</th><th>Author</th><th>Place</th></tr>
|
||||
{% for logbookentry in logbookentries %}
|
||||
<tr>
|
||||
<td>{{logbookentry.date}}</td>
|
||||
<td><a href="{% url logbookentry logbookentry.id %}">{{logbookentry.title|safe}}</td>
|
||||
<td><a href="{% url person logbookentry.author.person.href%}">{{logbookentry.author.name}}</a></td>
|
||||
<td>{{logbookentry.place}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
@ -8,9 +8,21 @@
|
||||
<h2>The unfinished front page</h2>
|
||||
<p>Some handy links into the less incomplete parts of this webpage</p>
|
||||
|
||||
<h3>{{message}}</h3>
|
||||
|
||||
<ul>
|
||||
<li><a href="{% url personindex %}">List osf people</a></li>
|
||||
<li><a href="/statistics">Statistics of what's loaded in the database</a></li>
|
||||
</ul>
|
||||
|
||||
<form action="" method="GET"><input type="submit" name="reload" value="Reload"></form>
|
||||
|
||||
<ul id="expeditionlist">
|
||||
<li>
|
||||
{% for expedition in expeditions %}
|
||||
<a href="{% url expedition expedition.year %}">{{expedition.name}}</a>
|
||||
{% endfor %}
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
@ -2,25 +2,52 @@
|
||||
{% load wiki_markup %}
|
||||
|
||||
{% block title %}Logbook {{logbookentry.id}}{% endblock %}
|
||||
{% block editLink %}<a href="{{settings.URL_ROOT}}admin/expo/logbookentry/{{logbookentry.id}}">edit </a>{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="logbookblock">
|
||||
<h2>{{logbookentry.title}} - {{logbookentry.date}}</h2>
|
||||
<h3>place (to be a link to cave shaped object): <u>{{logbookentry.place}}</u></h3>
|
||||
<ul>
|
||||
{% for persontrip in logbookentry.persontrip_set.all %}
|
||||
<li>
|
||||
<a href="/person/{{persontrip.personexpedition.person.id}}">{{persontrip.personexpedition}}</a>
|
||||
<a href="{{settings.URL_ROOT}}admin/expo/logbookentry/{{logbookentry.id}}">edit </a>
|
||||
{% ifequal persontrip.personexpedition logbookentry.author %}
|
||||
(author)
|
||||
{% endifequal %}
|
||||
{% if persontrip.timeunderground %}
|
||||
- T/U {{persontrip.timeunderground}}</p>
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
<div>{{logbookentry.text|wiki_to_html}}</div>
|
||||
</div>
|
||||
<h2>{{logbookentry.title|safe}}</h2>
|
||||
|
||||
<div id="col2">
|
||||
<p><a href="{% url expedition logbookentry.expedition.year %}">{{logbookentry.expedition.name}}</a></p>
|
||||
<p>place: {{logbookentry.place}}</p>
|
||||
|
||||
<table class="cavers">
|
||||
<tr><th>Caver</th><th>T/U</th><th>Prev</th><th>Next</th></tr>
|
||||
{% for persontrip in logbookentry.persontrip_set.all %}
|
||||
<tr>
|
||||
{% ifequal persontrip.person_expedition logbookentry.author %}
|
||||
<td class="author">
|
||||
{% else %}
|
||||
<td>
|
||||
{% endifequal %}
|
||||
<a href="{% url person persontrip.person_expedition.person.href %}">{{persontrip.person_expedition.person}}</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
{% if persontrip.timeunderground %}
|
||||
- T/U {{persontrip.timeunderground}}</p>
|
||||
{% endif %}
|
||||
</td>
|
||||
|
||||
<td>
|
||||
{% if persontrip.persontrip_prev %}
|
||||
<a href="{% url logbookentry persontrip.persontrip_prev.logbook_entry.id %}">{{persontrip.persontrip_prev.date}}</a>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if persontrip.persontrip_next %}
|
||||
<a href="{% url logbookentry persontrip.persontrip_next.logbook_entry.id %}">{{persontrip.persontrip_next.date}}</a>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div id="col1">
|
||||
<div class="logbookentry">
|
||||
<b>{{logbookentry.date}}</b>
|
||||
{{logbookentry.text|wiki_to_html}}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
@ -4,25 +4,25 @@
|
||||
{% block title %}Person {{person|wiki_to_html_short}}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="personblock"><a href="/person/{{person.id}}">{{person|wiki_to_html_short}}</a>
|
||||
<div class="personblock"><a href="{% url person person.href%}">{{person|wiki_to_html_short}}</a>
|
||||
<ul>
|
||||
{% for personexpedition in person.personexpedition_set.all %}
|
||||
<li>
|
||||
<table><tr><td>
|
||||
{{personexpedition.expedition}}
|
||||
</td><td>
|
||||
<div>
|
||||
<ul>
|
||||
{% for personexpedition in person.personexpedition_set.all %}
|
||||
<li>
|
||||
<table><tr><td>
|
||||
{{personexpedition.expedition}}
|
||||
</td><td>
|
||||
<div>
|
||||
<ul>
|
||||
{% for persontrip in personexpedition.persontrip_set.all %}
|
||||
<li><a href="/logbookentry/{{persontrip.logbookentry.id}}">{{persontrip.date}}</a>
|
||||
({{persontrip.logbookentry.place|wiki_to_html_short}}) -
|
||||
{{persontrip.logbookentry.title|wiki_to_html_short}}</li>
|
||||
{% endfor %}
|
||||
<ul>
|
||||
</div>
|
||||
</td></tr></table>
|
||||
</li>
|
||||
{% for persontrip in personexpedition.persontrip_set.all %}
|
||||
<li><a href="{% url logbookentry persontrip.logbook_entry.id %}">{{persontrip.date}}</a> {{persontrip.logbookentry}}
|
||||
({{persontrip.logbookentry.place|wiki_to_html_short}}) -
|
||||
{{persontrip.logbookentry.title|wiki_to_html_short}}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
<ul>
|
||||
</div>
|
||||
</td></tr></table>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
@ -4,23 +4,29 @@
|
||||
{% block title %}Person Index{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
|
||||
{% for person in persons %}
|
||||
<div class="personblock"><a href="/person/{{person.id}}">{{person|wiki_to_html_short}}</a>
|
||||
<ul>
|
||||
{% for personexpedition in person.personexpedition_set.all %}
|
||||
<li>
|
||||
<table><tr><td>
|
||||
{{personexpedition.expedition}}
|
||||
</td><td>
|
||||
<div>
|
||||
{% for persontrip in personexpedition.persontrip_set.all %}
|
||||
<a href="/logbookentry/{{persontrip.logbookentry.id}}">{{persontrip.date}}</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td></tr></table>
|
||||
</li>
|
||||
<div class="personblock"><a href="{% url person person.href%}">{{person|wiki_to_html_short}}</a>
|
||||
<ul>
|
||||
{% for personexpedition in person.personexpedition_set.all %}
|
||||
<li>
|
||||
|
||||
<table><tr><td>
|
||||
{{personexpedition.expedition}}
|
||||
</td><td>
|
||||
|
||||
<div>
|
||||
{% for persontrip in personexpedition.persontrip_set.all %}
|
||||
<a href="{% url logbookentry persontrip.logbook_entry.id %}">{{persontrip.date}}</a>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
</td></tr></table>
|
||||
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
{% endblock %}
|
||||
|
||||
6
urls.py
6
urls.py
@ -21,11 +21,13 @@ urlpatterns = patterns('',
|
||||
(r'^survex/(?P<survex_file>.*)\.err$', err),
|
||||
|
||||
url(r'^personindex$', personindex, name="personindex"),
|
||||
(r'^person/(?P<person_id>\d*)(?P<first_name>[a-zA-Z]*)[-_/\.\s(\%20)]*(?P<last_name>[a-zA-Z]*)/?$', person),
|
||||
url(r'^person/(.+)$', person, name="person"),
|
||||
|
||||
(r'^logbookentry/(.*)/?$', logbookentry),
|
||||
url(r'^logbookentry/(\d+)$', logbookentry, name="logbookentry"),
|
||||
url(r'^logbooksearch/(.*)/?$', logbookSearch),
|
||||
|
||||
url(r'^expedition/(\d+)$', expedition, name="expedition"),
|
||||
|
||||
url(r'^statistics/?$', stats, name="stats"),
|
||||
|
||||
(r'^survey/?$', surveyindex),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user