expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-29 05:11:52 +00:00
Code Issues 19 Projects 3 Activity

[svn] long putting in

Browse Source 
Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8065 by julian @ 11/7/2008 2:08 AM
This commit is contained in:
substantialnoninfringinguser 2009-05-13 05:19:07 +01:00
parent b33061a3ce
commit a8ad3a3b1e
3 changed files with 39 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
databaseReset.py
View File

@ -18,5 +18,5 @@ user.is_superuser = True
user.save() user.save()
import parsers.cavetab import parsers.cavetab
import parsers.survex #import parsers.survex
#import parsers.logbooks import parsers.logbooks

3
expo/models_logbooks.py
View File

@ -16,6 +16,7 @@ class Expedition(models.Model):
res = None res = None
for personexpedition in personexpeditions: for personexpedition in personexpeditions:
for possiblenameform in personexpedition.GetPossibleNameForms(): for possiblenameform in personexpedition.GetPossibleNameForms():
#print "nnn", possiblenameform
if name == possiblenameform: if name == possiblenameform:
assert not res, "Ambiguous: " + name assert not res, "Ambiguous: " + name
res = personexpedition res = personexpedition
@ -55,7 +56,7 @@ class PersonExpedition(models.Model):
class LogbookEntry(models.Model): class LogbookEntry(models.Model):
date = models.DateField() date = models.DateField()
author = models.ForeignKey(PersonExpedition,blank=True,null=True) # the person who writes it up doesn't have to have been on the trip author = models.ForeignKey(PersonExpedition,blank=True,null=True) # the person who writes it up doesn't have to have been on the trip
title = models.CharField(max_length=100) title = models.CharField(max_length=200)
# this will be a foreign key of the place the logbook is describing # this will be a foreign key of the place the logbook is describing
place = models.CharField(max_length=100,blank=True,null=True) place = models.CharField(max_length=100,blank=True,null=True)

89
parsers/logbooks.py
View File

@ -30,36 +30,16 @@ def LoadPersons():
for person in personreader: for person in personreader:
name = person[header["Name"]] name = person[header["Name"]]
name = re.sub("<.*?>", "", name) name = re.sub("<.*?>", "", name)
lname = name.split() mname = re.match("(\w+)(?:\s((?:van |ten )?\w+))?(?:\s\(([^)]*)\))?", name)
mbrack = re.match("\((.*?)\)", lname[-1])
if mbrack: if mname.group(3):
nickname = mbrack.group(1) nickname = mname.group(3)
del lname[-1]
elif name == "Anthony Day":
nickname = "Dour"
elif name == "Duncan Collis":
nickname = "Dunks"
elif name == "Mike Richardson":
nickname = "Mike TA"
elif name == "Hilary Greaves":
nickname = "Hils"
elif name == "Andrew Atkinson":
nickname = "Andy A"
elif name == "Wookey":
nickname = "Wook"
else: else:
nickname = "" nickname = ""
if len(lname) == 3: # van something firstname, lastname = mname.group(1), mname.group(2) or ""
firstname, lastname = lname[0], "%s %s" % (lname[1], lname[2])
elif len(lname) == 2: #print firstname, lastname, "NNN", nickname
firstname, lastname = lname[0], lname[1]
elif len(lname) == 1:
firstname, lastname = lname[0], ""
else:
assert False, lname
#print firstname, lastname
#assert lastname == person[header[""]], person #assert lastname == person[header[""]], person
pObject = models.Person(first_name = firstname, pObject = models.Person(first_name = firstname,
@ -76,7 +56,7 @@ def LoadPersons():
pyo.save() pyo.save()
# error # error
elif name == "Mike Richardson" and year == "2001": elif (firstname, lastname) == ("Mike", "Richardson") and year == "2001":
print "Mike Richardson(2001) error" print "Mike Richardson(2001) error"
pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname=nickname, is_guest=is_guest) pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname=nickname, is_guest=is_guest)
pyo.save() pyo.save()
@ -129,7 +109,7 @@ def GetTripPersons(trippeople, expedition):
def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, tu): def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, tu):
trippersons, author = GetTripPersons(trippeople, expedition) trippersons, author = GetTripPersons(trippeople, expedition)
lbo = models.LogbookEntry(date=date, place=place, title=title, text=text, author=author) lbo = models.LogbookEntry(date=date, place=place, title=title[:50], text=text, author=author)
lbo.save() lbo.save()
print "ttt", date, place print "ttt", date, place
for tripperson in trippersons: for tripperson in trippersons:
@ -137,7 +117,21 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, tu):
logbookentry=lbo, is_logbookentryauthor=(tripperson == author)) logbookentry=lbo, is_logbookentryauthor=(tripperson == author))
pto.save() pto.save()
# 2007, 2008 def ParseDate(tripdate, year):
mdatestandard = re.match("(\d\d\d\d)-(\d\d)-(\d\d)", tripdate)
mdategoof = re.match("(\d\d?)/0?(\d)/(20|19)?(\d\d)", tripdate)
if mdatestandard:
assert mdatestandard.group(1) == year, (tripdate, year)
year, month, day = int(mdatestandard.group(1)), int(mdatestandard.group(2)), int(mdatestandard.group(3))
elif mdategoof:
assert not mdategoof.group(3) or mdategoof.group(3) == year[:2]
yadd = int(year[:2]) * 100
day, month, year = int(mdategoof.group(1)), int(mdategoof.group(2)), int(mdategoof.group(4)) + yadd
else:
assert False, tripdate
return datetime.date(year, month, day)
# 2007, 2008, 2006
def Parselogwikitxt(year, expedition, txt): def Parselogwikitxt(year, expedition, txt):
trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt) trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt)
for triphead, triptext in trippara: for triphead, triptext in trippara:
@ -145,7 +139,7 @@ def Parselogwikitxt(year, expedition, txt):
assert len(tripheadp) == 3, (tripheadp, triptext) assert len(tripheadp) == 3, (tripheadp, triptext)
tripdate, tripplace, trippeople = tripheadp tripdate, tripplace, trippeople = tripheadp
tripsplace = tripplace.split(" - ") tripsplace = tripplace.split(" - ")
tripcave = tripsplace[0] tripcave = tripsplace[0].strip()
tul = re.findall("T/?U:?\s*(\d+(?:\.\d*)?|unknown)\s*(hrs|hours)?", triptext) tul = re.findall("T/?U:?\s*(\d+(?:\.\d*)?|unknown)\s*(hrs|hours)?", triptext)
if tul: if tul:
@ -156,8 +150,7 @@ def Parselogwikitxt(year, expedition, txt):
tu = "" tu = ""
#assert tripcave == "Journey", (triphead, triptext) #assert tripcave == "Journey", (triphead, triptext)
assert re.match("\d\d\d\d-\d\d-\d\d", tripdate), tripdate ldate = ParseDate(tripdate.strip(), year)
ldate = datetime.date(int(tripdate[:4]), int(tripdate[5:7]), int(tripdate[8:10]))
#print "\n", tripcave, "--- ppp", trippeople, len(triptext) #print "\n", tripcave, "--- ppp", trippeople, len(triptext)
EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, tu=tu) EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, tu=tu)
@ -176,15 +169,7 @@ def Parseloghtmltxt(year, expedition, txt):
assert s, trippara assert s, trippara
tripid, tripid1, tripdate, trippeople, triptitle, triptext, tu = s.groups() tripid, tripid1, tripdate, trippeople, triptitle, triptext, tu = s.groups()
mdatestandard = re.match("(\d\d\d\d)-(\d\d)-(\d\d)", tripdate) ldate = ParseDate(tripdate.strip(), year)
mdategoof = re.match("(\d\d?)/0?(\d)/(?:20)?(\d\d)", tripdate)
if mdatestandard:
year, month, day = int(mdatestandard.group(1)), int(mdatestandard.group(2)), int(mdatestandard.group(3))
elif mdategoof:
day, month, year = int(mdategoof.group(1)), int(mdategoof.group(2)), int(mdategoof.group(3)) + 2000
else:
assert False, tripdate
ldate = datetime.date(year, month, day)
#assert tripid[:-1] == "t" + tripdate, (tripid, tripdate) #assert tripid[:-1] == "t" + tripdate, (tripid, tripdate)
trippeople = re.sub("Ol(?!l)", "Olly", trippeople) trippeople = re.sub("Ol(?!l)", "Olly", trippeople)
trippeople = re.sub("Wook(?!e)", "Wookey", trippeople) trippeople = re.sub("Wook(?!e)", "Wookey", trippeople)
@ -210,13 +195,11 @@ def Parseloghtml01(year, expedition, txt):
tripid = mtripid and mtripid.group(1) or "" tripid = mtripid and mtripid.group(1) or ""
tripheader = re.sub("</?(?:[ab]|span)[^>]*>", "", tripheader) tripheader = re.sub("</?(?:[ab]|span)[^>]*>", "", tripheader)
# print [tripheader] #print [tripheader]
# continue #continue
tripdate, triptitle, trippeople = tripheader.split("|") tripdate, triptitle, trippeople = tripheader.split("|")
mdatestandard = re.match("\s*(\d\d\d\d)-(\d\d)-(\d\d)", tripdate) ldate = ParseDate(tripdate.strip(), year)
ldate = datetime.date(int(mdatestandard.group(1)), int(mdatestandard.group(2)), int(mdatestandard.group(3)))
mdatestandard = re.match("(\d\d\d\d)-(\d\d)-(\d\d)", tripdate)
mtu = re.search('<p[^>]*>(T/?U.*)', triptext) mtu = re.search('<p[^>]*>(T/?U.*)', triptext)
if mtu: if mtu:
@ -232,7 +215,7 @@ def Parseloghtml01(year, expedition, txt):
ltriptext = re.sub("\s*?\n\s*", " ", ltriptext) ltriptext = re.sub("\s*?\n\s*", " ", ltriptext)
ltriptext = re.sub("<p>", "\n\n", ltriptext).strip() ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!]", "NONASCII", ltriptext) ltriptext = re.sub("<p>", "\n\n", ltriptext).strip() ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!]", "NONASCII", ltriptext)
print ldate, trippeople.strip() #print ldate, trippeople.strip()
# could includ the tripid (url link for cross referencing) # could includ the tripid (url link for cross referencing)
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, tu=tu) EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, tu=tu)
@ -252,9 +235,8 @@ def Parseloghtml03(year, expedition, txt):
print sheader print sheader
# continue # continue
tripdate, triptitle, trippeople = sheader tripdate, triptitle, trippeople = sheader
mdategoof = re.match("(\d\d?)/(\d)/(\d\d)", tripdate) ldate = ParseDate(tripdate.strip(), year)
day, month, year = int(mdategoof.group(1)), int(mdategoof.group(2)), int(mdategoof.group(3)) + 2000 triptitles = triptitle.split(" , ")
ldate = datetime.date(year, month, day) triptitles = triptitle.split(" , ")
if len(triptitles) >= 2: if len(triptitles) >= 2:
tripcave = triptitles[0] tripcave = triptitles[0]
else: else:
@ -272,7 +254,7 @@ def LoadLogbooks():
yearlinks = [ yearlinks = [
("2008", "2008/logbook/2008logbook.txt", Parselogwikitxt), ("2008", "2008/logbook/2008logbook.txt", Parselogwikitxt),
("2007", "2007/logbook/2007logbook.txt", Parselogwikitxt), ("2007", "2007/logbook/2007logbook.txt", Parselogwikitxt),
#not done ("2006", "2006/logbook/logbook_06.txt"), ("2006", "2006/logbook/logbook_06.txt", Parselogwikitxt),
("2005", "2005/logbook.html", Parseloghtmltxt), ("2005", "2005/logbook.html", Parseloghtmltxt),
("2004", "2004/logbook.html", Parseloghtmltxt), ("2004", "2004/logbook.html", Parseloghtmltxt),
("2003", "2003/logbook.html", Parseloghtml03), ("2003", "2003/logbook.html", Parseloghtml03),
@ -280,9 +262,8 @@ def LoadLogbooks():
("2001", "2001/log.htm", Parseloghtml01), ("2001", "2001/log.htm", Parseloghtml01),
("2000", "2000/log.htm", Parseloghtml01), ("2000", "2000/log.htm", Parseloghtml01),
("1999", "1999/log.htm", Parseloghtml01), ("1999", "1999/log.htm", Parseloghtml01),
("1998", "1998/log.htm", Parseloghtml01),
# ("1998", "1998/log.htm", Parseloghtml01), ("1997", "1997/log.htm", Parseloghtml01),
# ("1997", "1997/log.htm", Parseloghtml01),
] ]
# yearlinks = [ ("1997", "1997/log.htm", Parseloghtml01), ] #overwrite # yearlinks = [ ("1997", "1997/log.htm", Parseloghtml01), ] #overwrite