mirror of
https://expo.survex.com/repositories/expoweb/.git/
synced 2025-12-10 07:38:27 +00:00
[svn r8034] Initial troggle checkin
This is a development site using Django 1.0
This commit is contained in:
julian
197
troggle/parsers/logbooks.py
Normal file
197
troggle/parsers/logbooks.py
Normal file
@@ -0,0 +1,197 @@
|
||||
#.-*- coding: utf-8 -*-
|
||||
|
||||
import settings
|
||||
import expo.models as models
|
||||
import csv
|
||||
import sqlite3
|
||||
import re
|
||||
import os
|
||||
import datetime
|
||||
|
||||
# Dave Johnson (Stonker) is hacked -- are there two of this DJ name
|
||||
# Dave Collins (Scout) is hacked
|
||||
# Letty ten Harkel has middle , tu = timeug or ""name removed
|
||||
# the <span lang=""sv""> have been removed
|
||||
# Dave Milne (Lummat)
|
||||
# Ben van Millingen
|
||||
# Rebecca Lawson (Becka)
|
||||
|
||||
persontab = open(os.path.join(settings.EXPOWEB, "noinfo", "folk.csv"))
|
||||
personreader = csv.reader(persontab)
|
||||
headers = personreader.next()
|
||||
header = dict(zip(headers, range(len(headers))))
|
||||
|
||||
|
||||
def LoadExpos():
|
||||
models.Expedition.objects.all().delete()
|
||||
y = models.Expedition(year = "2008", name = "CUCC expo2008")
|
||||
y.save()
|
||||
for year in headers[5:]:
|
||||
y = models.Expedition(year = year, name = "CUCC expo%s" % y)
|
||||
y.save()
|
||||
|
||||
def LoadPersons():
|
||||
models.Person.objects.all().delete()
|
||||
models.PersonExpedition.objects.all().delete()
|
||||
expoers2008 = """Edvin Deadman,Kathryn Hopkins,Djuke Veldhuis,Becka Lawson,Julian Todd,Natalie Uomini,Aaron Curtis,Tony Rooke,Ollie Stevens,Frank Tully,Martin Jahnke,Mark Shinwell,Jess Stirrups,Nial Peters,Serena Povia,Olly Madge,Steve Jones,Pete Harley,Eeva Makiranta,Keith Curtis""".split(",")
|
||||
expomissing = set(expoers2008)
|
||||
|
||||
for person in personreader:
|
||||
name = person[header["Name"]]
|
||||
name = re.sub("<.*?>", "", name)
|
||||
lname = name.split()
|
||||
if len(lname) >= 2:
|
||||
firstname, lastname = lname[0], lname[1]
|
||||
else:
|
||||
firstname, lastname = lname[0], ""
|
||||
print firstname, lastname
|
||||
#assert lastname == person[header[""]], person
|
||||
pObject = models.Person(first_name = firstname,
|
||||
last_name = lastname,
|
||||
is_guest = person[header["Guest"]] == "1",
|
||||
is_vfho = person[header["VfHO member"]],
|
||||
mug_shot = person[header["Mugshot"]])
|
||||
pObject.save()
|
||||
|
||||
for year, attended in zip(headers, person)[5:]:
|
||||
yo = models.Expedition.objects.filter(year = year)[0]
|
||||
if attended == "1" or attended == "-1":
|
||||
pyo = models.PersonExpedition(person = pObject, expedition = yo)
|
||||
pyo.save()
|
||||
|
||||
if name in expoers2008:
|
||||
print "2008:", name
|
||||
expomissing.discard(name)
|
||||
yo = models.Expedition.objects.filter(year = "2008")[0]
|
||||
pyo = models.PersonExpedition(person = pObject, expedition = yo)
|
||||
pyo.save()
|
||||
|
||||
|
||||
print expomissing
|
||||
for name in expomissing:
|
||||
firstname, lastname = name.split()
|
||||
pObject = models.Person(first_name = firstname,
|
||||
last_name = lastname,
|
||||
is_guest = name in ["Eeva Makiranta", "Kieth Curtis"],
|
||||
is_vfho = False,
|
||||
mug_shot = "")
|
||||
pObject.save()
|
||||
yo = models.Expedition.objects.filter(year = "2008")[0]
|
||||
pyo = models.PersonExpedition(person = pObject, expedition = yo)
|
||||
pyo.save()
|
||||
|
||||
|
||||
#
|
||||
# the logbook loading section
|
||||
#
|
||||
def GetTripPersons(trippeople, expedition):
|
||||
res = [ ]
|
||||
author = None
|
||||
for tripperson in re.split(",|\+|&| and ", trippeople):
|
||||
tripperson = tripperson.strip()
|
||||
mul = re.match("<u>(.*?)</u>$", tripperson)
|
||||
if mul:
|
||||
tripperson = mul.group(1)
|
||||
if tripperson and tripperson[0] != '*':
|
||||
#assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap)
|
||||
personyear = expedition.GetPersonExpedition(tripperson)
|
||||
print personyear
|
||||
res.append(personyear)
|
||||
if mul:
|
||||
author = personyear
|
||||
if not author:
|
||||
author = res[-1]
|
||||
return res, author
|
||||
|
||||
def Parselogwikitxt(year, personyearmap, txt):
|
||||
trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt)
|
||||
for triphead, triptext in trippara:
|
||||
tripheadp = triphead.split("|")
|
||||
assert len(tripheadp) == 3, tripheadp
|
||||
tripdate, tripplace, trippeople = tripheadp
|
||||
tripsplace = tripplace.split(" - ")
|
||||
tripcave = tripsplace[0]
|
||||
|
||||
tul = re.findall("T/?U:?\s*(\d+(?:\.\d*)?|unknown)\s*(hrs|hours)?", triptext)
|
||||
if tul:
|
||||
#assert len(tul) <= 1, (triphead, triptext)
|
||||
#assert tul[0][1] in ["hrs", "hours"], (triphead, triptext)
|
||||
triptime = tul[0][0]
|
||||
else:
|
||||
triptime = ""
|
||||
#assert tripcave == "Journey", (triphead, triptext)
|
||||
|
||||
assert re.match("\d\d\d\d-\d\d-\d\d", tripdate), tripdate
|
||||
ldate = datetime.date(int(tripdate[:4]), int(tripdate[5:7]), int(tripdate[8:10]))
|
||||
lbo = models.LogbookEntry(date = ldate, cave = tripcave, title = tripsplace[-1], text = triptext, tu = triptime)
|
||||
lbo.save()
|
||||
|
||||
trippersons, author = GetTripPersons(trippeople, personyearmap)
|
||||
for tripperson in trippersons:
|
||||
lbo.cavers.add(tripperson)
|
||||
# add the author
|
||||
|
||||
def Parseloghtmltxt(year, expedition, txt):
|
||||
tripparas = re.findall("<hr\s*/>([\s\S]*?)(?=<hr)", txt)
|
||||
for trippara in tripparas:
|
||||
s = re.match('''(?x)\s*(?:<a\s+id="(.*?)"\s*/>)?
|
||||
\s*<div\s+class="tripdate"\s*(?:id="(.*?)")?>(.*?)</div>
|
||||
\s*<div\s+class="trippeople">(.*?)</div>
|
||||
\s*<div\s+class="triptitle">(.*?)</div>
|
||||
([\s\S]*?)
|
||||
\s*(?:<div\s+class="timeug">(.*?)</div>)?
|
||||
\s*$
|
||||
''', trippara)
|
||||
assert s, trippara
|
||||
|
||||
tripid, tripid1, tripdate, trippeople, triptitle, triptext, timeug = s.groups()
|
||||
mdatestandard = re.match("(\d\d\d\d)-(\d\d)-(\d\d)", tripdate)
|
||||
mdategoof = re.match("(\d\d?)/(\d)/(\d\d)", tripdate)
|
||||
if mdatestandard:
|
||||
year, month, day = int(mdatestandard.group(1)), int(mdatestandard.group(2)), int(mdatestandard.group(3))
|
||||
elif mdategoof:
|
||||
day, month, year = int(mdategoof.group(1)), int(mdategoof.group(2)), int(mdategoof.group(3)) + 2000
|
||||
else:
|
||||
assert False, tripdate
|
||||
ldate = datetime.date(year, month, day)
|
||||
#assert tripid[:-1] == "t" + tripdate, (tripid, tripdate)
|
||||
trippersons, author = GetTripPersons(trippeople, expedition)
|
||||
tripcave = ""
|
||||
lbo = models.LogbookEntry(date = ldate, place = tripcave, title = triptitle, text = triptext, author=author)
|
||||
lbo.save()
|
||||
tu = timeug or ""
|
||||
|
||||
for tripperson in trippersons:
|
||||
pto = models.PersonTrip(personexpedition = tripperson, place=tripcave, date=ldate, timeunderground=tu, logbookentry=lbo)
|
||||
pto.save()
|
||||
|
||||
|
||||
|
||||
def LoadLogbooks():
|
||||
models.LogbookEntry.objects.all().delete()
|
||||
expowebbase = os.path.join(settings.EXPOWEB, "years") # this could be a url
|
||||
yearlinks = [
|
||||
# ("2008", "2008/logbook/2008logbook.txt"),
|
||||
# ("2007", "2007/logbook/2007logbook.txt"),
|
||||
# ("2005", "2005/logbook.html"),
|
||||
("2004", "2004/logbook.html"),
|
||||
# ("2003", "2003/logbook.html"),
|
||||
]
|
||||
|
||||
for year, lloc in yearlinks:
|
||||
expedition = models.Expedition.objects.filter(year = year)[0]
|
||||
fin = open(os.path.join(expowebbase, lloc))
|
||||
txt = fin.read()
|
||||
fin.close()
|
||||
#print personyearmap
|
||||
if year >= "2007":
|
||||
Parselogwikitxt(year, personyearmap, txt)
|
||||
else:
|
||||
Parseloghtmltxt(year, expedition, txt)
|
||||
|
||||
# command line run through the loading stages
|
||||
LoadExpos()
|
||||
LoadPersons()
|
||||
LoadLogbooks()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user