expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-12-14 20:17:06 +00:00
Code Issues 18 Projects 3 Activity

[svn] parsing of 2007 logbook. still problems

Browse Source 
Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8039 by julian @ 10/27/2008 2:03 AM
This commit is contained in:
substantialnoninfringinguser
2009-05-13 05:14:41 +01:00
parent 18f6731b5b
commit 0e8c7af706
2 changed files with 26 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
parsers/cavetab.py
View File

@@ -4,7 +4,11 @@ import settings
import expo.models as models import expo.models as models
import csv import csv
import time import time
<<<<<<< .mine
#import sqlite3
=======
# import sqlite3 # import sqlite3
>>>>>>> .r8038
import re import re
import os import os
@@ -61,8 +65,13 @@ caveReader = csv.reader(cavetab)
caveReader.next() # Strip out column headers caveReader.next() # Strip out column headers
<<<<<<< .mine
def save(x): #There seems to be an intermitent problem with sqlite and Vista, uncomment the lines below and the "import sqlite3" above to fix it
# try:
=======
def save(x): #There seems to be an intermitent problem with sqlite and Vista, uncomment the lines below, and the import sqlite3 statment above to fix it def save(x): #There seems to be an intermitent problem with sqlite and Vista, uncomment the lines below, and the import sqlite3 statment above to fix it
# try: # try:
>>>>>>> .r8038
x.save() x.save()
# except sqlite3.OperationalError: # except sqlite3.OperationalError:
# print "Error" # print "Error"
@@ -269,4 +278,4 @@ for line in caveReader :
entrance_letter = '' entrance_letter = ''
newCaveAndEntrance = models.CaveAndEntrance(cave = newCave, entrance = newEntrance, entrance_letter = entrance_letter) newCaveAndEntrance = models.CaveAndEntrance(cave = newCave, entrance = newEntrance, entrance_letter = entrance_letter)
save(newCaveAndEntrance) save(newCaveAndEntrance)

26
parsers/logbooks.py
View File

@@ -98,7 +98,7 @@ def GetTripPersons(trippeople, expedition):
tripperson = tripperson.strip() tripperson = tripperson.strip()
mul = re.match("<u>(.*?)</u>$", tripperson) mul = re.match("<u>(.*?)</u>$", tripperson)
if mul: if mul:
tripperson = mul.group(1) tripperson = mul.group(1).strip()
if tripperson and tripperson[0] != '*': if tripperson and tripperson[0] != '*':
#assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap) #assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap)
personyear = expedition.GetPersonExpedition(tripperson) personyear = expedition.GetPersonExpedition(tripperson)
@@ -110,7 +110,7 @@ def GetTripPersons(trippeople, expedition):
author = res[-1] author = res[-1]
return res, author return res, author
def Parselogwikitxt(year, personyearmap, txt): def Parselogwikitxt(year, expedition, txt):
trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt) trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt)
for triphead, triptext in trippara: for triphead, triptext in trippara:
tripheadp = triphead.split("|") tripheadp = triphead.split("|")
@@ -130,13 +130,18 @@ def Parselogwikitxt(year, personyearmap, txt):
assert re.match("\d\d\d\d-\d\d-\d\d", tripdate), tripdate assert re.match("\d\d\d\d-\d\d-\d\d", tripdate), tripdate
ldate = datetime.date(int(tripdate[:4]), int(tripdate[5:7]), int(tripdate[8:10])) ldate = datetime.date(int(tripdate[:4]), int(tripdate[5:7]), int(tripdate[8:10]))
lbo = models.LogbookEntry(date = ldate, cave = tripcave, title = tripsplace[-1], text = triptext, tu = triptime) print "ppp", trippeople, len(triptext)
trippersons, author = GetTripPersons(trippeople, expedition)
triptext = triptext[:10] # seems to have aproblem with this
print "ttt", triptext
lbo = models.LogbookEntry(date = ldate, place = tripcave, title = tripsplace[-1], text = triptext, author=author)
lbo.save() lbo.save()
trippersons, author = GetTripPersons(trippeople, personyearmap) print "ppp", trippersons
for tripperson in trippersons: for tripperson in trippersons:
lbo.cavers.add(tripperson) pto = models.PersonTrip(personexpedition = tripperson, place=tripcave, date=ldate, timeunderground=triptime,
# add the author logbookentry=lbo, is_logbookentryauthor=(tripperson == author))
pto.save()
def Parseloghtmltxt(year, expedition, txt): def Parseloghtmltxt(year, expedition, txt):
tripparas = re.findall("<hr\s*/>([\s\S]*?)(?=<hr)", txt) tripparas = re.findall("<hr\s*/>([\s\S]*?)(?=<hr)", txt)
@@ -169,6 +174,7 @@ def Parseloghtmltxt(year, expedition, txt):
lbo.save() lbo.save()
tu = timeug or "" tu = timeug or ""
print "ppp", trippeople, trippersons
for tripperson in trippersons: for tripperson in trippersons:
pto = models.PersonTrip(personexpedition = tripperson, place=tripcave, date=ldate, timeunderground=tu, pto = models.PersonTrip(personexpedition = tripperson, place=tripcave, date=ldate, timeunderground=tu,
logbookentry=lbo, is_logbookentryauthor=(tripperson == author)) logbookentry=lbo, is_logbookentryauthor=(tripperson == author))
@@ -181,10 +187,10 @@ def LoadLogbooks():
expowebbase = os.path.join(settings.EXPOWEB, "years") expowebbase = os.path.join(settings.EXPOWEB, "years")
yearlinks = [ yearlinks = [
# ("2008", "2008/logbook/2008logbook.txt"), # ("2008", "2008/logbook/2008logbook.txt"),
# ("2007", "2007/logbook/2007logbook.txt"), ("2007", "2007/logbook/2007logbook.txt"),
# ("2005", "2005/logbook.html"), ("2005", "2005/logbook.html"),
("2004", "2004/logbook.html"), ("2004", "2004/logbook.html"),
# ("2003", "2003/logbook.html"), #not done ("2003", "2003/logbook.html"),
] ]
for year, lloc in yearlinks: for year, lloc in yearlinks:
@@ -193,7 +199,7 @@ def LoadLogbooks():
txt = fin.read() txt = fin.read()
fin.close() fin.close()
if year >= "2007": if year >= "2007":
Parselogwikitxt(year, personyearmap, txt) Parselogwikitxt(year, expedition, txt)
else: else:
Parseloghtmltxt(year, expedition, txt) Parseloghtmltxt(year, expedition, txt)