diff --git a/databaseReset.py b/databaseReset.py index 68f2e6a..9b23199 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -24,4 +24,5 @@ import parsers.logbooks parsers.logbooks.LoadLogbooks() import parsers.QMs import parsers.survex +parsers.survex.LoadAllSurvexBlocks() import parsers.surveys \ No newline at end of file diff --git a/expo/models.py b/expo/models.py index 6d38b4f..2e8dc88 100644 --- a/expo/models.py +++ b/expo/models.py @@ -7,6 +7,7 @@ from django.core.files.storage import FileSystemStorage import os import troggle.settings as settings import datetime + from models_survex import * class Expedition(models.Model): @@ -27,6 +28,7 @@ class Expedition(models.Model): date+=datetime.timedelta(days=1) return res + # deprecated def GetPersonExpedition(self, name): person_expeditions = PersonExpedition.objects.filter(expedition=self) res = None @@ -38,6 +40,8 @@ class Expedition(models.Model): res = person_expedition return res + + class Person(models.Model): first_name = models.CharField(max_length=100) @@ -64,6 +68,7 @@ class PersonExpedition(models.Model): is_guest = models.BooleanField(default=False) nickname = models.CharField(max_length=100,blank=True,null=True) + # deprecated def GetPossibleNameForms(self): res = [ ] if self.person.last_name: diff --git a/expo/models_survex.py b/expo/models_survex.py index 80e5f5b..4b65357 100644 --- a/expo/models_survex.py +++ b/expo/models_survex.py @@ -6,6 +6,8 @@ class SurvexBlock(models.Model): name = models.CharField(max_length=100, blank=True, null=True) parent = models.ForeignKey('SurvexBlock', blank=True, null=True) text = models.TextField() + + # non-useful representation of incomplete data start_year = models.IntegerField(blank=True, null=True) start_month = models.IntegerField(blank=True, null=True) start_day = models.IntegerField(blank=True, null=True) @@ -13,8 +15,13 @@ class SurvexBlock(models.Model): end_month = models.IntegerField(blank=True, null=True) end_day = models.IntegerField(blank=True, null=True) - person = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True) + date = models.DateField(blank=True, null=True) + survexpath = models.CharField(max_length=100) + + # superfluous + person = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True) + # code for where in the survex data files this block sits begin_file = models.CharField(max_length=200) begin_char = models.IntegerField() end_file = models.CharField(max_length=200, blank=True, null=True) @@ -25,8 +32,8 @@ class SurvexBlock(models.Model): def filecontents(self): f = os.path.join(settings.SURVEX_DATA, self.begin_file) - fin = open(f) - res = fin.read() + fin = open(f, "rb") + res = fin.read().decode("latin1") fin.close() return res @@ -43,4 +50,5 @@ class PersonRole(models.Model): class Role(models.Model): name = models.CharField(max_length=50) def __unicode__(self): - return unicode(self.name) \ No newline at end of file + return unicode(self.name) + diff --git a/expo/templatetags/survex_markup.py b/expo/templatetags/survex_markup.py index c9992a9..464a04b 100644 --- a/expo/templatetags/survex_markup.py +++ b/expo/templatetags/survex_markup.py @@ -6,6 +6,7 @@ import re register = template.Library() +# seems to add extra lines between the commented lines, which isn't so great. regexes = [] regexes.append((re.compile(r"(;.*)$", re.IGNORECASE|re.MULTILINE), r'\1\n')) diff --git a/expo/views_caves.py b/expo/views_caves.py index 045dbcf..4e691e5 100644 --- a/expo/views_caves.py +++ b/expo/views_caves.py @@ -1,5 +1,6 @@ from django.shortcuts import render_to_response from troggle.expo.models import Cave, CaveAndEntrance, Survey, Expedition +import troggle.expo.models as models import troggle.settings as settings from troggle.expo.forms import CaveForm import search @@ -20,6 +21,11 @@ def ent(request, cave_id, ent_letter): 'entrance': cave_and_ent.entrance, 'letter': cave_and_ent.entrance_letter, 'settings': settings}) +def survexblock(request, survexpath): + survexblock = models.SurvexBlock.objects.get(survexpath=survexpath) + #ftext = survexblock.filecontents() + ftext = survexblock.text + return render_to_response('survexblock.html', {'survexblock':survexblock, 'ftext':ftext, 'settings':settings }) def caveSearch(request): query_string = '' @@ -54,4 +60,5 @@ def survey(request,year,wallet_number): dictToPass=locals() dictToPass.update({'settings':settings}) - return render_to_response('survey.html',dictToPass) \ No newline at end of file + return render_to_response('survey.html',dictToPass) + diff --git a/expo/views_logbooks.py b/expo/views_logbooks.py index 935eebd..7d18ba1 100644 --- a/expo/views_logbooks.py +++ b/expo/views_logbooks.py @@ -1,7 +1,10 @@ from django.shortcuts import render_to_response from troggle.expo.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry import troggle.settings as settings + from troggle.parsers.logbooks import LoadLogbookForExpedition +from troggle.parsers.people import GetPersonExpeditionNameLookup + import search import re @@ -17,7 +20,7 @@ def expedition(request, expeditionname): message = "No message" if "reload" in request.GET: message = LoadLogbookForExpedition(expedition) - + #message = str(GetPersonExpeditionNameLookup(expedition).keys()) logbookentries = expedition.logbookentry_set.order_by('date') return render_to_response('expedition.html', {'expedition': expedition, 'expedition_next':expedition_next, 'expedition_prev':expedition_prev, 'logbookentries':logbookentries, 'message':message, 'settings': settings}) diff --git a/expo/views_other.py b/expo/views_other.py index 43cdbca..7d5d655 100644 --- a/expo/views_other.py +++ b/expo/views_other.py @@ -5,6 +5,7 @@ from django import forms from django.db.models import Q from troggle.parsers.people import LoadPersonsExpos import re +from troggle.parsers.survex import LoadAllSurvexBlocks import randSent from django.core.urlresolvers import reverse @@ -19,9 +20,13 @@ def stats(request): def frontPage(request): message = "no test message" #reverse('personn', kwargs={"name":"hkjhjh"}) - if "reload" in request.GET: + if "reloadexpos" in request.GET: message = LoadPersonsExpos() message = "Reloaded personexpos" + if "reloadsurvex" in request.GET: + message = LoadAllSurvexBlocks() + message = "Reloaded survexblocks" + #'randSent':randSent.randomLogbookSentence(), expeditions = Expedition.objects.all() return render_to_response('index.html', {'expeditions':expeditions, 'settings':settings, "message":message}) diff --git a/expo/views_survex.py b/expo/views_survex.py index c356363..067d4e3 100644 --- a/expo/views_survex.py +++ b/expo/views_survex.py @@ -41,4 +41,4 @@ def process(survex_file): cwd = os.getcwd() os.chdir(os.path.split(settings.SURVEX_DATA + survex_file)[0]) os.system(settings.CAVERN + " --log " +settings.SURVEX_DATA + survex_file + ".svx") - os.chdir(cwd) \ No newline at end of file + os.chdir(cwd) diff --git a/localsettingsserver.py b/localsettingsserver.py index 86657a3..4148336 100644 --- a/localsettingsserver.py +++ b/localsettingsserver.py @@ -11,6 +11,7 @@ EXPOWEB = '/home/mjg/expoweb/' SURVEYS = '/home/mjg/surveys/' SURVEYS_URL = 'http://framos.lawoftheland.co.uk/troggle/survey_scans/' +FILES = "http://framos.lawoftheland.co.uk/troggle/survey_files/" SVX_URL = 'http://framos.lawoftheland.co.uk/troggle/survex/' diff --git a/media/css/main2.css b/media/css/main2.css index d1b8139..710c5a7 100644 --- a/media/css/main2.css +++ b/media/css/main2.css @@ -125,6 +125,15 @@ ul#expeditionlist width: 300px } +div.survexblock +{ + width:50%; + background-color:#e0e0e0; +} +p.indent +{ + margin-left:10px; +} #expoHeader { width:100%; diff --git a/parsers/logbooks.py b/parsers/logbooks.py index a2cf58a..3de28d7 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -2,6 +2,9 @@ import troggle.settings as settings import troggle.expo.models as models + +from troggle.parsers.people import GetPersonExpeditionNameLookup + import csv import re import datetime @@ -21,7 +24,7 @@ def GetTripPersons(trippeople, expedition, logtime_underground): tripperson = mul.group(1).strip() if tripperson and tripperson[0] != '*': #assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap) - personyear = expedition.GetPersonExpedition(tripperson) + personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower()) if not personyear: print "NoMatchFor: '%s'" % tripperson res.append((personyear, logtime_underground)) diff --git a/parsers/people.py b/parsers/people.py index c59d3d8..93d8a0e 100644 --- a/parsers/people.py +++ b/parsers/people.py @@ -120,3 +120,40 @@ def LoadPersonsExpos(): pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname="", is_guest=is_guest) pyo.save() +# expedition name lookup cached for speed (it's a very big list) +Gpersonexpeditionnamelookup = { } +def GetPersonExpeditionNameLookup(expedition): + global Gpersonexpeditionnamelookup + res = Gpersonexpeditionnamelookup.get(expedition.name) + if res: + return res + + res = {} + duplicates = set() + + personexpeditions = models.PersonExpedition.objects.filter(expedition=expedition) + for personexpedition in personexpeditions: + possnames = [ ] + f = personexpedition.person.first_name.lower() + l = personexpedition.person.last_name.lower() + if l: + possnames.append(f + " " + l) + possnames.append(f + " " + l[0]) + possnames.append(f + l[0]) + possnames.append(f[0] + " " + l) + possnames.append(f) + if personexpedition.nickname: + possnames.append(personexpedition.nickname.lower()) + + for possname in possnames: + if possname in res: + duplicates.add(possname) + else: + res[possname] = personexpedition + + for possname in duplicates: + del res[possname] + + Gpersonexpeditionnamelookup[expedition.name] = res + return res + diff --git a/parsers/survex.py b/parsers/survex.py index 1f688c3..c8805a1 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1,5 +1,8 @@ import settings import expo.models as models + +from troggle.parsers.people import GetPersonExpeditionNameLookup + import re import os @@ -51,17 +54,20 @@ def fileIterator(directory, filename): yield survex_file, char, line char = char + len(line) + def make_model(name, parent, iter_lines, sf, c, l): + m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l) + m.survexpath = m.name if parent: - m = models.SurvexBlock(name = name, parent = parent, begin_file = sf, begin_char = c, text = l) - else: - m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l) + m.parent = parent + m.survexpath = m.parent.survexpath + "." + m.name m.save() + # horrible local function def saveEnd(survex_file, count): if m.start_year and team: try: - exp = models.Expedition.objects.get(year = str(file_year[1])) + exp = models.Expedition.objects.get(year = str(m.start_year)) for file_, (role, names) in team: if names.strip("\t").strip(" ") == "both" or names.strip("\t").strip(" ") == "Both": names = reduce(lambda x, y: x + u" & " + y, @@ -69,21 +75,31 @@ def make_model(name, parent, iter_lines, sf, c, l): if names.strip("\t").strip(" ") != "both" and names.strip("\t").strip(" ") != "Both"]) for name in re.split("&|/|\+|,|;", names): + sname = name.strip(". ").lower() try: - models.PersonRole(personexpedition = exp.GetPersonExpedition(name.strip(" ")), - person = exp.GetPersonExpedition(name.strip(" ")).person, + personexpedition = GetPersonExpeditionNameLookup(exp).get(sname) + if personexpedition: + models.PersonRole(personexpedition = personexpedition, + person = personexpedition.person, survex_block = m, role = models.Role.objects.get(name = roles[role])).save() + else: + print "no person", exp, sname, role except AttributeError: - print ("Person not found: " + name + " in " + file_).encode('ascii', 'xmlcharrefreplace') + print ("Person not found: " + name + " in " + file_ + " " + role).encode('ascii', 'xmlcharrefreplace') except AssertionError, inst: print (unicode(inst) + ": " + unicode(file_year[0])).encode('ascii', 'xmlcharrefreplace') - except models.Expedition.DoesNotExist: - print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace') + #except models.Expedition.DoesNotExist: + # print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace') m.end_file = survex_file m.end_char = count + + if m.start_day: + m.date = "%04d-%02d-%02d" % (int(m.start_year), int(m.start_month), int(m.start_day)) + m.save() + team = [] file_year = None for survex_file, count, line in iter_lines: @@ -120,8 +136,17 @@ def make_model(name, parent, iter_lines, sf, c, l): saveEnd(survex_file, count) -for role in ["Insts", "Notes", "Pics", "Tape", "Other"]: - models.Role(name = role).save() +#def LoadSurvexBlocks(): +# survex_file = os.path.join(directory, filename + ".svx") +# f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb") + + +def LoadAllSurvexBlocks(): + models.Role.objects.all().delete() + models.SurvexBlock.objects.all().delete() + for role in ["Insts", "Notes", "Pics", "Tape", "Other"]: + models.Role(name = role).save() + filename = "all" + make_model("all", None, fileIterator("", filename), filename, 0, "") + -filename = "all" -make_model("", None, fileIterator("", filename), filename, 0, "") diff --git a/templates/base.html b/templates/base.html index d189ae6..bca30f0 100644 --- a/templates/base.html +++ b/templates/base.html @@ -9,7 +9,9 @@ {% block javascript %}{% endblock %} + +
@@ -26,6 +28,7 @@ {% endblock %}
+ diff --git a/templates/expedition.html b/templates/expedition.html index bcbed62..a7add92 100644 --- a/templates/expedition.html +++ b/templates/expedition.html @@ -18,7 +18,7 @@ CaverFromTo {% for personexpedition in expedition.personexpedition_set.all %} - {{personexpedition.person}} + {{personexpedition.person}} {{personexpedition.date_from}} {{personexpedition.date_to}} @@ -37,7 +37,7 @@ {{logbookentry.date}} {{logbookentry.title|safe}} - {{logbookentry.author.name}} + {{logbookentry.author.name}} {{logbookentry.place}} {% endfor %} diff --git a/templates/index.html b/templates/index.html index 667fa28..0122bdd 100644 --- a/templates/index.html +++ b/templates/index.html @@ -6,7 +6,12 @@ {% block content %}

The unfinished front page

-

Some handy links into the less incomplete parts of this webpage

+Work down through an expedition page link +

Remaining work: continue to build up the network; tables of trips per year per person; parse 1996 logbook; +continue to correct the name matching and spelling; detect T/U on log entries; match caves to log entries; see the cave list; +simplify the survex parsing code (if necessary); vast front-page layout table of folks and caving trips and years; +links between logbooks and survex blocks to cave things; where are the subcaves; mini-tree of survexblocks; connect sketches +to caves to survey blocks and render thumbnailwise; all images to start appearing in pages; and so on

{{message}}

@@ -15,7 +20,10 @@
  • Statistics of what's loaded in the database
    • -
    +
    + + +