[svn] survey block object

Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8199 by julian @ 1/19/2009 12:22 AM

databaseReset.py

@@ -24,4 +24,5 @@ import parsers.logbooks
 parsers.logbooks.LoadLogbooks()
 import parsers.QMs
 import parsers.survex
+parsers.survex.LoadAllSurvexBlocks()
 import parsers.surveys

expo/models.py

@@ -7,6 +7,7 @@ from django.core.files.storage import FileSystemStorage
 import os
 import troggle.settings as settings
 import datetime
+
 from models_survex import *
 
 class Expedition(models.Model):
@@ -27,6 +28,7 @@ class Expedition(models.Model):
 			date+=datetime.timedelta(days=1)
 		return res
 
+    # deprecated
     def GetPersonExpedition(self, name):
         person_expeditions = PersonExpedition.objects.filter(expedition=self)
         res = None
@@ -38,6 +40,8 @@ class Expedition(models.Model):
                     res = person_expedition
         return res
 
+    
+
 
 class Person(models.Model):
     first_name  = models.CharField(max_length=100)
@@ -64,6 +68,7 @@ class PersonExpedition(models.Model):
     is_guest    = models.BooleanField(default=False)  
     nickname    = models.CharField(max_length=100,blank=True,null=True)
     
+    # deprecated
     def GetPossibleNameForms(self):
         res = [ ]
         if self.person.last_name:

expo/models_survex.py

@@ -6,6 +6,8 @@ class SurvexBlock(models.Model):
     name = models.CharField(max_length=100, blank=True, null=True)
     parent = models.ForeignKey('SurvexBlock', blank=True, null=True)
     text = models.TextField()
+    
+    # non-useful representation of incomplete data
     start_year = models.IntegerField(blank=True, null=True)
     start_month = models.IntegerField(blank=True, null=True)
     start_day = models.IntegerField(blank=True, null=True)
@@ -13,8 +15,13 @@ class SurvexBlock(models.Model):
     end_month = models.IntegerField(blank=True, null=True)
     end_day = models.IntegerField(blank=True, null=True)
     
-    person = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True)
+    date    = models.DateField(blank=True, null=True)
+    survexpath  = models.CharField(max_length=100)
+    
+    # superfluous
+    person  = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True)
    
+    # code for where in the survex data files this block sits
     begin_file = models.CharField(max_length=200)
     begin_char = models.IntegerField()
     end_file = models.CharField(max_length=200, blank=True, null=True)
@@ -25,8 +32,8 @@ class SurvexBlock(models.Model):
     
     def filecontents(self):
         f = os.path.join(settings.SURVEX_DATA, self.begin_file)
-        fin = open(f)
+        fin = open(f, "rb")
-        res = fin.read()
+        res = fin.read().decode("latin1")
         fin.close()
         return res
         
@@ -43,4 +50,5 @@ class PersonRole(models.Model):
 class Role(models.Model):
     name = models.CharField(max_length=50)
     def __unicode__(self):
-        return unicode(self.name)
+        return unicode(self.name)
+    

expo/templatetags/survex_markup.py

@@ -6,6 +6,7 @@ import re
 
 register = template.Library()
 
+# seems to add extra lines between the commented lines, which isn't so great.
 regexes = []
 regexes.append((re.compile(r"(;.*)$", re.IGNORECASE|re.MULTILINE),
                r'<span class = "comment">\1</span>\n'))

expo/views_caves.py

@@ -1,5 +1,6 @@
 from django.shortcuts import render_to_response
 from troggle.expo.models import Cave, CaveAndEntrance, Survey, Expedition
+import troggle.expo.models as models
 import troggle.settings as settings
 from troggle.expo.forms import CaveForm
 import search
@@ -20,6 +21,11 @@ def ent(request, cave_id, ent_letter):
                                                 'entrance': cave_and_ent.entrance,
                                                 'letter': cave_and_ent.entrance_letter,
                                                 'settings': settings})
+def survexblock(request, survexpath):
+    survexblock = models.SurvexBlock.objects.get(survexpath=survexpath)
+    #ftext = survexblock.filecontents()
+    ftext = survexblock.text
+    return render_to_response('survexblock.html', {'survexblock':survexblock, 'ftext':ftext, 'settings':settings })
 
 def caveSearch(request):
     query_string = ''
@@ -54,4 +60,5 @@ def survey(request,year,wallet_number):
     dictToPass=locals()
     dictToPass.update({'settings':settings})
     
-    return render_to_response('survey.html',dictToPass)
+    return render_to_response('survey.html',dictToPass)
+    

expo/views_logbooks.py

@@ -1,7 +1,10 @@
 from django.shortcuts import render_to_response
 from troggle.expo.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry
 import troggle.settings as settings
+
 from troggle.parsers.logbooks import LoadLogbookForExpedition
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
 import search
 import re
 
@@ -17,7 +20,7 @@ def expedition(request, expeditionname):
     message = "No message"
     if "reload" in request.GET:
         message = LoadLogbookForExpedition(expedition)
-    
+    #message = str(GetPersonExpeditionNameLookup(expedition).keys())
     logbookentries = expedition.logbookentry_set.order_by('date')
     return render_to_response('expedition.html', {'expedition': expedition, 'expedition_next':expedition_next, 'expedition_prev':expedition_prev, 'logbookentries':logbookentries, 'message':message, 'settings': settings})
 

expo/views_other.py

@@ -5,6 +5,7 @@ from django import forms
 from django.db.models import Q
 from troggle.parsers.people import LoadPersonsExpos
 import re
+from troggle.parsers.survex import LoadAllSurvexBlocks
 import randSent
 
 from django.core.urlresolvers import reverse
@@ -19,9 +20,13 @@ def stats(request):
 
 def frontPage(request):
     message = "no test message"  #reverse('personn', kwargs={"name":"hkjhjh"}) 
-    if "reload" in request.GET:
+    if "reloadexpos" in request.GET:
         message = LoadPersonsExpos()
         message = "Reloaded personexpos"
+    if "reloadsurvex" in request.GET:
+        message = LoadAllSurvexBlocks()
+        message = "Reloaded survexblocks"
+
     #'randSent':randSent.randomLogbookSentence(),
     expeditions =  Expedition.objects.all()
     return render_to_response('index.html', {'expeditions':expeditions, 'settings':settings, "message":message})

expo/views_survex.py

@@ -41,4 +41,4 @@ def process(survex_file):
     cwd = os.getcwd()
     os.chdir(os.path.split(settings.SURVEX_DATA + survex_file)[0])
     os.system(settings.CAVERN + " --log " +settings.SURVEX_DATA + survex_file + ".svx")
-    os.chdir(cwd)
+    os.chdir(cwd)

localsettingsserver.py

@@ -11,6 +11,7 @@ EXPOWEB = '/home/mjg/expoweb/'
 SURVEYS = '/home/mjg/surveys/'
 
 SURVEYS_URL = 'http://framos.lawoftheland.co.uk/troggle/survey_scans/'
+FILES = "http://framos.lawoftheland.co.uk/troggle/survey_files/"
 
 SVX_URL = 'http://framos.lawoftheland.co.uk/troggle/survex/'
 

media/css/main2.css

@@ -125,6 +125,15 @@ ul#expeditionlist
     width: 300px
 }
 
+div.survexblock
+{
+    width:50%;
+    background-color:#e0e0e0;
+}
+p.indent
+{
+    margin-left:10px;
+}
 
 #expoHeader {
 	width:100%;

parsers/logbooks.py

@@ -2,6 +2,9 @@
 
 import troggle.settings as settings
 import troggle.expo.models as models
+
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
 import csv
 import re
 import datetime
@@ -21,7 +24,7 @@ def GetTripPersons(trippeople, expedition, logtime_underground):
             tripperson = mul.group(1).strip()
         if tripperson and tripperson[0] != '*':
             #assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap)
-            personyear = expedition.GetPersonExpedition(tripperson)
+            personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower())
             if not personyear:
                 print "NoMatchFor: '%s'" % tripperson    
             res.append((personyear, logtime_underground))

parsers/people.py

@@ -120,3 +120,40 @@ def LoadPersonsExpos():
         pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname="", is_guest=is_guest)
         pyo.save()
 
+# expedition name lookup cached for speed (it's a very big list)
+Gpersonexpeditionnamelookup = { }
+def GetPersonExpeditionNameLookup(expedition):
+    global Gpersonexpeditionnamelookup
+    res = Gpersonexpeditionnamelookup.get(expedition.name)
+    if res:
+        return res
+    
+    res = {}
+    duplicates = set()
+    
+    personexpeditions = models.PersonExpedition.objects.filter(expedition=expedition)
+    for personexpedition in personexpeditions:
+        possnames = [ ]
+        f = personexpedition.person.first_name.lower()
+        l = personexpedition.person.last_name.lower()
+        if l:
+            possnames.append(f + " " + l)
+            possnames.append(f + " " + l[0])
+            possnames.append(f + l[0])
+            possnames.append(f[0] + " " + l)
+        possnames.append(f)
+        if personexpedition.nickname:
+            possnames.append(personexpedition.nickname.lower())
+        
+        for possname in possnames:
+            if possname in res:
+                duplicates.add(possname)
+            else:
+                res[possname] = personexpedition
+        
+    for possname in duplicates:
+        del res[possname]
+    
+    Gpersonexpeditionnamelookup[expedition.name] = res
+    return res
+

parsers/survex.py

@@ -1,5 +1,8 @@
 import settings
 import expo.models as models
+
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
 import re
 import os
 
@@ -51,17 +54,20 @@ def fileIterator(directory, filename):
             yield survex_file, char, line
         char = char + len(line)
 
+
 def make_model(name, parent, iter_lines, sf, c, l):
+    m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l)
+    m.survexpath = m.name
     if parent:
-        m = models.SurvexBlock(name = name, parent = parent, begin_file = sf, begin_char = c, text = l)
+        m.parent = parent
-    else:
+        m.survexpath = m.parent.survexpath + "." + m.name
-        m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l)
     m.save()
 
+    # horrible local function
     def saveEnd(survex_file, count):
           if m.start_year and team:
               try:
-                  exp = models.Expedition.objects.get(year = str(file_year[1]))
+                  exp = models.Expedition.objects.get(year = str(m.start_year))
                   for file_, (role, names) in team:
                       if names.strip("\t").strip(" ") == "both" or names.strip("\t").strip(" ") == "Both":
                           names = reduce(lambda x, y: x + u" & " + y,
@@ -69,21 +75,31 @@ def make_model(name, parent, iter_lines, sf, c, l):
                                                 if names.strip("\t").strip(" ") != "both"
                                                    and names.strip("\t").strip(" ") != "Both"])
                       for name in re.split("&|/|\+|,|;", names):
+                          sname = name.strip(". ").lower()
                           try:
-                              models.PersonRole(personexpedition = exp.GetPersonExpedition(name.strip(" ")),
+                              personexpedition = GetPersonExpeditionNameLookup(exp).get(sname)
-                                                person = exp.GetPersonExpedition(name.strip(" ")).person,
+                              if personexpedition:
+                                  models.PersonRole(personexpedition = personexpedition,
+                                                person = personexpedition.person,
                                                 survex_block = m,
                                                 role = models.Role.objects.get(name = roles[role])).save()
+                              else:
+                                  print "no person", exp, sname, role
                           except AttributeError:
-                              print ("Person not found: " + name + " in " + file_).encode('ascii', 'xmlcharrefreplace')
+                              print ("Person not found: " + name + " in " + file_ + "  " + role).encode('ascii', 'xmlcharrefreplace')
               except AssertionError, inst:
                   print (unicode(inst) + ": " + unicode(file_year[0])).encode('ascii', 'xmlcharrefreplace')
-              except models.Expedition.DoesNotExist:
+              #except models.Expedition.DoesNotExist:
-                  print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace')
+              #    print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace')
 
           m.end_file = survex_file
           m.end_char = count
+          
+          if m.start_day:
+              m.date = "%04d-%02d-%02d" % (int(m.start_year), int(m.start_month), int(m.start_day))
+          
           m.save()
+    
     team = []
     file_year = None
     for survex_file, count, line in iter_lines:
@@ -120,8 +136,17 @@ def make_model(name, parent, iter_lines, sf, c, l):
     saveEnd(survex_file, count)
 
 
-for role in ["Insts", "Notes", "Pics", "Tape", "Other"]:
+#def LoadSurvexBlocks():
-    models.Role(name = role).save()
+#    survex_file = os.path.join(directory, filename + ".svx")
+#    f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb")
+
+
+def LoadAllSurvexBlocks():
+    models.Role.objects.all().delete()
+    models.SurvexBlock.objects.all().delete()
+    for role in ["Insts", "Notes", "Pics", "Tape", "Other"]:
+        models.Role(name = role).save()
+    filename = "all"
+    make_model("all", None, fileIterator("", filename), filename, 0, "")
+
 
-filename = "all"
-make_model("", None, fileIterator("", filename), filename, 0, "")

templates/base.html

@@ -9,7 +9,9 @@
 <script src="{{ settings.MEDIA_URL }}js/base.js" type="text/javascript"></script>
 {% block javascript %}{% endblock %}
 </head>
+
 <body>
+<a href="/">
 <div style="float:none">
   <div id="expoHeader" style="background:#222"> <img src="{{ settings.MEDIA_URL }}loserBanner.jpg" style="position:relative;width:inherit;height:inherit;"/>
     <div id="expoHeaderText">
@@ -26,6 +28,7 @@
     {% endblock %}
   </div>
 </div>
+</a>
 <div id="nav" style="float:left; top:200; background:#999; width:200; height:100%"> {% block nav %}
   
   {% endblock %} </div>

templates/expedition.html

@@ -18,7 +18,7 @@
 <tr><th>Caver</th><th>From</th><th>To</th></tr>
 {% for personexpedition in expedition.personexpedition_set.all %}
   <tr>
-    <td><a href="{% url personexpedition personexpedition.person.href personexpedition.expedition.year%}">{{personexpedition.person}}</a></td>
+    <td><a href="{% url personexpedition personexpedition.person.href personexpedition.expedition.year %}">{{personexpedition.person}}</a></td>
     <td>{{personexpedition.date_from}}</td>
     <td>{{personexpedition.date_to}}</td>
   </tr>
@@ -37,7 +37,7 @@
   <tr>
     <td>{{logbookentry.date}}</td>
     <td><a href="{% url logbookentry logbookentry.href %}">{{logbookentry.title|safe}}</td>
-    <td><a href="{% url person logbookentry.author.person.href%}">{{logbookentry.author.name}}</a></td>
+    <td><a href="{% url personexpedition logbookentry.author.person.href logbookentry.author.expedition.year %}">{{logbookentry.author.name}}</a></td>
     <td>{{logbookentry.place}}</td>
   </tr>
 {% endfor %}

templates/index.html

@@ -6,7 +6,12 @@
 {% block content %}
 
 <h2>The unfinished front page</h2>
-<p>Some handy links into the less incomplete parts of this webpage</p>
+<b>Work down through an expedition page link</b>
+<p class="indent">Remaining work: continue to build up the network; tables of trips per year per person; parse 1996 logbook; 
+continue to correct the name matching and spelling; detect T/U on log entries; match caves to log entries; see the cave list; 
+simplify the survex parsing code (if necessary); vast front-page layout table of folks and caving trips and years; 
+links between logbooks and survex blocks to cave things; where are the subcaves; mini-tree of survexblocks; connect sketches 
+to caves to survey blocks and render thumbnailwise; all images to start appearing in pages; and so on</p>
 
 <h3>{{message}}</h3>
 
@@ -15,7 +20,10 @@
     <li><a href="/statistics">Statistics of what's loaded in the database</a></li>
 </ul>
 
-<form action="" method="GET"><input type="submit" name="reload" value="Reload"></form>
+<form action="" method="GET">
+  <input type="submit" name="reloadexpos" value="Reload Expos">
+  <input type="submit" name="reloadsurvex" value="Reload Survex">
+</form>
 
 <ul id="expeditionlist">
     <li>

templates/logbookentry.html

@@ -25,11 +25,10 @@
 <tr>
   {% ifequal persontrip.person_expedition logbookentry.author %}
     <td class="author">
-{{persontrip.person_expedition.person.personrole_set.count}}
   {% else %}
     <td>
   {% endifequal %}
-  <a href="{% url person persontrip.person_expedition.person.href %}">{{persontrip.person_expedition.person}}</a>
+  <a href="{% url personexpedition persontrip.person_expedition.person.href persontrip.person_expedition.expedition.year %}">{{persontrip.person_expedition.person}}</a>
   </td>
   
   <td>

templates/personexpedition.html

@@ -7,13 +7,19 @@
 {% block content %}
 <h2>{{personexpedition.person}}: {{personexpedition.expedition}} ({{personexpedition.date_from}} - {{personexpedition.date_to}})</h2>
 
+<h3>{{message}}</h3>
+
+<p>Needs links fore and back through expeditions attended by this person (or as a complete barchart type list with one date in bold)</p>
+<p>Needs lists below to be sorted by date, and the duplicates removed from survey role list</p>
+<p>Finally, a correspondence between these two columns</p>
+
 <div id="col2">
 <table class="survexcontibutions">
 <tr><th>Date</th><th>Place</th><th>Role</th></tr>
 {% for personrole in personexpedition.personrole_set.all %}
   <tr>
-    <td>{{personrole.survex_block.start_month}}</td>
+    <td>{{personrole.survex_block.date}}</td>
-    <td>{{personrole.survex_block.name}}</td>
+    <td><a href="{% url survexblock personrole.survex_block.survexpath %}">{{personrole.survex_block.survexpath}}</a></td>
     <td>{{personrole.role}}</td>
   </tr>
 {% endfor %}

templates/survexblock.html

@@ -0,0 +1,47 @@
+{% extends "base.html" %}
+{% load wiki_markup %}
+{% load survex_markup %}
+
+
+{% block title %}Survex Block{% endblock %}
+
+{% block content %}
+<h2>Survex Block {{survexblock.survexpath}}</h2>
+
+<p>Needs duplicates removed from right hand column</p>
+<p>Needs links to survex file presentation</p>
+<p>Needs to start dealing with misspellings of names (prob by editing the originals)</p>
+
+<div id="col2">
+
+{% if survexblock.parent %}
+    <p>Survey block above:</p>
+    <p class="indent"><a href="{% url survexblock survexblock.parent.survexpath %}">{{survexblock.parent.survexpath}}</a></p>
+{% endif %}
+
+{% if survexblock.survexblock_set.all %}
+    <p>Survey blocks below:</p>
+{% for survexblockdown in survexblock.survexblock_set.all %}
+    <p class="indent"><a href="{% url survexblock survexblockdown.survexpath %}">{{survexblockdown.survexpath}}</a></p>
+{% endfor %}
+{% endif %}
+
+<p>Date: {{survexblock.date}}</p>
+
+<table>
+{% for personrole in survexblock.personrole_set.all %}
+<tr>
+  <td><a href="{% url personexpedition personrole.personexpedition.person.href personrole.personexpedition.expedition.year%}">{{personrole.personexpedition.person}}</a></td>
+  <td>{{personrole.role}}</td>
+</tr>
+{% endfor %}
+</table>
+
+
+</div>
+
+<div id="col1">
+{{ftext|survex_to_html}}
+</div>
+
+{% endblock %}

urls.py

@@ -28,7 +28,9 @@ urlpatterns = patterns('',
     
     url(r'^expedition/(\d+)$', expedition, name="expedition"),
     url(r'^personexpedition/(.+?)/(\d+)$', personexpedition, name="personexpedition"),
-    
+    url(r'^survexblock/(.+)$', survexblock, name="survexblock"),
+
+        
     url(r'^statistics/?$', stats, name="stats"),
     
     (r'^survey/?$', surveyindex),