[svn] survey block object

Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8199 by julian @ 1/19/2009 12:22 AM

parsers/logbooks.py

@@ -2,6 +2,9 @@
 
 import troggle.settings as settings
 import troggle.expo.models as models
+
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
 import csv
 import re
 import datetime
@@ -21,7 +24,7 @@ def GetTripPersons(trippeople, expedition, logtime_underground):
             tripperson = mul.group(1).strip()
         if tripperson and tripperson[0] != '*':
             #assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap)
-            personyear = expedition.GetPersonExpedition(tripperson)
+            personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower())
             if not personyear:
                 print "NoMatchFor: '%s'" % tripperson    
             res.append((personyear, logtime_underground))

parsers/people.py

@@ -120,3 +120,40 @@ def LoadPersonsExpos():
         pyo = models.PersonExpedition(person = pObject, expedition = yo, nickname="", is_guest=is_guest)
         pyo.save()
 
+# expedition name lookup cached for speed (it's a very big list)
+Gpersonexpeditionnamelookup = { }
+def GetPersonExpeditionNameLookup(expedition):
+    global Gpersonexpeditionnamelookup
+    res = Gpersonexpeditionnamelookup.get(expedition.name)
+    if res:
+        return res
+    
+    res = {}
+    duplicates = set()
+    
+    personexpeditions = models.PersonExpedition.objects.filter(expedition=expedition)
+    for personexpedition in personexpeditions:
+        possnames = [ ]
+        f = personexpedition.person.first_name.lower()
+        l = personexpedition.person.last_name.lower()
+        if l:
+            possnames.append(f + " " + l)
+            possnames.append(f + " " + l[0])
+            possnames.append(f + l[0])
+            possnames.append(f[0] + " " + l)
+        possnames.append(f)
+        if personexpedition.nickname:
+            possnames.append(personexpedition.nickname.lower())
+        
+        for possname in possnames:
+            if possname in res:
+                duplicates.add(possname)
+            else:
+                res[possname] = personexpedition
+        
+    for possname in duplicates:
+        del res[possname]
+    
+    Gpersonexpeditionnamelookup[expedition.name] = res
+    return res
+

parsers/survex.py

@@ -1,5 +1,8 @@
 import settings
 import expo.models as models
+
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
 import re
 import os
 
@@ -51,17 +54,20 @@ def fileIterator(directory, filename):
             yield survex_file, char, line
         char = char + len(line)
 
+
 def make_model(name, parent, iter_lines, sf, c, l):
+    m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l)
+    m.survexpath = m.name
     if parent:
-        m = models.SurvexBlock(name = name, parent = parent, begin_file = sf, begin_char = c, text = l)
-    else:
-        m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l)
+        m.parent = parent
+        m.survexpath = m.parent.survexpath + "." + m.name
     m.save()
 
+    # horrible local function
     def saveEnd(survex_file, count):
           if m.start_year and team:
               try:
-                  exp = models.Expedition.objects.get(year = str(file_year[1]))
+                  exp = models.Expedition.objects.get(year = str(m.start_year))
                   for file_, (role, names) in team:
                       if names.strip("\t").strip(" ") == "both" or names.strip("\t").strip(" ") == "Both":
                           names = reduce(lambda x, y: x + u" & " + y,
@@ -69,21 +75,31 @@ def make_model(name, parent, iter_lines, sf, c, l):
                                                 if names.strip("\t").strip(" ") != "both"
                                                    and names.strip("\t").strip(" ") != "Both"])
                       for name in re.split("&|/|\+|,|;", names):
+                          sname = name.strip(". ").lower()
                           try:
-                              models.PersonRole(personexpedition = exp.GetPersonExpedition(name.strip(" ")),
-                                                person = exp.GetPersonExpedition(name.strip(" ")).person,
+                              personexpedition = GetPersonExpeditionNameLookup(exp).get(sname)
+                              if personexpedition:
+                                  models.PersonRole(personexpedition = personexpedition,
+                                                person = personexpedition.person,
                                                 survex_block = m,
                                                 role = models.Role.objects.get(name = roles[role])).save()
+                              else:
+                                  print "no person", exp, sname, role
                           except AttributeError:
-                              print ("Person not found: " + name + " in " + file_).encode('ascii', 'xmlcharrefreplace')
+                              print ("Person not found: " + name + " in " + file_ + "  " + role).encode('ascii', 'xmlcharrefreplace')
               except AssertionError, inst:
                   print (unicode(inst) + ": " + unicode(file_year[0])).encode('ascii', 'xmlcharrefreplace')
-              except models.Expedition.DoesNotExist:
-                  print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace')
+              #except models.Expedition.DoesNotExist:
+              #    print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace')
 
           m.end_file = survex_file
           m.end_char = count
+          
+          if m.start_day:
+              m.date = "%04d-%02d-%02d" % (int(m.start_year), int(m.start_month), int(m.start_day))
+          
           m.save()
+    
     team = []
     file_year = None
     for survex_file, count, line in iter_lines:
@@ -120,8 +136,17 @@ def make_model(name, parent, iter_lines, sf, c, l):
     saveEnd(survex_file, count)
 
 
-for role in ["Insts", "Notes", "Pics", "Tape", "Other"]:
-    models.Role(name = role).save()
+#def LoadSurvexBlocks():
+#    survex_file = os.path.join(directory, filename + ".svx")
+#    f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb")
+
+
+def LoadAllSurvexBlocks():
+    models.Role.objects.all().delete()
+    models.SurvexBlock.objects.all().delete()
+    for role in ["Insts", "Notes", "Pics", "Tape", "Other"]:
+        models.Role(name = role).save()
+    filename = "all"
+    make_model("all", None, fileIterator("", filename), filename, 0, "")
+
 
-filename = "all"
-make_model("", None, fileIterator("", filename), filename, 0, "")