make 2008 logbook correctly parse

core/models_survex.py

@@ -155,6 +155,9 @@ class SurvexScansFolder(models.Model):
     fpath               = models.CharField(max_length=200)
     walletname          = models.CharField(max_length=200)
     
+    class Meta:
+        ordering = ('walletname',)
+    
     def get_absolute_url(self):
         return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansfolder', kwargs={"path":re.sub("#", "%23", self.walletname)}))
     
@@ -163,18 +166,25 @@ class SurvexScanSingle(models.Model):
     name                = models.CharField(max_length=200)
     survexscansfolder   = models.ForeignKey("SurvexScansFolder", null=True)
     
+    class Meta:
+        ordering = ('name',)
+    
     def get_absolute_url(self):
         return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansingle', kwargs={"path":re.sub("#", "%23", self.survexscansfolder.walletname), "file":self.name}))
     
         
 class TunnelFile(models.Model):
     tunnelpath          = models.CharField(max_length=200)
+    tunnelname          = models.CharField(max_length=200)
     bfontcolours        = models.BooleanField()
+    survexscansfolders  = models.ManyToManyField("SurvexScansFolder")
     survexscans         = models.ManyToManyField("SurvexScanSingle")
     survexblocks        = models.ManyToManyField("SurvexBlock")
     tunnelcontains      = models.ManyToManyField("TunnelFile")  # case when its a frame type
     filesize            = models.IntegerField(default=0)
     npaths              = models.IntegerField(default=0)
+    survextitles        = models.ManyToManyField("SurvexTitle")
+    
     
     class Meta:
         ordering = ('tunnelpath',)

core/view_surveys.py

@@ -5,6 +5,7 @@ from django.http import HttpResponse, Http404
 import os, stat
 import re
 from troggle.core.models import SurvexScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile
+import parsers.surveys
 
 # inline fileabstraction into here if it's not going to be useful anywhere else 
 # keep things simple and ignore exceptions everywhere for now
@@ -182,43 +183,50 @@ def tunneldata(request):
     tunnelfiles = TunnelFile.objects.all()
     return render_to_response('tunnelfiles.html', { 'tunnelfiles':tunnelfiles, 'settings': settings })
     
+
 def tunnelfile(request, path):
     tunnelfile = TunnelFile.objects.get(tunnelpath=path)
     tfile = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
+    return HttpResponse(content=open(tfile), mimetype="text/plain")
     
-    # just output the file
+def tunnelfileupload(request, path):
-    if not request.POST:
+    tunnelfile = TunnelFile.objects.get(tunnelpath=path)
-        return HttpResponse(content=open(tfile), mimetype="text/plain")
+    tfile = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
     
     project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"]
     print (project, user, tunnelversion)
-    for uploadedfile in request.FILES.values():
-        if uploadedfile.field_name != "sketch":
-            return HttpResponse(content="Error: non-sketch file uploaded", mimetype="text/plain")
-        if uploadedfile.content_type != "text/plain":
-            return HttpResponse(content="Error: non-plain content type", mimetype="text/plain")
     
-        # could use this to add new files
-        if os.path.split(path)[1] != uploadedfile.name:  
-            return HttpResponse(content="Error: name disagrees", mimetype="text/plain")
     
-        orgsize = tunnelfile.filesize   # = os.stat(tfile)[stat.ST_SIZE]
+    assert len(request.FILES.values()) == 1, "only one file to upload"
     
-        ttext = uploadedfile.read()
+    uploadedfile = request.FILES.values()[0]
         
-        # could check that the user and projects agree here
+    if uploadedfile.field_name != "sketch":
+        return HttpResponse(content="Error: non-sketch file uploaded", mimetype="text/plain")
+    if uploadedfile.content_type != "text/plain":
+        return HttpResponse(content="Error: non-plain content type", mimetype="text/plain")
         
-        fout = open(tfile, "w")
+    # could use this to add new files
-        fout.write(ttext)
+    if os.path.split(path)[1] != uploadedfile.name:  
-        fout.close()
+        return HttpResponse(content="Error: name disagrees", mimetype="text/plain")
     
-        # redo its settings of 
+    orgsize = tunnelfile.filesize   # = os.stat(tfile)[stat.ST_SIZE]
-        tunnelfile.filesize = os.stat(tfile)[stat.ST_SIZE]
-        tunnelfile.save()
         
-        uploadedfile.close()
+    ttext = uploadedfile.read()
-        message = "File size %d overwritten with size %d" % (orgsize, tunnelfile.filesize)
+    
-        return HttpResponse(content=message, mimetype="text/plain")
+    # could check that the user and projects agree here
+    
+    fout = open(tfile, "w")
+    fout.write(ttext)
+    fout.close()
+    
+    # redo its settings of 
+    parsers.surveys.SetTunnelfileInfo(tunnelfile)
+    tunnelfile.save()
+    
+    uploadedfile.close()
+    message = "File size %d overwritten with size %d" % (orgsize, tunnelfile.filesize)
+    return HttpResponse(content=message, mimetype="text/plain")
 
     
     

databaseReset.py

@@ -63,7 +63,7 @@ def import_surveys():
 
 def import_surveyscans():
     import parsers.surveys
-    parsers.surveys.LoadListScans(settings.SURVEY_SCANS)
+    parsers.surveys.LoadListScans()
 
     
 def import_descriptions():

parsers/logbooks.py

@@ -120,7 +120,7 @@ def Parselogwikitxt(year, expedition, txt):
     trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt)
     for triphead, triptext in trippara:
         tripheadp = triphead.split("|")
-        #print tripheadp
+        #print "ttt", tripheadp
         assert len(tripheadp) == 3, (tripheadp, triptext)
         tripdate, tripplace, trippeople = tripheadp
         tripsplace = tripplace.split(" - ")
@@ -135,7 +135,7 @@ def Parselogwikitxt(year, expedition, txt):
             tu = ""
             #assert tripcave == "Journey", (triphead, triptext)
 
-        print tripdate
+        #print tripdate
         ldate = ParseDate(tripdate.strip(), year)
         #print "\n", tripcave, "---   ppp", trippeople, len(triptext)
         EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
@@ -155,7 +155,8 @@ def Parseloghtmltxt(year, expedition, txt):
                             \s*$
                      ''', trippara)
         if not s:
-            print "can't parse: ", trippara  # this is 2007 which needs editing
+            if not re.search("Rigging Guide", trippara):
+                print "can't parse: ", trippara  # this is 2007 which needs editing
             #assert s, trippara
             continue
 
@@ -218,7 +219,7 @@ def Parseloghtml01(year, expedition, txt):
         ltriptext = re.sub("</?b>", "'''", ltriptext)
         
 
-        print ldate, trippeople.strip()
+        #print ldate, trippeople.strip()
             # could includ the tripid (url link for cross referencing)
         EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
 
@@ -236,8 +237,7 @@ def Parseloghtml03(year, expedition, txt):
         if re.match("T/U|Time underwater", sheader[-1]):
             tu = sheader.pop()
         if len(sheader) != 3:
-            print sheader
+            print "header not three pieces", sheader
-        #    continue
         tripdate, triptitle, trippeople = sheader
         ldate = ParseDate(tripdate.strip(), year)
         triptitles = triptitle.split(" , ")

parsers/surveys.py

@@ -165,13 +165,42 @@ def GetListDir(sdir):
                 res.append((f, ff, os.path.isdir(ff)))
     return res
         
+        
+        
+
+
+def LoadListScansFile(survexscansfolder):
+    gld = [ ]
+    
+    # flatten out any directories in these book files
+    for (fyf, ffyf, fisdiryf) in GetListDir(survexscansfolder.fpath):
+        if fisdiryf:
+            gld.extend(GetListDir(ffyf))
+        else:
+            gld.append((fyf, ffyf, fisdiryf))
+    
+    for (fyf, ffyf, fisdiryf) in gld:
+        assert not fisdiryf, ffyf
+        if re.search("\.(?:png|jpg|jpeg)(?i)$", fyf):
+            survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)
+            survexscansingle.save()
+
+        
 # this iterates through the scans directories (either here or on the remote server)
 # and builds up the models we can access later
-def LoadListScans(surveyscansdir):
+def LoadListScans():
     SurvexScanSingle.objects.all().delete()
     SurvexScansFolder.objects.all().delete()
 
-    for f, ff, fisdir in GetListDir(surveyscansdir):
+    # first do the smkhs (large kh survey scans) directory
+    survexscansfoldersmkhs = SurvexScansFolder(fpath=os.path.join(settings.SURVEY_SCANS, "smkhs"), walletname="smkhs") 
+    if os.path.isdir(survexscansfoldersmkhs.fpath):
+        survexscansfoldersmkhs.save()
+        LoadListScansFile(survexscansfoldersmkhs)
+        
+    
+    # iterate into the surveyscans directory
+    for f, ff, fisdir in GetListDir(os.path.join(settings.SURVEY_SCANS, "surveyscans")):
         if not fisdir:
             continue
         
@@ -181,27 +210,45 @@ def LoadListScans(surveyscansdir):
                 assert fisdiry, ffy
                 survexscansfolder = SurvexScansFolder(fpath=ffy, walletname=fy)
                 survexscansfolder.save()
-                for fyf, ffyf, fisdiryf in GetListDir(ffy):
+                LoadListScansFile(survexscansfolder)
-                    assert not fisdiryf, ffyf
+        
-                    survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)
+        # do the 
-                    survexscansingle.save()
         elif f != "thumbs":
             survexscansfolder = SurvexScansFolder(fpath=ff, walletname=f)
             survexscansfolder.save()
-            gld = [ ]
+            LoadListScansFile(survexscansfolder)
             
-            # flatten out any directories in these book files
-            for (fyf, ffyf, fisdiryf) in GetListDir(ff):
-                if fisdiryf:
-                    gld.extend(GetListDir(ffyf))
-                else:
-                    gld.append((fyf, ffyf, fisdiryf))
 
-            for (fyf, ffyf, fisdiryf) in gld:
+def FindTunnelScan(tunnelfile, path):
-                assert not fisdiryf, ffyf
+    scansfolder, scansfile = None, None
-                survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)
+    mscansdir = re.search("(\d\d\d\d#\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg))$", path)
-                survexscansingle.save()
+    if mscansdir:
+        scansfolderl = SurvexScansFolder.objects.filter(walletname=mscansdir.group(1))
+        if len(scansfolderl):
+            assert len(scansfolderl) == 1
+            scansfolder = scansfolderl[0]
+        if scansfolder:
+            scansfilel = scansfolder.survexscansingle_set.filter(name=mscansdir.group(2))
+            if len(scansfilel):
+                assert len(scansfilel) == 1
+                scansfile = scansfilel[0]
             
+        if scansfolder:
+            tunnelfile.survexscansfolders.add(scansfolder)
+        if scansfile:
+            tunnelfile.survexscans.add(scansfile)
+    
+    elif path and not re.search("\.(?:png|jpg)$(?i)", path):
+        name = os.path.split(path)[1]
+        print "ttt", tunnelfile.tunnelpath, path, name
+        rtunnelfilel = TunnelFile.objects.filter(tunnelname=name)
+        if len(rtunnelfilel):
+            assert len(rtunnelfilel) == 1, ("two paths with name of", path, "need more discrimination coded")
+            rtunnelfile = rtunnelfilel[0]
+            #print "ttt", tunnelfile.tunnelpath, path, name, rtunnelfile.tunnelpath
+            tunnelfile.tunnelcontains.add(rtunnelfile)
+
+    tunnelfile.save()
 
 
 def SetTunnelfileInfo(tunnelfile):
@@ -219,11 +266,13 @@ def SetTunnelfileInfo(tunnelfile):
     
     # <tunnelxml tunnelversion="version2009-06-21 Matienzo" tunnelproject="ireby" tunneluser="goatchurch" tunneldate="2009-06-29 23:22:17">
     # <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0">
-    print tunnelfile.tunnelpath, re.findall('<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"', ttext)
+    for path, style in re.findall('<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"', ttext):
-#    npaths              = models.IntegerField()
+        FindTunnelScan(tunnelfile, path)
-#    survexscans         = models.ManyToManyField("SurvexScanSingle")
+    
-#    survexblocks        = models.ManyToManyField("SurvexBlock")
+    # should also scan and look for survex blocks that might have been included
-#    tunnelcontains      = models.ManyToManyField("TunnelFile")  # case when its a frame type
+    # and also survex titles as well.  
+    
+    tunnelfile.save()
 
 
 def LoadTunnelFiles():
@@ -240,9 +289,11 @@ def LoadTunnelFiles():
             if os.path.isdir(ff):
                 tunneldirs.append(lf)
             elif f[-4:] == ".xml":
-                tunnelfile = TunnelFile(tunnelpath=lf)
+                tunnelfile = TunnelFile(tunnelpath=lf, tunnelname=os.path.split(f[:-4])[1])
                 tunnelfile.save()
-                SetTunnelfileInfo(tunnelfile)
+                
+    for tunnelfile in TunnelFile.objects.all():
+        SetTunnelfileInfo(tunnelfile)
     
         
 

templates/survexscansfolders.html

@@ -13,7 +13,11 @@
   <tr>
     <td><a href="{{survexscansfolder.get_absolute_url}}">{{survexscansfolder.walletname}}</a></td>
     <td>{{survexscansfolder.survexscansingle_set.all|length}}</td>
-    <td>{{survexscansfolder.survexblock_set.all|length}}</td>
+    <td>
+    {% for survexblock in survexscansfolder.survexblock_set.all %}
+      <a href="{% url svx survexblock.survexfile.path %}">{{survexblock}}</a>
+    {% endfor %}
+    </td>
   </tr>
 {% endfor %}
 </table>

templates/tunnelfiles.html

@@ -8,15 +8,34 @@
 
 <h3>All Tunnel files</h3>
 <table>
-<tr><th>File</th><th>Font</th><th>Frame</th><th>SurvexBlocks</th><th>Size</th><th>Paths</td></tr>
+<tr><th>File</th><th>Font</th><th>SurvexBlocks</th><th>Size</th><th>Paths</th><th>Scans folder</th><th>Scan files</th><th>Frames</th></tr>
 {% for tunnelfile in tunnelfiles %}
   <tr>
     <td><a href="{% url tunnelfile tunnelfile.tunnelpath %}">{{tunnelfile.tunnelpath}}</a></td>
     <td>{{tunnelfile.bfontcolours}}</td>
     <td></td>
-    <td></td>
     <td>{{tunnelfile.filesize}}</td>
     <td>{{tunnelfile.npaths}}</td>   
+
+    <td>
+    {% for survexscansfolder in tunnelfile.survexscansfolders.all %}
+      <a href="{{survexscansfolder.get_absolute_url}}">{{survexscansfolder.walletname}}</a>
+    {% endfor %}
+    </td>   
+
+    <td>
+    {% for survexscansingle in tunnelfile.survexscans.all %}
+      <a href="{{survexscansingle.get_absolute_url}}">{{survexscansingle.name}}</a>
+    {% endfor %}
+    </td>   
+
+    <td>
+    {% for rtunnelfile in tunnelfile.tunnelcontains.all %}
+      <a href="{% url tunnelfile rtunnelfile.tunnelpath %}">{{rtunnelfile.tunnelpath}}</a>
+    {% endfor %}
+    </td>   
+
+
   </tr>
 {% endfor %}
 </table>

urls.py

@@ -111,6 +111,8 @@ urlpatterns = patterns('',
     
     url(r'^tunneldata/$',                          view_surveys.tunneldata,         name="tunneldata"), 
     url(r'^tunneldataraw/(?P<path>.+?\.xml)$',     view_surveys.tunnelfile,         name="tunnelfile"), 
+    url(r'^tunneldataraw/(?P<path>.+?\.xml)/upload$',view_surveys.tunnelfileupload, name="tunnelfileupload"), 
+    
     #url(r'^tunneldatainfo/(?P<path>.+?\.xml)$',    view_surveys.tunnelfileinfo,     name="tunnelfileinfo"), 
     
     (r'^photos/(?P<path>.*)$', 'django.views.static.serve',