survey scans features added

core/admin.py

@@ -19,7 +19,7 @@ class TroggleModelAdmin(admin.ModelAdmin):
         js = ('js/jquery.js','js/QM_helper.js')
 
 class RoleInline(admin.TabularInline):
-    model = PersonRole
+    model = SurvexPersonRole
     extra = 4
 
 class SurvexBlockAdmin(TroggleModelAdmin):
@@ -116,7 +116,7 @@ admin.site.register(Entrance, EntranceAdmin)
 admin.site.register(SurvexBlock, SurvexBlockAdmin)
 admin.site.register(Expedition)
 admin.site.register(Person,PersonAdmin)
-admin.site.register(PersonRole)
+admin.site.register(SurvexPersonRole)
 admin.site.register(PersonExpedition,PersonExpeditionAdmin)
 admin.site.register(LogbookEntry, LogbookEntryAdmin)
 #admin.site.register(PersonTrip)
@@ -124,6 +124,9 @@ admin.site.register(QM, QMAdmin)
 admin.site.register(Survey, SurveyAdmin)
 admin.site.register(ScannedImage)
 
+admin.site.register(SurvexScansFolder)
+admin.site.register(SurvexScanSingle)
+
 def export_as_json(modeladmin, request, queryset):
     response = HttpResponse(mimetype="text/json")
     response['Content-Disposition'] = 'attachment; filename=troggle_output.json'

core/models_survex.py

@@ -1,6 +1,9 @@
 from django.db import models
 from django.conf import settings
 import os
+import urlparse
+import re
+from django.core.urlresolvers import reverse
 
 
 ###########################################################
@@ -62,6 +65,8 @@ class SurvexLeg(models.Model):
     tape        = models.FloatField()
     compass     = models.FloatField()
     clino       = models.FloatField()
+
+
 #
 # Single SurvexBlock 
 # 
@@ -77,7 +82,10 @@ class SurvexBlock(models.Model):
     survexfile = models.ForeignKey("SurvexFile", blank=True, null=True)
     begin_char = models.IntegerField()  # code for where in the survex data files this block sits
     survexpath = models.CharField(max_length=200)   # the path for the survex stations
-    refscandir = models.CharField(max_length=100)
+    
+    survexscansfolder = models.ForeignKey("SurvexScansFolder", null=True)  
+    #refscandir = models.CharField(max_length=100)
+    
     totalleglength = models.FloatField()
     
     class Meta:
@@ -114,10 +122,7 @@ class SurvexTitle(models.Model):
 #
 # member of a SurvexBlock
 #
-class PersonRole(models.Model):
-    survexblock        = models.ForeignKey('SurvexBlock')
-    
-    ROLE_CHOICES = (
+ROLE_CHOICES = (
         ('insts','Instruments'),
         ('dog','Other'),
         ('notes','Notes'),
@@ -128,8 +133,10 @@ class PersonRole(models.Model):
         ('disto','Disto'),
         ('consultant','Consultant'),
         )
-    nrole = models.CharField(choices=ROLE_CHOICES, max_length=200, blank=True, null=True)
 
+class SurvexPersonRole(models.Model):
+    survexblock         = models.ForeignKey('SurvexBlock')
+    nrole               = models.CharField(choices=ROLE_CHOICES, max_length=200, blank=True, null=True)
         # increasing levels of precision
     personname          = models.CharField(max_length=100)
     person              = models.ForeignKey('Person', blank=True, null=True)
@@ -140,3 +147,20 @@ class PersonRole(models.Model):
         return unicode(self.person) + " - " + unicode(self.survexblock) + " - " + unicode(self.nrole)
         
     
+class SurvexScansFolder(models.Model):
+    fpath               = models.CharField(max_length=200)
+    walletname          = models.CharField(max_length=200)
+    
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansfolder', kwargs={"path":re.sub("#", "%23", self.walletname)}))
+    
+class SurvexScanSingle(models.Model):
+    ffile               = models.CharField(max_length=200)
+    name                = models.CharField(max_length=200)
+    survexscansfolder   = models.ForeignKey("SurvexScansFolder", null=True)
+    
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansingle', kwargs={"path":re.sub("#", "%23", self.survexscansfolder.walletname), "file":self.name}))
+    
+        
+    

core/view_surveys.py

@@ -4,10 +4,12 @@ from django.shortcuts import render_to_response
 from django.http import HttpResponse, Http404
 import os
 import re
+from troggle.core.models import SurvexScansFolder, SurvexScanSingle, SurvexBlock
 
 # inline fileabstraction into here if it's not going to be useful anywhere else 
 # keep things simple and ignore exceptions everywhere for now
 
+
 def getMimeType(extension):
     try:
         return {"txt": "text/plain",
@@ -159,3 +161,18 @@ def jgtuploadfile(request):
     print "gothere"
     return render_to_response('fileupload.html', {'message':message, 'filesuploaded':filesuploaded, 'settings': settings})
 
+def surveyscansfolder(request, path):
+    #print [ s.walletname  for s in SurvexScansFolder.objects.all() ]
+    survexscansfolder = SurvexScansFolder.objects.get(walletname=path)
+    return render_to_response('survexscansfolder.html', { 'survexscansfolder':survexscansfolder, 'settings': settings })
+
+def surveyscansingle(request, path, file):
+    survexscansfolder = SurvexScansFolder.objects.get(walletname=path)
+    survexscansingle = SurvexScanSingle.objects.get(survexscansfolder=survexscansfolder, name=file)
+    return HttpResponse(content=open(survexscansingle.ffile), mimetype="image/png")
+    #return render_to_response('survexscansfolder.html', { 'survexscansfolder':survexscansfolder, 'settings': settings })
+    
+def surveyscansfolders(request):
+    survexscansfolders = SurvexScansFolder.objects.all()
+    return render_to_response('survexscansfolders.html', { 'survexscansfolders':survexscansfolders, 'settings': settings })
+    

core/views_survex.py

@@ -8,7 +8,7 @@ import datetime
 import difflib
 
 from troggle.core.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry, Cave
-from troggle.core.models import SurvexBlock, PersonRole, SurvexFile, SurvexDirectory, SurvexTitle
+from troggle.core.models import SurvexBlock, SurvexPersonRole, SurvexFile, SurvexDirectory, SurvexTitle
 from parsers.people import GetPersonExpeditionNameLookup
 
 import troggle.settings as settings
@@ -306,7 +306,9 @@ def survexcaveslist(request):
 # parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people
 # doesn't use recursion.  just writes it twice
 def survexcavesingle(request, survex_cave):
+    breload = False
     cave = Cave.objects.get(kataster_number=survex_cave)
+    if breload:
         parsers.survex.ReloadSurvexCave(survex_cave)
     return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })
 

databaseReset.py

@@ -61,6 +61,11 @@ def import_surveys():
     import parsers.surveys
     parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
 
+def import_surveyscans():
+    import parsers.surveys
+    parsers.surveys.LoadListScans(settings.SURVEY_SCANS)
+
+    
 def import_descriptions():
     import parsers.descriptions
     parsers.descriptions.getDescriptions()
@@ -77,6 +82,7 @@ def reset():
     make_dirs()
     import_cavetab()
     import_people()
+    import_surveyscans()
     import_survex()
     import_logbooks()
     import_QMs()
@@ -104,6 +110,8 @@ if __name__ == "__main__":
     import sys
     if "desc" in sys.argv:
         resetdesc()
+    elif "scans" in sys.argv:
+        import_surveyscans()
     elif "reset" in sys.argv:
         reset()
     elif "survex" in sys.argv:

parsers/survex.py

@@ -68,7 +68,12 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
         # detect ref line pointing to the scans directory
         mref = comment and re.match('.*?ref.*?(\d+)\s*#\s*(\d+)', comment)
         if mref:
-            survexblock.refscandir = "%s/%s%%23%s" % (mref.group(1), mref.group(1), mref.group(2))
+            refscan = "%s#%s" % (mref.group(1), mref.group(2))
+            print refscan
+            survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan)
+            if survexscansfolders:
+                survexblock.survexscansfolder = survexscansfolders[0]
+                #survexblock.refscandir = "%s/%s%%23%s" % (mref.group(1), mref.group(1), mref.group(2))
                 survexblock.save()   
             continue
         
@@ -128,7 +133,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
                         personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
                         if (personexpedition, tm) not in teammembers:
                             teammembers.append((personexpedition, tm))
-                            personrole = models.PersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
+                            personrole = models.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
                             if personexpedition:
                                 personrole.person=personexpedition.person
                             personrole.save()

parsers/surveys.py

@@ -2,8 +2,8 @@ import sys, os, types, logging
 #sys.path.append('C:\\Expo\\expoweb')
 #from troggle import *
 #os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings'
-import troggle.settings as settings
-from troggle.core.models import *
+import settings
+from core.models import *
 from PIL import Image
 #import settings
 #import core.models as models
@@ -146,3 +146,56 @@ def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the
         return file.info['interlace']
     else:
         return False
+
+
+# handles url or file
+def GetListDir(sdir):
+    res = [ ]
+    if sdir[:7] == "http://":
+        s = urllib.urlopen(sdir)
+    else:
+        for f in os.listdir(sdir):
+            if f[0] != ".":
+                ff = os.path.join(sdir, f)
+                res.append((f, ff, os.path.isdir(ff)))
+    return res
+        
+# this iterates through the scans directories (either here or on the remote server)
+# and builds up the models we can access later
+def LoadListScans(surveyscansdir):
+    SurvexScanSingle.objects.all().delete()
+    SurvexScansFolder.objects.all().delete()
+
+    for f, ff, fisdir in GetListDir(surveyscansdir):
+        if not fisdir:
+            continue
+        
+        # do the year folders
+        if re.match("\d\d\d\d$", f):
+            for fy, ffy, fisdiry in GetListDir(ff):
+                assert fisdiry, ffy
+                survexscansfolder = SurvexScansFolder(fpath=ffy, walletname=fy)
+                survexscansfolder.save()
+                for fyf, ffyf, fisdiryf in GetListDir(ffy):
+                    assert not fisdiryf, ffyf
+                    survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)
+                    survexscansingle.save()
+        elif f != "thumbs":
+            survexscansfolder = SurvexScansFolder(fpath=ff, walletname=f)
+            survexscansfolder.save()
+            gld = [ ]
+            
+            # flatten out any directories in these book files
+            for (fyf, ffyf, fisdiryf) in GetListDir(ff):
+                if fisdiryf:
+                    gld.extend(GetListDir(ffyf))
+                else:
+                    gld.append((fyf, ffyf, fisdiryf))
+            
+            for (fyf, ffyf, fisdiryf) in gld:
+                assert not fisdiryf, ffyf
+                survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)
+                survexscansingle.save()
+            
+            
+        

templates/survexblock.html

@@ -23,9 +23,9 @@
 
 {% if survexblock.survexblock_set.all %}
   <p>Survey blocks below:</p>
-{% for survexblockdown in survexblock.survexblock_set.all %}
+  {% for survexblockdown in survexblock.survexblock_set.all %}
     <p class="indent"><a href="{% url survexblock survexblockdown.survexpath %}">{{survexblockdown.survexpath}}</a></p>
-{% endfor %}
+  {% endfor %}
 {% endif %}
 
 <p>Date: {{survexblock.date}}</p>
@@ -43,7 +43,7 @@
 </div>
 
 <div class="survexblock">
-{{ftext|survex_to_html}}
+  {{ftext|survex_to_html}}
 </div>
 
 {% endblock %}

templates/survexscansfolder.html

@@ -0,0 +1,23 @@
+{% extends "base.html" %}
+{% load wiki_markup %}
+{% load survex_markup %}
+
+{% block title %}Survex Scans Folder{% endblock %}
+
+{% block content %}
+
+<h3>Survex Scans in: {{survexscansfolder.walletname}}</h3>
+<table>
+{% for survexscansingle in survexscansfolder.survexscansingle_set.all %}
+  <tr>
+    <td class="survexscansingle"><a href="{{survexscansingle.get_absolute_url}}">{{survexscansingle.name}}</a></td>
+    <td>
+    {% for survexblock in survexscansingle.survexblock_set %}
+      {{survexblock}}
+    {% endfor %}
+    </td>
+  </tr>
+{% endfor %}
+</table>
+
+{% endblock %}

templates/survexscansfolders.html

@@ -0,0 +1,21 @@
+{% extends "base.html" %}
+{% load wiki_markup %}
+{% load survex_markup %}
+
+{% block title %}All Survex scans folders{% endblock %}
+
+{% block content %}
+
+<h3>All Survex scans folders</h3>
+<table>
+<tr><th>Scans folder</th><th>Files</th><th>Survex blocks</th></tr>
+{% for survexscansfolder in survexscansfolders %}
+  <tr>
+    <td><a href="{{survexscansfolder.get_absolute_url}}">{{survexscansfolder.walletname}}</a></td>
+    <td>{{survexscansfolder.survexscansingle_set.all|length}}</td>
+    <td>{{survexscansfolder.survexblock_set.all|length}}</td>
+  </tr>
+{% endfor %}
+</table>
+
+{% endblock %}

templates/svxcavesingle.html

@@ -66,8 +66,8 @@
   </td>
 
   <td>
-  {% if survexblock.refscandir %}
-    <b><a href="/survey_scans/{{survexblock.refscandir}}/">scans</a></b>
+  {% if survexblock.survexscansfolder %}
+    <b><a href="{{survexblock.survexscansfolder.get_absolute_url}}">{{survexblock.survexscansfolder.walletname}}</a></b>
   {% endif %}
   </td>
 </tr>

templates/svxfile.html

@@ -52,7 +52,6 @@ $(document).ready(function()
     <input type="submit" name="diff" value="Diffy" />
     <input type="submit" name="save" value="Save"/>
     <input type="submit" name="process" value="Process" title="executes cavern"/>
-    (Not implemented: <input type="submit" name="svncheckin" value="svn check-in"/>)
 </form>
 
 <div id="difflistajax">

urls.py

@@ -12,6 +12,9 @@ from django.views.generic.list_detail import object_list
 from django.contrib import admin
 admin.autodiscover()
 
+
+# type url probably means it's used.
+
 urlpatterns = patterns('',
     
     url(r'^$',              views_other.frontpage,      name="frontpage"),
@@ -102,8 +105,11 @@ urlpatterns = patterns('',
 
     
             
-     (r'^survey_scans/(?P<path>.*)$', 'django.views.static.serve',
-        {'document_root': settings.SURVEY_SCANS, 'show_indexes':True}),
+    #(r'^survey_scans/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.SURVEY_SCANS, 'show_indexes':True}),
+    url(r'^survey_scans/$',                        view_surveys.surveyscansfolders, name="surveyscansfolders"), 
+    url(r'^survey_scans/(?P<path>[^/]+)/$',        view_surveys.surveyscansfolder,  name="surveyscansfolder"), 
+    url(r'^survey_scans/(?P<path>[^/]+)/(?P<file>[^/]+(?:png|jpg))$', 
+                                                   view_surveys.surveyscansingle,   name="surveyscansingle"), 
 
     (r'^photos/(?P<path>.*)$', 'django.views.static.serve',
         {'document_root': settings.PHOTOS_ROOT, 'show_indexes':True}),