[svn r8336] Dynamic thumbnail generation for photos and survey scans using imagekit, further improving registration system, other misc.

troggle/expo/imagekit_specs.py

@@ -0,0 +1,23 @@
+from imagekit.specs import ImageSpec
+from imagekit import processors
+
+class ResizeThumb(processors.Resize): 
+    width = 100 
+    height = 75 
+    crop = True
+
+class ResizeDisplay(processors.Resize):
+    width = 600 
+    
+class EnhanceThumb(processors.Adjustment): 
+    contrast = 1.2 
+    sharpness = 1.1 
+
+class Thumbnail(ImageSpec): 
+    access_as = 'thumbnail_image' 
+    pre_cache = True 
+    processors = [ResizeThumb, EnhanceThumb] 
+
+class Display(ImageSpec):
+    increment_count = True
+    processors = [ResizeDisplay]

troggle/expo/models.py

@@ -11,6 +11,7 @@ from django.conf import settings
 import datetime
 from decimal import Decimal, getcontext
 from  django.core.urlresolvers import reverse
+from imagekit.models import ImageModel
 getcontext().prec=2 #use 2 significant figures for decimal calculations
 
 from models_survex import *
@@ -25,6 +26,15 @@ class TroggleModel(models.Model):
     class Meta:
 	    abstract = True
 
+class TroggleImageModel(ImageModel):
+    new_since_parsing = models.BooleanField(default=False, editable=False)
+    
+    def get_admin_url(self):
+        return settings.URL_ROOT + "/admin/expo/" + self._meta.object_name.lower() + "/" + str(self.pk)
+
+    class Meta:
+	    abstract = True
+
 class Expedition(TroggleModel):
     year        = models.CharField(max_length=20, unique=True)
     name        = models.CharField(max_length=100)
@@ -517,7 +527,7 @@ class QM(TroggleModel):
 	return res
 
 photoFileStorage = FileSystemStorage(location=settings.PHOTOS_ROOT, base_url=settings.PHOTOS_URL)
-class Photo(TroggleModel): 
+class Photo(TroggleImageModel): 
     caption = models.CharField(max_length=1000,blank=True,null=True)
     contains_logbookentry = models.ForeignKey(LogbookEntry,blank=True,null=True)
     contains_person = models.ManyToManyField(Person,blank=True,null=True)
@@ -527,10 +537,14 @@ class Photo(TroggleModel):
     contains_entrance = models.ForeignKey(Entrance, related_name="photo_file",blank=True,null=True)
     nearest_survey_point = models.ForeignKey(SurveyStation,blank=True,null=True)
     nearest_QM = models.ForeignKey(QM,blank=True,null=True)
-    
-    
     lon_utm = models.FloatField(blank=True,null=True)
     lat_utm = models.FloatField(blank=True,null=True)
+    
+    class IKOptions:
+        spec_module = 'expo.imagekit_specs'
+        cache_dir = 'thumbs'
+        image_field = 'file'
+        
     #content_type = models.ForeignKey(ContentType)
     #object_id = models.PositiveIntegerField()
     #location = generic.GenericForeignKey('content_type', 'object_id')
@@ -545,7 +559,7 @@ def get_scan_path(instance, filename):
     number="%02d" % instance.survey.wallet_number + str(instance.survey.wallet_letter) #using %02d string formatting because convention was 2009#01
     return os.path.join('./',year,year+r'#'+number,instance.contents+str(instance.number_in_wallet)+r'.jpg')
 
-class ScannedImage(TroggleModel): 
+class ScannedImage(TroggleImageModel): 
     file = models.ImageField(storage=scansFileStorage, upload_to=get_scan_path)
     scanned_by = models.ForeignKey(Person,blank=True, null=True)
     scanned_on = models.DateField(null=True)
@@ -554,6 +568,11 @@ class ScannedImage(TroggleModel):
     number_in_wallet = models.IntegerField(null=True)
     lon_utm = models.FloatField(blank=True,null=True)
     lat_utm = models.FloatField(blank=True,null=True)
+
+    class IKOptions:
+        spec_module = 'expo.imagekit_specs'
+        cache_dir = 'thumbs'
+        image_field = 'file'
     #content_type = models.ForeignKey(ContentType)
     #object_id = models.PositiveIntegerField()
     #location = generic.GenericForeignKey('content_type', 'object_id')

troggle/localsettingswindows.py

@@ -1,20 +1,48 @@
-DATABASE_ENGINE = 'mysql'           # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
-DATABASE_NAME = 'troggle'             # Or path to database file if using sqlite3.
-DATABASE_USER = 'troggle'             # Not used with sqlite3.
-DATABASE_PASSWORD = 'troggle'         # Not used with sqlite3.
-DATABASE_HOST = 'localhost'             # Set to empty string for localhost. Not used with sqlite3.
-DATABASE_PORT = '3306'             # Set to empty string for default. Not used with sqlite3.
+DATABASE_ENGINE = ''           # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
+DATABASE_NAME = ''             # Or path to database file if using sqlite3.
+DATABASE_USER = ''             # Not used with sqlite3.
+DATABASE_PASSWORD = ''         # Not used with sqlite3.
+DATABASE_HOST = ''             # Set to empty string for localhost. Not used with sqlite3.
+DATABASE_PORT = ''             # Set to empty string for default. Not used with sqlite3.
 
-SURVEX_DATA = 'c:\\loser\\'
-CAVERN = '"C:\\Program Files\\Survex\\cavern"'
-EXPOWEB = 'C:\\expoweb\\'
+SURVEX_DATA = 'c:\\Expo\\loser\\'
+CAVERN = 'cavern'
+EXPOWEB = 'C:\\Expo\\expoweb\\'
+SURVEYS = 'E:\\surveys\\'
+SURVEY_SCANS = 'E:\\surveys\\surveyscans'
+
+LOGFILE = open(EXPOWEB+'troggle\\parsing_log.txt',"a+b")
+
+PHOTOS = 'C:\\Expo\\expoweb\\photos'
+
+URL_ROOT = 'http://127.0.0.1:8000'
 
 PYTHON_PATH = 'C:\\expoweb\\troggle\\'
 
-URL_ROOT = "/"
+MEDIA_ROOT = 'C:/Expo/expoweb/troggle/media/'
+
+#FILES = "http://framos.lawoftheland.co.uk/troggle/survey_files/"
+
+EMAIL_HOST = "smtp.gmail.com"
+
+EMAIL_HOST_USER = "cuccexpo@gmail.com"
+
+EMAIL_HOST_PASSWORD = ""
+
+EMAIL_PORT=587
+
+EMAIL_USE_TLS = True
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash if there is a path component (optional in other cases).
+# Examples: "http://media.lawrence.com", "http://example.com/media/"
+
+
+
 
 TEMPLATE_DIRS = (
-    "c:/expoweb/troggle/templates",
+    "C:/Expo/expoweb/troggle/templates",
+    
     # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
     # Always use forward slashes, even on Windows.
     # Don't forget to use absolute paths, not relative paths.

troggle/media/css/main3.css

@@ -180,7 +180,7 @@ table.trad td, table.trad th	{ margin: 0pt; border: 1px solid #aaa;
 
 
 /* You are not expected to understand this. It is necessary. */
-/* The above is the most fucktarded comment I have ever read. AC, 24 APR 2009 */
+/* The above is the most fucktarded comment I have ever read :-) AC, 24 APR 2009 */
 table.centre			{ margin-left: auto; margin-right: auto; }
 table.centre td			{ text-align: left; }
 
@@ -235,7 +235,7 @@ a.redtext:link {
 }
 div.figure {
   width: 20%;
-  border: thin silver solid;
+  border: thin white solid;
   margin: 0.5em;
   padding: 0.5em;
   display: inline;	

troggle/parsers/surveys.py

@@ -1,19 +1,19 @@
 import sys
 import os
 import types
-sys.path.append('C:\\Expo\\expoweb')
-from troggle import *
-os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings'
+#sys.path.append('C:\\Expo\\expoweb')
+#from troggle import *
+#os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings'
 import troggle.settings as settings
 from troggle.expo.models import *
-
+from PIL import Image
 #import settings
 #import expo.models as models
 import csv
 import re
 import datetime
 
-def readSurveysFromCSV():
+def readSurveysFromCSV(logfile=None):
     try:
         surveytab = open(os.path.join(settings.SURVEYS, "Surveys.csv"))
     except IOError:
@@ -29,8 +29,18 @@ def readSurveysFromCSV():
     if Expedition.objects.count()==0:
         print "There are no expeditions in the database. Please run the logbook parser."
         sys.exit()
+
+    if logfile:
+        logfile.write("Deleting all scanned images")
     ScannedImage.objects.all().delete()
+    
+    if logfile:
+        logfile.write("Deleting all survey objects")
     Survey.objects.all().delete()
+    
+    if logfile:
+        logfile.write("Beginning to import surveys from "+str(os.path.join(settings.SURVEYS, "Surveys.csv"))+"\n"+"-"*60+"\n")
+    
     for survey in surveyreader:
         walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']]) #I hate this, but some surveys have a letter eg 2000#34a. This line deals with that.
     #    print walletNumberLetter.groups()
@@ -47,7 +57,9 @@ def readSurveysFromCSV():
             #try and find the sketch_scan
             pass
         surveyobj.save()
-        print "added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r",
+
+        if logfile:
+            logfile.write("added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r")
 
 def listdir(*directories):
     try:
@@ -59,7 +71,7 @@ def listdir(*directories):
         return [folder.rstrip(r"/") for folder in folders]
 
 # add survey scans
-def parseSurveyScans(year):
+def parseSurveyScans(year, logfile=None):
 #    yearFileList = listdir(year.year)
     yearPath=os.path.join(settings.SURVEY_SCANS, year.year)
     yearFileList=os.listdir(yearPath)
@@ -92,17 +104,27 @@ def parseSurveyScans(year):
                 survey=Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=year)[0]
             except Survey.MultipleObjectsReturned:
                 survey=Survey.objects.filter(wallet_number=surveyNumber, expedition=year)[0]
-               
+            file=os.path.join(year.year, surveyFolder, scan)
             scanObj = ScannedImage(
-                file=os.path.join(year.year, surveyFolder, scan),
+                file=file,
                 contents=scanType,
                 number_in_wallet=scanNumber,
-                survey=survey
+                survey=survey,
+                new_since_parsing=False,
                 )
             #print "Added scanned image at " + str(scanObj)
+            if scanFormat=="png":
+                if isInterlacedPNG(os.path.join(settings.SURVEY_SCANS,file)):
+                    print file + "is an interlaced PNG. No can do."
+                continue
             scanObj.save()
 
-def parseSurveys():
+def parseSurveys(logfile=None):
     readSurveysFromCSV()                
     for year in Expedition.objects.filter(year__gte=2000):   #expos since 2000, because paths and filenames were nonstandard before then
         parseSurveyScans(year)
+
+def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the thumbnail engine can't handle them (uses PIL)
+    file=Image.open(filePath)
+    return file.info['interlace']
+    

troggle/settings.py

@@ -78,10 +78,11 @@ INSTALLED_APPS = (
     'django.contrib.sessions',
     'django.contrib.sites',
     'django.contrib.redirects',
-    #'photologue',
+    #'troggle.photologue',
     #'troggle.reversion',
-    #'django_evolution',
+    'django_evolution',
     'troggle.registration',
     'troggle.profiles',
-    'troggle.expo'
+    'troggle.expo',
+    
 )

troggle/templates/base.html

@@ -19,7 +19,7 @@
     {% if user.username %}
     You are logged in as {{ user.username }} 
     	{% if user.person %}(<a href="{{ user.person.get_absolute_url }}">{{ user.person }}</a>)
-        	{% else %} <a href="{% url profiles_create_profile %}">sort your profile</a>
+        	{% else %} <a href={% url profiles_select_profile %}>sort your profile</a>
         {% endif %}.
     | <a href="{% url auth_logout %}">Log out</a> {% else %} <a href="{% url registration_register %}">Sign up</a> | <a href="{% url auth_login %}">Log in</a> {% endif %}
     {% endblock%}

troggle/templates/controlPanel.html

@@ -13,7 +13,7 @@
 <table>
 <tr><td>caves from cavetab2.csv using parsers\cavetab.py</td><td> <input type="checkbox" class="parser" name="import_cavetab"/></td></tr>
 <tr><td>logbook entries using parsers\logbooks.py</td><td><input type="checkbox" name="import_logbooks"/></td></tr>
-<tr><td>people from folk.csv using parsers\people.py</td><td><input type="checkbox" name="import_folk"/></td></tr>
+<tr><td>people from folk.csv using parsers\people.py</td><td><input type="checkbox" name="import_people"/></td></tr>
 <tr><td>QMs using parsers\QMs.py</td><td><input type="checkbox" name="QMs" value="import_QMs" /></td></tr>
 <tr><td>survey scans using parsers\surveys.py</td><td><input type="checkbox" name="import_surveys" /></td></tr>
 <tr><td>survex data using parsers\survex.py</td><td><input type="checkbox" name="survex" value="import_survex" /></td></tr>

troggle/templates/person.html

@@ -11,13 +11,13 @@
 
 {% block content %}
 {% if person.blurb %}
-{{person.blurb}}
+{{person.blurb|safe}}
 {% endif %}
 
 {% for pic in person.photo_set.all %}
 {% if pic.is_mugshot %}
     <div class="figure">
-      <p> <img src="{{ pic.file.url }}" class="thumbnail" />
+      <p> <img src="{{ pic.thumbnail_image.url }}" class="thumbnail" />
       <p> {{ pic.caption }}
       </p>
       </p>

troggle/templates/survey.html

@@ -179,11 +179,11 @@
 		<center>[ There are no surveys in the database for this year. Put link in to add one. ]</center>
 	{% endif %}
   </div>
-  <div id="notesContent" class="behind">
+  <div id="notesContent" class="behind" style="overflow: auto">
     <h3>Scanned notes for {{ current_survey }}.</h3>
     {% for noteItem in notes %}
     <div class="figure">
-      <p> <img src="{{ noteItem.correctURL }}" class="thumbnail">
+      <p> <img src="{{ noteItem.thumbnail_image.url }}" class="thumbnail">
       <p> File at: <a href="{{ noteItem.correctURL }}"> {{ noteItem.file.name }} </a> <br />
         Scanned by: {{ noteItem.scanned_by }} <br />
         On: {{ noteItem.scanned_on }} <br />
@@ -201,8 +201,8 @@
     <h3>Scanned plan sketch files for {{ current_survey }}.</h3>
     {% for sketchItem in planSketches %}
     <div class="figure">
-      <p> <img src="{{ sketchItem.correctURL }}" class="thumbnail" />
-      <p> File at: <a href="{{ sketchItem.correctUrl }}"> {{ sketchItem.file.name }} </a> <br />
+      <p> <img src="{{ sketchItem.thumbnail_image.url }}" class="thumbnail" />
+      <p> File at: <a href="{{ sketchItem.correctURL }}"> {{ sketchItem.file.name }} </a> <br />
         Scanned by: {{ sketchItem.scanned_by }} <br />
         On: {{ sketchItem.scanned_on }} <br />
       </p>

troggle/urls.py

@@ -68,6 +68,8 @@ urlpatterns = patterns('',
     
 #    (r'^personform/(.*)$', personForm),
 
+    (r'^photologue/', include('photologue.urls')),
+
     (r'^site_media/(?P<path>.*)$', 'django.views.static.serve',
         {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}),