From 2596e27e7d1bdd36675bbea882ecbedc67e06951 Mon Sep 17 00:00:00 2001
From: cucc <devnull@localhost>
Date: Mon, 11 May 2009 01:05:33 +0200
Subject: [PATCH] [svn r8336] Dynamic thumbnail generation for photos and
survey scans using imagekit, further improving registration system, other
misc.
---
troggle/expo/imagekit_specs.py | 23 +++++++++++++
troggle/expo/models.py | 27 +++++++++++++---
troggle/localsettingswindows.py | 50 ++++++++++++++++++++++-------
troggle/media/css/main3.css | 4 +--
troggle/parsers/surveys.py | 44 ++++++++++++++++++-------
troggle/settings.py | 7 ++--
troggle/templates/base.html | 2 +-
troggle/templates/controlPanel.html | 2 +-
troggle/templates/person.html | 4 +--
troggle/templates/survey.html | 8 ++---
troggle/urls.py | 2 ++
11 files changed, 134 insertions(+), 39 deletions(-)
create mode 100644 troggle/expo/imagekit_specs.py
diff --git a/troggle/expo/imagekit_specs.py b/troggle/expo/imagekit_specs.py
new file mode 100644
index 000000000..243cb9f76
--- /dev/null
+++ b/troggle/expo/imagekit_specs.py
@@ -0,0 +1,23 @@
+from imagekit.specs import ImageSpec
+from imagekit import processors
+
+class ResizeThumb(processors.Resize):
+ width = 100
+ height = 75
+ crop = True
+
+class ResizeDisplay(processors.Resize):
+ width = 600
+
+class EnhanceThumb(processors.Adjustment):
+ contrast = 1.2
+ sharpness = 1.1
+
+class Thumbnail(ImageSpec):
+ access_as = 'thumbnail_image'
+ pre_cache = True
+ processors = [ResizeThumb, EnhanceThumb]
+
+class Display(ImageSpec):
+ increment_count = True
+ processors = [ResizeDisplay]
diff --git a/troggle/expo/models.py b/troggle/expo/models.py
index b7ad2d756..a3953d93c 100644
--- a/troggle/expo/models.py
+++ b/troggle/expo/models.py
@@ -11,6 +11,7 @@ from django.conf import settings
import datetime
from decimal import Decimal, getcontext
from django.core.urlresolvers import reverse
+from imagekit.models import ImageModel
getcontext().prec=2 #use 2 significant figures for decimal calculations
from models_survex import *
@@ -25,6 +26,15 @@ class TroggleModel(models.Model):
class Meta:
abstract = True
+class TroggleImageModel(ImageModel):
+ new_since_parsing = models.BooleanField(default=False, editable=False)
+
+ def get_admin_url(self):
+ return settings.URL_ROOT + "/admin/expo/" + self._meta.object_name.lower() + "/" + str(self.pk)
+
+ class Meta:
+ abstract = True
+
class Expedition(TroggleModel):
year = models.CharField(max_length=20, unique=True)
name = models.CharField(max_length=100)
@@ -517,7 +527,7 @@ class QM(TroggleModel):
return res
photoFileStorage = FileSystemStorage(location=settings.PHOTOS_ROOT, base_url=settings.PHOTOS_URL)
-class Photo(TroggleModel):
+class Photo(TroggleImageModel):
caption = models.CharField(max_length=1000,blank=True,null=True)
contains_logbookentry = models.ForeignKey(LogbookEntry,blank=True,null=True)
contains_person = models.ManyToManyField(Person,blank=True,null=True)
@@ -527,10 +537,14 @@ class Photo(TroggleModel):
contains_entrance = models.ForeignKey(Entrance, related_name="photo_file",blank=True,null=True)
nearest_survey_point = models.ForeignKey(SurveyStation,blank=True,null=True)
nearest_QM = models.ForeignKey(QM,blank=True,null=True)
-
-
lon_utm = models.FloatField(blank=True,null=True)
lat_utm = models.FloatField(blank=True,null=True)
+
+ class IKOptions:
+ spec_module = 'expo.imagekit_specs'
+ cache_dir = 'thumbs'
+ image_field = 'file'
+
#content_type = models.ForeignKey(ContentType)
#object_id = models.PositiveIntegerField()
#location = generic.GenericForeignKey('content_type', 'object_id')
@@ -545,7 +559,7 @@ def get_scan_path(instance, filename):
number="%02d" % instance.survey.wallet_number + str(instance.survey.wallet_letter) #using %02d string formatting because convention was 2009#01
return os.path.join('./',year,year+r'#'+number,instance.contents+str(instance.number_in_wallet)+r'.jpg')
-class ScannedImage(TroggleModel):
+class ScannedImage(TroggleImageModel):
file = models.ImageField(storage=scansFileStorage, upload_to=get_scan_path)
scanned_by = models.ForeignKey(Person,blank=True, null=True)
scanned_on = models.DateField(null=True)
@@ -554,6 +568,11 @@ class ScannedImage(TroggleModel):
number_in_wallet = models.IntegerField(null=True)
lon_utm = models.FloatField(blank=True,null=True)
lat_utm = models.FloatField(blank=True,null=True)
+
+ class IKOptions:
+ spec_module = 'expo.imagekit_specs'
+ cache_dir = 'thumbs'
+ image_field = 'file'
#content_type = models.ForeignKey(ContentType)
#object_id = models.PositiveIntegerField()
#location = generic.GenericForeignKey('content_type', 'object_id')
diff --git a/troggle/localsettingswindows.py b/troggle/localsettingswindows.py
index a264e8d01..53225b2a1 100644
--- a/troggle/localsettingswindows.py
+++ b/troggle/localsettingswindows.py
@@ -1,20 +1,48 @@
-DATABASE_ENGINE = 'mysql' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
-DATABASE_NAME = 'troggle' # Or path to database file if using sqlite3.
-DATABASE_USER = 'troggle' # Not used with sqlite3.
-DATABASE_PASSWORD = 'troggle' # Not used with sqlite3.
-DATABASE_HOST = 'localhost' # Set to empty string for localhost. Not used with sqlite3.
-DATABASE_PORT = '3306' # Set to empty string for default. Not used with sqlite3.
+DATABASE_ENGINE = '' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
+DATABASE_NAME = '' # Or path to database file if using sqlite3.
+DATABASE_USER = '' # Not used with sqlite3.
+DATABASE_PASSWORD = '' # Not used with sqlite3.
+DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3.
+DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3.
-SURVEX_DATA = 'c:\\loser\\'
-CAVERN = '"C:\\Program Files\\Survex\\cavern"'
-EXPOWEB = 'C:\\expoweb\\'
+SURVEX_DATA = 'c:\\Expo\\loser\\'
+CAVERN = 'cavern'
+EXPOWEB = 'C:\\Expo\\expoweb\\'
+SURVEYS = 'E:\\surveys\\'
+SURVEY_SCANS = 'E:\\surveys\\surveyscans'
+
+LOGFILE = open(EXPOWEB+'troggle\\parsing_log.txt',"a+b")
+
+PHOTOS = 'C:\\Expo\\expoweb\\photos'
+
+URL_ROOT = 'http://127.0.0.1:8000'
PYTHON_PATH = 'C:\\expoweb\\troggle\\'
-URL_ROOT = "/"
+MEDIA_ROOT = 'C:/Expo/expoweb/troggle/media/'
+
+#FILES = "http://framos.lawoftheland.co.uk/troggle/survey_files/"
+
+EMAIL_HOST = "smtp.gmail.com"
+
+EMAIL_HOST_USER = "cuccexpo@gmail.com"
+
+EMAIL_HOST_PASSWORD = ""
+
+EMAIL_PORT=587
+
+EMAIL_USE_TLS = True
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash if there is a path component (optional in other cases).
+# Examples: "http://media.lawrence.com", "http://example.com/media/"
+
+
+
TEMPLATE_DIRS = (
- "c:/expoweb/troggle/templates",
+ "C:/Expo/expoweb/troggle/templates",
+
# Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
diff --git a/troggle/media/css/main3.css b/troggle/media/css/main3.css
index 4dbac3237..d875da9e3 100644
--- a/troggle/media/css/main3.css
+++ b/troggle/media/css/main3.css
@@ -180,7 +180,7 @@ table.trad td, table.trad th { margin: 0pt; border: 1px solid #aaa;
/* You are not expected to understand this. It is necessary. */
-/* The above is the most fucktarded comment I have ever read. AC, 24 APR 2009 */
+/* The above is the most fucktarded comment I have ever read :-) AC, 24 APR 2009 */
table.centre { margin-left: auto; margin-right: auto; }
table.centre td { text-align: left; }
@@ -235,7 +235,7 @@ a.redtext:link {
}
div.figure {
width: 20%;
- border: thin silver solid;
+ border: thin white solid;
margin: 0.5em;
padding: 0.5em;
display: inline;
diff --git a/troggle/parsers/surveys.py b/troggle/parsers/surveys.py
index c7c666d1a..e2a502d2e 100644
--- a/troggle/parsers/surveys.py
+++ b/troggle/parsers/surveys.py
@@ -1,19 +1,19 @@
import sys
import os
import types
-sys.path.append('C:\\Expo\\expoweb')
-from troggle import *
-os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings'
+#sys.path.append('C:\\Expo\\expoweb')
+#from troggle import *
+#os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings'
import troggle.settings as settings
from troggle.expo.models import *
-
+from PIL import Image
#import settings
#import expo.models as models
import csv
import re
import datetime
-def readSurveysFromCSV():
+def readSurveysFromCSV(logfile=None):
try:
surveytab = open(os.path.join(settings.SURVEYS, "Surveys.csv"))
except IOError:
@@ -29,8 +29,18 @@ def readSurveysFromCSV():
if Expedition.objects.count()==0:
print "There are no expeditions in the database. Please run the logbook parser."
sys.exit()
+
+ if logfile:
+ logfile.write("Deleting all scanned images")
ScannedImage.objects.all().delete()
+
+ if logfile:
+ logfile.write("Deleting all survey objects")
Survey.objects.all().delete()
+
+ if logfile:
+ logfile.write("Beginning to import surveys from "+str(os.path.join(settings.SURVEYS, "Surveys.csv"))+"\n"+"-"*60+"\n")
+
for survey in surveyreader:
walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']]) #I hate this, but some surveys have a letter eg 2000#34a. This line deals with that.
# print walletNumberLetter.groups()
@@ -47,7 +57,9 @@ def readSurveysFromCSV():
#try and find the sketch_scan
pass
surveyobj.save()
- print "added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r",
+
+ if logfile:
+ logfile.write("added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r")
def listdir(*directories):
try:
@@ -59,7 +71,7 @@ def listdir(*directories):
return [folder.rstrip(r"/") for folder in folders]
# add survey scans
-def parseSurveyScans(year):
+def parseSurveyScans(year, logfile=None):
# yearFileList = listdir(year.year)
yearPath=os.path.join(settings.SURVEY_SCANS, year.year)
yearFileList=os.listdir(yearPath)
@@ -92,17 +104,27 @@ def parseSurveyScans(year):
survey=Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=year)[0]
except Survey.MultipleObjectsReturned:
survey=Survey.objects.filter(wallet_number=surveyNumber, expedition=year)[0]
-
+ file=os.path.join(year.year, surveyFolder, scan)
scanObj = ScannedImage(
- file=os.path.join(year.year, surveyFolder, scan),
+ file=file,
contents=scanType,
number_in_wallet=scanNumber,
- survey=survey
+ survey=survey,
+ new_since_parsing=False,
)
#print "Added scanned image at " + str(scanObj)
+ if scanFormat=="png":
+ if isInterlacedPNG(os.path.join(settings.SURVEY_SCANS,file)):
+ print file + "is an interlaced PNG. No can do."
+ continue
scanObj.save()
-def parseSurveys():
+def parseSurveys(logfile=None):
readSurveysFromCSV()
for year in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then
parseSurveyScans(year)
+
+def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the thumbnail engine can't handle them (uses PIL)
+ file=Image.open(filePath)
+ return file.info['interlace']
+
\ No newline at end of file
diff --git a/troggle/settings.py b/troggle/settings.py
index 11b1f2fb5..1e578db38 100644
--- a/troggle/settings.py
+++ b/troggle/settings.py
@@ -78,10 +78,11 @@ INSTALLED_APPS = (
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.redirects',
- #'photologue',
+ #'troggle.photologue',
#'troggle.reversion',
- #'django_evolution',
+ 'django_evolution',
'troggle.registration',
'troggle.profiles',
- 'troggle.expo'
+ 'troggle.expo',
+
)
diff --git a/troggle/templates/base.html b/troggle/templates/base.html
index c44212fb6..2a8e758e8 100644
--- a/troggle/templates/base.html
+++ b/troggle/templates/base.html
@@ -19,7 +19,7 @@
{% if user.username %}
You are logged in as {{ user.username }}
{% if user.person %}(<a href="{{ user.person.get_absolute_url }}">{{ user.person }}</a>)
- {% else %} <a href="{% url profiles_create_profile %}">sort your profile</a>
+ {% else %} <a href={% url profiles_select_profile %}>sort your profile</a>
{% endif %}.
| <a href="{% url auth_logout %}">Log out</a> {% else %} <a href="{% url registration_register %}">Sign up</a> | <a href="{% url auth_login %}">Log in</a> {% endif %}
{% endblock%}
diff --git a/troggle/templates/controlPanel.html b/troggle/templates/controlPanel.html
index 3fad8e61b..d60a32731 100644
--- a/troggle/templates/controlPanel.html
+++ b/troggle/templates/controlPanel.html
@@ -13,7 +13,7 @@
<table>
<tr><td>caves from cavetab2.csv using parsers\cavetab.py</td><td> <input type="checkbox" class="parser" name="import_cavetab"/></td></tr>
<tr><td>logbook entries using parsers\logbooks.py</td><td><input type="checkbox" name="import_logbooks"/></td></tr>
-<tr><td>people from folk.csv using parsers\people.py</td><td><input type="checkbox" name="import_folk"/></td></tr>
+<tr><td>people from folk.csv using parsers\people.py</td><td><input type="checkbox" name="import_people"/></td></tr>
<tr><td>QMs using parsers\QMs.py</td><td><input type="checkbox" name="QMs" value="import_QMs" /></td></tr>
<tr><td>survey scans using parsers\surveys.py</td><td><input type="checkbox" name="import_surveys" /></td></tr>
<tr><td>survex data using parsers\survex.py</td><td><input type="checkbox" name="survex" value="import_survex" /></td></tr>
diff --git a/troggle/templates/person.html b/troggle/templates/person.html
index d28f82a11..adc6aba97 100644
--- a/troggle/templates/person.html
+++ b/troggle/templates/person.html
@@ -11,13 +11,13 @@
{% block content %}
{% if person.blurb %}
-{{person.blurb}}
+{{person.blurb|safe}}
{% endif %}
{% for pic in person.photo_set.all %}
{% if pic.is_mugshot %}
<div class="figure">
- <p> <img src="{{ pic.file.url }}" class="thumbnail" />
+ <p> <img src="{{ pic.thumbnail_image.url }}" class="thumbnail" />
<p> {{ pic.caption }}
</p>
</p>
diff --git a/troggle/templates/survey.html b/troggle/templates/survey.html
index fcdf65e92..a0395b6a4 100644
--- a/troggle/templates/survey.html
+++ b/troggle/templates/survey.html
@@ -179,11 +179,11 @@
<center>[ There are no surveys in the database for this year. Put link in to add one. ]</center>
{% endif %}
</div>
- <div id="notesContent" class="behind">
+ <div id="notesContent" class="behind" style="overflow: auto">
<h3>Scanned notes for {{ current_survey }}.</h3>
{% for noteItem in notes %}
<div class="figure">
- <p> <img src="{{ noteItem.correctURL }}" class="thumbnail">
+ <p> <img src="{{ noteItem.thumbnail_image.url }}" class="thumbnail">
<p> File at: <a href="{{ noteItem.correctURL }}"> {{ noteItem.file.name }} </a> <br />
Scanned by: {{ noteItem.scanned_by }} <br />
On: {{ noteItem.scanned_on }} <br />
@@ -201,8 +201,8 @@
<h3>Scanned plan sketch files for {{ current_survey }}.</h3>
{% for sketchItem in planSketches %}
<div class="figure">
- <p> <img src="{{ sketchItem.correctURL }}" class="thumbnail" />
- <p> File at: <a href="{{ sketchItem.correctUrl }}"> {{ sketchItem.file.name }} </a> <br />
+ <p> <img src="{{ sketchItem.thumbnail_image.url }}" class="thumbnail" />
+ <p> File at: <a href="{{ sketchItem.correctURL }}"> {{ sketchItem.file.name }} </a> <br />
Scanned by: {{ sketchItem.scanned_by }} <br />
On: {{ sketchItem.scanned_on }} <br />
</p>
diff --git a/troggle/urls.py b/troggle/urls.py
index 4d39f84c7..780fdb0b2 100644
--- a/troggle/urls.py
+++ b/troggle/urls.py
@@ -68,6 +68,8 @@ urlpatterns = patterns('',
# (r'^personform/(.*)$', personForm),
+ (r'^photologue/', include('photologue.urls')),
+
(r'^site_media/(?P<path>.*)$', 'django.views.static.serve',
{'document_root': settings.MEDIA_ROOT, 'show_indexes': True}),