From 8c68a8a0d749327eed5431a04f03f5ec4472969a Mon Sep 17 00:00:00 2001 From: substantialnoninfringinguser Date: Wed, 13 May 2009 06:23:57 +0100 Subject: [PATCH] [svn] Dynamic thumbnail generation for photos and survey scans using imagekit, further improving registration system, other misc. Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8336 by cucc @ 5/10/2009 11:05 PM --- expo/imagekit_specs.py | 23 +++++++++++++++++ expo/models.py | 27 +++++++++++++++++--- localsettingswindows.py | 50 +++++++++++++++++++++++++++++-------- media/css/main3.css | 4 +-- parsers/surveys.py | 44 ++++++++++++++++++++++++-------- settings.py | 7 +++--- templates/base.html | 2 +- templates/controlPanel.html | 2 +- templates/person.html | 4 +-- templates/survey.html | 8 +++--- urls.py | 2 ++ 11 files changed, 134 insertions(+), 39 deletions(-) create mode 100644 expo/imagekit_specs.py diff --git a/expo/imagekit_specs.py b/expo/imagekit_specs.py new file mode 100644 index 0000000..243cb9f --- /dev/null +++ b/expo/imagekit_specs.py @@ -0,0 +1,23 @@ +from imagekit.specs import ImageSpec +from imagekit import processors + +class ResizeThumb(processors.Resize): + width = 100 + height = 75 + crop = True + +class ResizeDisplay(processors.Resize): + width = 600 + +class EnhanceThumb(processors.Adjustment): + contrast = 1.2 + sharpness = 1.1 + +class Thumbnail(ImageSpec): + access_as = 'thumbnail_image' + pre_cache = True + processors = [ResizeThumb, EnhanceThumb] + +class Display(ImageSpec): + increment_count = True + processors = [ResizeDisplay] diff --git a/expo/models.py b/expo/models.py index b7ad2d7..a3953d9 100644 --- a/expo/models.py +++ b/expo/models.py @@ -11,6 +11,7 @@ from django.conf import settings import datetime from decimal import Decimal, getcontext from django.core.urlresolvers import reverse +from imagekit.models import ImageModel getcontext().prec=2 #use 2 significant figures for decimal calculations from models_survex import * @@ -25,6 +26,15 @@ class TroggleModel(models.Model): class Meta: abstract = True +class TroggleImageModel(ImageModel): + new_since_parsing = models.BooleanField(default=False, editable=False) + + def get_admin_url(self): + return settings.URL_ROOT + "/admin/expo/" + self._meta.object_name.lower() + "/" + str(self.pk) + + class Meta: + abstract = True + class Expedition(TroggleModel): year = models.CharField(max_length=20, unique=True) name = models.CharField(max_length=100) @@ -517,7 +527,7 @@ class QM(TroggleModel): return res photoFileStorage = FileSystemStorage(location=settings.PHOTOS_ROOT, base_url=settings.PHOTOS_URL) -class Photo(TroggleModel): +class Photo(TroggleImageModel): caption = models.CharField(max_length=1000,blank=True,null=True) contains_logbookentry = models.ForeignKey(LogbookEntry,blank=True,null=True) contains_person = models.ManyToManyField(Person,blank=True,null=True) @@ -527,10 +537,14 @@ class Photo(TroggleModel): contains_entrance = models.ForeignKey(Entrance, related_name="photo_file",blank=True,null=True) nearest_survey_point = models.ForeignKey(SurveyStation,blank=True,null=True) nearest_QM = models.ForeignKey(QM,blank=True,null=True) - - lon_utm = models.FloatField(blank=True,null=True) lat_utm = models.FloatField(blank=True,null=True) + + class IKOptions: + spec_module = 'expo.imagekit_specs' + cache_dir = 'thumbs' + image_field = 'file' + #content_type = models.ForeignKey(ContentType) #object_id = models.PositiveIntegerField() #location = generic.GenericForeignKey('content_type', 'object_id') @@ -545,7 +559,7 @@ def get_scan_path(instance, filename): number="%02d" % instance.survey.wallet_number + str(instance.survey.wallet_letter) #using %02d string formatting because convention was 2009#01 return os.path.join('./',year,year+r'#'+number,instance.contents+str(instance.number_in_wallet)+r'.jpg') -class ScannedImage(TroggleModel): +class ScannedImage(TroggleImageModel): file = models.ImageField(storage=scansFileStorage, upload_to=get_scan_path) scanned_by = models.ForeignKey(Person,blank=True, null=True) scanned_on = models.DateField(null=True) @@ -554,6 +568,11 @@ class ScannedImage(TroggleModel): number_in_wallet = models.IntegerField(null=True) lon_utm = models.FloatField(blank=True,null=True) lat_utm = models.FloatField(blank=True,null=True) + + class IKOptions: + spec_module = 'expo.imagekit_specs' + cache_dir = 'thumbs' + image_field = 'file' #content_type = models.ForeignKey(ContentType) #object_id = models.PositiveIntegerField() #location = generic.GenericForeignKey('content_type', 'object_id') diff --git a/localsettingswindows.py b/localsettingswindows.py index a264e8d..53225b2 100644 --- a/localsettingswindows.py +++ b/localsettingswindows.py @@ -1,20 +1,48 @@ -DATABASE_ENGINE = 'mysql' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. -DATABASE_NAME = 'troggle' # Or path to database file if using sqlite3. -DATABASE_USER = 'troggle' # Not used with sqlite3. -DATABASE_PASSWORD = 'troggle' # Not used with sqlite3. -DATABASE_HOST = 'localhost' # Set to empty string for localhost. Not used with sqlite3. -DATABASE_PORT = '3306' # Set to empty string for default. Not used with sqlite3. +DATABASE_ENGINE = '' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. +DATABASE_NAME = '' # Or path to database file if using sqlite3. +DATABASE_USER = '' # Not used with sqlite3. +DATABASE_PASSWORD = '' # Not used with sqlite3. +DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. +DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. -SURVEX_DATA = 'c:\\loser\\' -CAVERN = '"C:\\Program Files\\Survex\\cavern"' -EXPOWEB = 'C:\\expoweb\\' +SURVEX_DATA = 'c:\\Expo\\loser\\' +CAVERN = 'cavern' +EXPOWEB = 'C:\\Expo\\expoweb\\' +SURVEYS = 'E:\\surveys\\' +SURVEY_SCANS = 'E:\\surveys\\surveyscans' + +LOGFILE = open(EXPOWEB+'troggle\\parsing_log.txt',"a+b") + +PHOTOS = 'C:\\Expo\\expoweb\\photos' + +URL_ROOT = 'http://127.0.0.1:8000' PYTHON_PATH = 'C:\\expoweb\\troggle\\' -URL_ROOT = "/" +MEDIA_ROOT = 'C:/Expo/expoweb/troggle/media/' + +#FILES = "http://framos.lawoftheland.co.uk/troggle/survey_files/" + +EMAIL_HOST = "smtp.gmail.com" + +EMAIL_HOST_USER = "cuccexpo@gmail.com" + +EMAIL_HOST_PASSWORD = "" + +EMAIL_PORT=587 + +EMAIL_USE_TLS = True + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash if there is a path component (optional in other cases). +# Examples: "http://media.lawrence.com", "http://example.com/media/" + + + TEMPLATE_DIRS = ( - "c:/expoweb/troggle/templates", + "C:/Expo/expoweb/troggle/templates", + # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". # Always use forward slashes, even on Windows. # Don't forget to use absolute paths, not relative paths. diff --git a/media/css/main3.css b/media/css/main3.css index 4dbac32..d875da9 100644 --- a/media/css/main3.css +++ b/media/css/main3.css @@ -180,7 +180,7 @@ table.trad td, table.trad th { margin: 0pt; border: 1px solid #aaa; /* You are not expected to understand this. It is necessary. */ -/* The above is the most fucktarded comment I have ever read. AC, 24 APR 2009 */ +/* The above is the most fucktarded comment I have ever read :-) AC, 24 APR 2009 */ table.centre { margin-left: auto; margin-right: auto; } table.centre td { text-align: left; } @@ -235,7 +235,7 @@ a.redtext:link { } div.figure { width: 20%; - border: thin silver solid; + border: thin white solid; margin: 0.5em; padding: 0.5em; display: inline; diff --git a/parsers/surveys.py b/parsers/surveys.py index c7c666d..e2a502d 100644 --- a/parsers/surveys.py +++ b/parsers/surveys.py @@ -1,19 +1,19 @@ import sys import os import types -sys.path.append('C:\\Expo\\expoweb') -from troggle import * -os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings' +#sys.path.append('C:\\Expo\\expoweb') +#from troggle import * +#os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings' import troggle.settings as settings from troggle.expo.models import * - +from PIL import Image #import settings #import expo.models as models import csv import re import datetime -def readSurveysFromCSV(): +def readSurveysFromCSV(logfile=None): try: surveytab = open(os.path.join(settings.SURVEYS, "Surveys.csv")) except IOError: @@ -29,8 +29,18 @@ def readSurveysFromCSV(): if Expedition.objects.count()==0: print "There are no expeditions in the database. Please run the logbook parser." sys.exit() + + if logfile: + logfile.write("Deleting all scanned images") ScannedImage.objects.all().delete() + + if logfile: + logfile.write("Deleting all survey objects") Survey.objects.all().delete() + + if logfile: + logfile.write("Beginning to import surveys from "+str(os.path.join(settings.SURVEYS, "Surveys.csv"))+"\n"+"-"*60+"\n") + for survey in surveyreader: walletNumberLetter = re.match(r'(?P\d*)(?P[a-zA-Z]*)',survey[header['Survey Number']]) #I hate this, but some surveys have a letter eg 2000#34a. This line deals with that. # print walletNumberLetter.groups() @@ -47,7 +57,9 @@ def readSurveysFromCSV(): #try and find the sketch_scan pass surveyobj.save() - print "added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r", + + if logfile: + logfile.write("added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r") def listdir(*directories): try: @@ -59,7 +71,7 @@ def listdir(*directories): return [folder.rstrip(r"/") for folder in folders] # add survey scans -def parseSurveyScans(year): +def parseSurveyScans(year, logfile=None): # yearFileList = listdir(year.year) yearPath=os.path.join(settings.SURVEY_SCANS, year.year) yearFileList=os.listdir(yearPath) @@ -92,17 +104,27 @@ def parseSurveyScans(year): survey=Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=year)[0] except Survey.MultipleObjectsReturned: survey=Survey.objects.filter(wallet_number=surveyNumber, expedition=year)[0] - + file=os.path.join(year.year, surveyFolder, scan) scanObj = ScannedImage( - file=os.path.join(year.year, surveyFolder, scan), + file=file, contents=scanType, number_in_wallet=scanNumber, - survey=survey + survey=survey, + new_since_parsing=False, ) #print "Added scanned image at " + str(scanObj) + if scanFormat=="png": + if isInterlacedPNG(os.path.join(settings.SURVEY_SCANS,file)): + print file + "is an interlaced PNG. No can do." + continue scanObj.save() -def parseSurveys(): +def parseSurveys(logfile=None): readSurveysFromCSV() for year in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then parseSurveyScans(year) + +def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the thumbnail engine can't handle them (uses PIL) + file=Image.open(filePath) + return file.info['interlace'] + \ No newline at end of file diff --git a/settings.py b/settings.py index 11b1f2f..1e578db 100644 --- a/settings.py +++ b/settings.py @@ -78,10 +78,11 @@ INSTALLED_APPS = ( 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.redirects', - #'photologue', + #'troggle.photologue', #'troggle.reversion', - #'django_evolution', + 'django_evolution', 'troggle.registration', 'troggle.profiles', - 'troggle.expo' + 'troggle.expo', + ) diff --git a/templates/base.html b/templates/base.html index c44212f..2a8e758 100644 --- a/templates/base.html +++ b/templates/base.html @@ -19,7 +19,7 @@ {% if user.username %} You are logged in as {{ user.username }} {% if user.person %}({{ user.person }}) - {% else %} sort your profile + {% else %} sort your profile {% endif %}. | Log out {% else %} Sign up | Log in {% endif %} {% endblock%} diff --git a/templates/controlPanel.html b/templates/controlPanel.html index 3fad8e6..d60a327 100644 --- a/templates/controlPanel.html +++ b/templates/controlPanel.html @@ -13,7 +13,7 @@ - + diff --git a/templates/person.html b/templates/person.html index d28f82a..adc6aba 100644 --- a/templates/person.html +++ b/templates/person.html @@ -11,13 +11,13 @@ {% block content %} {% if person.blurb %} -{{person.blurb}} +{{person.blurb|safe}} {% endif %} {% for pic in person.photo_set.all %} {% if pic.is_mugshot %}
-

+

{{ pic.caption }}

diff --git a/templates/survey.html b/templates/survey.html index fcdf65e..a0395b6 100644 --- a/templates/survey.html +++ b/templates/survey.html @@ -179,11 +179,11 @@
[ There are no surveys in the database for this year. Put link in to add one. ]
{% endif %}
-
+

Scanned notes for {{ current_survey }}.

{% for noteItem in notes %}
-

+

File at: {{ noteItem.file.name }}
Scanned by: {{ noteItem.scanned_by }}
On: {{ noteItem.scanned_on }}
@@ -201,8 +201,8 @@

Scanned plan sketch files for {{ current_survey }}.

{% for sketchItem in planSketches %}
-

-

File at: {{ sketchItem.file.name }}
+

+

File at: {{ sketchItem.file.name }}
Scanned by: {{ sketchItem.scanned_by }}
On: {{ sketchItem.scanned_on }}

diff --git a/urls.py b/urls.py index 4d39f84..780fdb0 100644 --- a/urls.py +++ b/urls.py @@ -68,6 +68,8 @@ urlpatterns = patterns('', # (r'^personform/(.*)$', personForm), + (r'^photologue/', include('photologue.urls')), + (r'^site_media/(?P.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}),
caves from cavetab2.csv using parsers\cavetab.py
logbook entries using parsers\logbooks.py
people from folk.csv using parsers\people.py
people from folk.csv using parsers\people.py
QMs using parsers\QMs.py
survey scans using parsers\surveys.py
survex data using parsers\survex.py