expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-03-03 08:59:36 +00:00
Code Issues18 Projects3 Activity

[svn] Fixes to deal with reorganization of expo surveys repository. Now that survey scans and Surveys.csv are in different directories, we have two settings variables, settings.SURVEYS for the root of the survey repo, and settings.SURVEY_SCANS for the surveyscans directory.

Browse Source 
Fixed tab / indent muck in surveys parser. Commented out some "file abstraction" stuff for the time being.
Copied from http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8335 by cucc @ 5/10/2009 7:26 AM
This commit is contained in:
substantialnoninfringinguser 2009-05-13 06:22:53 +01:00
parent 1b06243dab
commit 21204f1bc8
4 changed files with 54 additions and 46 deletions

1
expo/admin.py

@ -24,6 +24,7 @@ class ScannedImageInline(admin.TabularInline):
class SurveyAdmin(TroggleModelAdmin): class SurveyAdmin(TroggleModelAdmin):
inlines = (ScannedImageInline,) inlines = (ScannedImageInline,)
search_fields = ('expedition__year','wallet_number')
class QMInline(admin.TabularInline): class QMInline(admin.TabularInline):
model=QM model=QM

2
expo/models.py

@ -538,7 +538,7 @@ class Photo(TroggleModel):
def __str__(self): def __str__(self):
return self.caption return self.caption
scansFileStorage = FileSystemStorage(location=settings.SURVEYS, base_url=settings.SURVEYS_URL) scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL)
def get_scan_path(instance, filename): def get_scan_path(instance, filename):
year=instance.survey.expedition.year year=instance.survey.expedition.year
print "WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number print "WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number

95
parsers/surveys.py

@ -5,7 +5,7 @@ sys.path.append('C:\\Expo\\expoweb')
from troggle import * from troggle import *
os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings' os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings'
import troggle.settings as settings import troggle.settings as settings
import troggle.expo.models as models from troggle.expo.models import *
#import settings #import settings
#import expo.models as models #import expo.models as models
@ -13,40 +13,41 @@ import csv
import re import re
import datetime import datetime
try: def readSurveysFromCSV():
surveytab = open(os.path.join(settings.SURVEYS, "Surveys.csv")) try:
except IOError: surveytab = open(os.path.join(settings.SURVEYS, "Surveys.csv"))
import cStringIO, urllib except IOError:
surveytab = cStringIO.StringIO(urllib.urlopen(settings.SURVEYS + "download/Surveys.csv").read()) import cStringIO, urllib
dialect=csv.Sniffer().sniff(surveytab.read()) surveytab = cStringIO.StringIO(urllib.urlopen(settings.SURVEYS + "download/Surveys.csv").read())
surveytab.seek(0,0) dialect=csv.Sniffer().sniff(surveytab.read())
surveyreader = csv.reader(surveytab,dialect=dialect) surveytab.seek(0,0)
headers = surveyreader.next() surveyreader = csv.reader(surveytab,dialect=dialect)
header = dict(zip(headers, range(len(headers)))) #set up a dictionary where the indexes are header names and the values are column numbers headers = surveyreader.next()
header = dict(zip(headers, range(len(headers)))) #set up a dictionary where the indexes are header names and the values are column numbers
# test if the expeditions have been added yet # test if the expeditions have been added yet
if len(models.Expedition.objects.all())==0: if Expedition.objects.count()==0:
print "There are no expeditions in the database. Please run the logbook parser." print "There are no expeditions in the database. Please run the logbook parser."
sys.exit() sys.exit()
models.ScannedImage.objects.all().delete() ScannedImage.objects.all().delete()
models.Survey.objects.all().delete() Survey.objects.all().delete()
for survey in surveyreader: for survey in surveyreader:
walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']]) #I hate this, but some surveys have a letter eg 2000#34a. This line deals with that. walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']]) #I hate this, but some surveys have a letter eg 2000#34a. This line deals with that.
# print walletNumberLetter.groups() # print walletNumberLetter.groups()
surveyobj = models.Survey( surveyobj = Survey(
expedition = models.Expedition.objects.filter(year=survey[header['Year']])[0], expedition = Expedition.objects.filter(year=survey[header['Year']])[0],
wallet_number = walletNumberLetter.group('number'), wallet_number = walletNumberLetter.group('number'),
comments = survey[header['Comments']], comments = survey[header['Comments']],
location = survey[header['Location']] location = survey[header['Location']]
) )
surveyobj.wallet_letter = walletNumberLetter.group('letter') surveyobj.wallet_letter = walletNumberLetter.group('letter')
if survey[header['Finished']]=='Yes': if survey[header['Finished']]=='Yes':
#try and find the sketch_scan #try and find the sketch_scan
pass pass
surveyobj.save() surveyobj.save()
print "added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r", print "added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r",
def listdir(*directories): def listdir(*directories):
try: try:
@ -59,11 +60,15 @@ def listdir(*directories):
# add survey scans # add survey scans
def parseSurveyScans(year): def parseSurveyScans(year):
yearFileList = listdir(year.year) # yearFileList = listdir(year.year)
yearPath=os.path.join(settings.SURVEY_SCANS, year.year)
yearFileList=os.listdir(yearPath)
print yearFileList
for surveyFolder in yearFileList: for surveyFolder in yearFileList:
try: try:
surveyNumber=re.match(r'\d\d\d\d#0*(\d+)',surveyFolder).groups() surveyNumber=re.match(r'\d\d\d\d#0*(\d+)',surveyFolder).groups()
scanList = listdir(year.year, surveyFolder) # scanList = listdir(year.year, surveyFolder)
scanList=os.listdir(os.path.join(yearPath,surveyFolder))
except AttributeError: except AttributeError:
print surveyFolder + " ignored", print surveyFolder + " ignored",
continue continue
@ -73,10 +78,10 @@ def parseSurveyScans(year):
scanChopped=re.match(r'(?i).*(notes|elev|plan|elevation|extend)(\d*)\.(png|jpg|jpeg)',scan).groups() scanChopped=re.match(r'(?i).*(notes|elev|plan|elevation|extend)(\d*)\.(png|jpg|jpeg)',scan).groups()
scanType,scanNumber,scanFormat=scanChopped scanType,scanNumber,scanFormat=scanChopped
except AttributeError: except AttributeError:
print "Adding scans: " + scan + " ignored \r", print scan + " ignored \r",
continue continue
if scanType == 'elev' or scanType == 'extend': if scanType == 'elev' or scanType == 'extend':
scanType = 'elevation' scanType = 'elevation'
if scanNumber=='': if scanNumber=='':
scanNumber=1 scanNumber=1
@ -84,11 +89,11 @@ def parseSurveyScans(year):
if type(surveyNumber)==types.TupleType: if type(surveyNumber)==types.TupleType:
surveyNumber=surveyNumber[0] surveyNumber=surveyNumber[0]
try: try:
survey=models.Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=year)[0] survey=Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=year)[0]
except models.Survey.MultipleObjectsReturned: except Survey.MultipleObjectsReturned:
survey=models.Survey.objects.filter(wallet_number=surveyNumber, expedition=year)[0] survey=Survey.objects.filter(wallet_number=surveyNumber, expedition=year)[0]
scanObj = models.ScannedImage( scanObj = ScannedImage(
file=os.path.join(year.year, surveyFolder, scan), file=os.path.join(year.year, surveyFolder, scan),
contents=scanType, contents=scanType,
number_in_wallet=scanNumber, number_in_wallet=scanNumber,
@ -96,6 +101,8 @@ def parseSurveyScans(year):
) )
#print "Added scanned image at " + str(scanObj) #print "Added scanned image at " + str(scanObj)
scanObj.save() scanObj.save()
for year in models.Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then def parseSurveys():
parseSurveyScans(year) readSurveysFromCSV()
for year in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then
parseSurveyScans(year)

2
urls.py

@ -76,7 +76,7 @@ urlpatterns = patterns('',
#(r'^survey_files/upload/(?P<path>.*)$', view_surveys.upload), #(r'^survey_files/upload/(?P<path>.*)$', view_surveys.upload),
(r'^survey_scans/(?P<path>.*)$', 'django.views.static.serve', (r'^survey_scans/(?P<path>.*)$', 'django.views.static.serve',
{'document_root': settings.SURVEYS, 'show_indexes':True}), {'document_root': settings.SURVEY_SCANS, 'show_indexes':True}),
(r'^photos/(?P<path>.*)$', 'django.views.static.serve', (r'^photos/(?P<path>.*)$', 'django.views.static.serve',
{'document_root': settings.PHOTOS_ROOT, 'show_indexes':True}), {'document_root': settings.PHOTOS_ROOT, 'show_indexes':True}),