From 99ab1e22e9dfd8ca058cb2256ee01b6492659438 Mon Sep 17 00:00:00 2001
From: substantialnoninfringinguser <substantialnoninfringinguser@gmail.com>
Date: Wed, 13 May 2009 05:38:37 +0100
Subject: [PATCH] [svn] MOre work on abstracting the file locations on scanned
 survey notes Copied from
 http://cucc@cucc.survex.com/svn/trunk/expoweb/troggle/, rev. 8190 by julian @
 1/18/2009 9:04 PM

---
 expo/fileAbstraction.py | 10 ++++++++--
 expo/models.py          |  1 +
 parsers/surveys.py      | 13 ++++++++-----
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/expo/fileAbstraction.py b/expo/fileAbstraction.py
index 4431480..299a488 100644
--- a/expo/fileAbstraction.py
+++ b/expo/fileAbstraction.py
@@ -25,9 +25,15 @@ def listdir(*path):
             c = reduce(urljoin, strippedpath)
         else:
             c = ""
-        print strippedpath, c
-        return urllib.urlopen(settings.FILES + "listdir/" + c)
+        c = c.replace("#", "%23")
+        print "FILE: ", settings.FILES + "listdir/" + c
+        return urllib.urlopen(settings.FILES + "listdir/" + c).read()
+        
+def dirsAsList(*path):
+    return [d for d in listdir(*path).split("\n") if len(d) > 0 and d[-1] == "/"]
 
+def filesAsList(*path):
+    return [d for d in listdir(*path).split("\n") if len(d) > 0 and d[-1] != "/"]
 
 def readFile(*path):
     try:
diff --git a/expo/models.py b/expo/models.py
index ebaf9ab..6d38b4f 100644
--- a/expo/models.py
+++ b/expo/models.py
@@ -350,6 +350,7 @@ class Photo(models.Model):
 scansFileStorage = FileSystemStorage(location=settings.SURVEYS, base_url=settings.SURVEYS_URL)
 def get_scan_path(instance, filename):
     year=instance.survey.expedition.year
+    print "WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number
     number="%02d" % instance.survey.wallet_number + str(instance.survey.wallet_letter) #using %02d string formatting because convention was 2009#01
     return os.path.join('./',year,year+r'#'+number,instance.contents+str(instance.number_in_wallet)+r'.jpg')
 
diff --git a/parsers/surveys.py b/parsers/surveys.py
index 5582bf8..687bbb0 100644
--- a/parsers/surveys.py
+++ b/parsers/surveys.py
@@ -14,10 +14,11 @@ import troggle.expo.fileAbstraction as fileAbstraction
 import csv
 import re
 import datetime
+import cStringIO
 
 surveytab = fileAbstraction.readFile("Surveys.csv")
 dialect=csv.Sniffer().sniff(surveytab)
-surveyreader = csv.reader(surveytab,dialect=dialect)
+surveyreader = csv.reader(cStringIO.StringIO(surveytab),dialect=dialect)
 print surveyreader
 headers = surveyreader.next()
 header = dict(zip(headers, range(len(headers)))) #set up a dictionary where the indexes are header names and the values are column numbers
@@ -50,12 +51,13 @@ for survey in surveyreader:
 
 # add survey scans
 def parseSurveyScans(year):
-    yearDirList = [d for d in fileAbstraction.listdir(year.year).split("\n") if d[-1] == "/"]
+    yearDirList = fileAbstraction.dirsAsList(year.year)
     for surveyFolder in yearDirList:
         print surveyFolder
         try:
             surveyNumber=re.match(r'\d\d\d\d#0*(\d+)',surveyFolder).groups()
-            scanList=fileAbstraction.listdir(yearPath, surveyFolder).split("\n")
+            scanList=fileAbstraction.filesAsList(year.year, surveyFolder)
+            print "BAR: ", year.year, surveyFolder, scanList
         except AttributeError:
             print surveyFolder + " ignored"
             continue
@@ -63,6 +65,7 @@ def parseSurveyScans(year):
         for scan in scanList:
             try:
                 scanChopped=re.match(r'(?i).*(notes|elev|plan|elevation|extend)(\d*)\.(png|jpg|jpeg)',scan).groups()
+                print "BAR: ", scanChopped
                 scanType,scanNumber,scanFormat=scanChopped
             except AttributeError:
                 print "Adding scans: " + scan + " ignored"
@@ -79,14 +82,14 @@ def parseSurveyScans(year):
                 survey=models.Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=year)[0]
             except models.Survey.MultipleObjectsReturned:
                 survey=models.Survey.objects.filter(wallet_number=surveyNumber, expedition=year)[0]
-               
+
             scanObj = models.ScannedImage(
                 file=os.path.join(year.year, surveyFolder, scan),
                 contents=scanType,
                 number_in_wallet=scanNumber,
                 survey=survey
                 )
-            #print "Added scanned image at " + str(scanObj)
+            print "Added scanned image at " + str(scanObj)
             scanObj.save()
                 
 for year in models.Expedition.objects.filter(year__gte=2000):   #expos since 2000, because paths and filenames were nonstandard before then