From 9043cef847fb15eff810362b7d1da1556228b7f1 Mon Sep 17 00:00:00 2001 From: goatchurch Date: Mon, 14 Sep 2009 23:23:09 +0100 Subject: [PATCH] rolled back a bad update --- parsers/surveys.py | 147 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 122 insertions(+), 25 deletions(-) diff --git a/parsers/surveys.py b/parsers/surveys.py index a67f8fe..8c06c9a 100644 --- a/parsers/surveys.py +++ b/parsers/surveys.py @@ -1,4 +1,4 @@ -import sys, os, types, logging +import sys, os, types, logging, stat #sys.path.append('C:\\Expo\\expoweb') #from troggle import * #os.environ['DJANGO_SETTINGS_MODULE']='troggle.settings' @@ -24,12 +24,13 @@ def get_or_create_placeholder(year): placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs) return placeholder_logbook_entry +# dead def readSurveysFromCSV(): try: # could probably combine these two surveytab = open(os.path.join(settings.SURVEY_SCANS, "Surveys.csv")) except IOError: import cStringIO, urllib - surveytab = cStringIO.StringIO(urllib.urlopen(settings.SURVEY_SCANS + "Surveys.csv").read()) + surveytab = cStringIO.StringIO(urllib.urlopen(settings.SURVEY_SCANS + "/Surveys.csv").read()) dialect=csv.Sniffer().sniff(surveytab.read()) surveytab.seek(0,0) surveyreader = csv.reader(surveytab,dialect=dialect) @@ -75,6 +76,7 @@ def readSurveysFromCSV(): logging.info("added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r") +# dead def listdir(*directories): try: return os.listdir(os.path.join(settings.SURVEYS, *directories)) @@ -134,11 +136,13 @@ def parseSurveyScans(year, logfile=None): continue scanObj.save() +# dead def parseSurveys(logfile=None): readSurveysFromCSV() for year in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then parseSurveyScans(year) +# dead def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the thumbnail engine can't handle them (uses PIL) file=Image.open(filePath) print filePath @@ -148,10 +152,11 @@ def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the return False -# handles url or file +# handles url or file, so we can refer to a set of scans on another server def GetListDir(sdir): res = [ ] if sdir[:7] == "http://": + assert False, "Not written" s = urllib.urlopen(sdir) else: for f in os.listdir(sdir): @@ -160,44 +165,136 @@ def GetListDir(sdir): res.append((f, ff, os.path.isdir(ff))) return res + + + + +def LoadListScansFile(survexscansfolder): + gld = [ ] + + # flatten out any directories in these book files + for (fyf, ffyf, fisdiryf) in GetListDir(survexscansfolder.fpath): + if fisdiryf: + gld.extend(GetListDir(ffyf)) + else: + gld.append((fyf, ffyf, fisdiryf)) + + for (fyf, ffyf, fisdiryf) in gld: + assert not fisdiryf, ffyf + if re.search("\.(?:png|jpg|jpeg)(?i)$", fyf): + survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder) + survexscansingle.save() + + # this iterates through the scans directories (either here or on the remote server) # and builds up the models we can access later -def LoadListScans(surveyscansdir): +def LoadListScans(): SurvexScanSingle.objects.all().delete() SurvexScansFolder.objects.all().delete() - for f, ff, fisdir in GetListDir(surveyscansdir): + # first do the smkhs (large kh survey scans) directory + survexscansfoldersmkhs = SurvexScansFolder(fpath=os.path.join(settings.SURVEY_SCANS, "smkhs"), walletname="smkhs") + if os.path.isdir(survexscansfoldersmkhs.fpath): + survexscansfoldersmkhs.save() + LoadListScansFile(survexscansfoldersmkhs) + + + # iterate into the surveyscans directory + for f, ff, fisdir in GetListDir(os.path.join(settings.SURVEY_SCANS, "surveyscans")): if not fisdir: continue # do the year folders if re.match("\d\d\d\d$", f): for fy, ffy, fisdiry in GetListDir(ff): - if not fisdiry: - assert fy == "index", ffy - continue + assert fisdiry, ffy survexscansfolder = SurvexScansFolder(fpath=ffy, walletname=fy) survexscansfolder.save() - for fyf, ffyf, fisdiryf in GetListDir(ffy): - assert not fisdiryf, ffyf - survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder) - survexscansingle.save() + LoadListScansFile(survexscansfolder) + + # do the elif f != "thumbs": survexscansfolder = SurvexScansFolder(fpath=ff, walletname=f) survexscansfolder.save() - gld = [ ] - - # flatten out any directories in these book files - for (fyf, ffyf, fisdiryf) in GetListDir(ff): - if fisdiryf: - gld.extend(GetListDir(ffyf)) - else: - gld.append((fyf, ffyf, fisdiryf)) - - for (fyf, ffyf, fisdiryf) in gld: - assert not fisdiryf, ffyf - survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder) - survexscansingle.save() + LoadListScansFile(survexscansfolder) + +def FindTunnelScan(tunnelfile, path): + scansfolder, scansfile = None, None + mscansdir = re.search("(\d\d\d\d#\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg))$", path) + if mscansdir: + scansfolderl = SurvexScansFolder.objects.filter(walletname=mscansdir.group(1)) + if len(scansfolderl): + assert len(scansfolderl) == 1 + scansfolder = scansfolderl[0] + if scansfolder: + scansfilel = scansfolder.survexscansingle_set.filter(name=mscansdir.group(2)) + if len(scansfilel): + assert len(scansfilel) == 1 + scansfile = scansfilel[0] + if scansfolder: + tunnelfile.survexscansfolders.add(scansfolder) + if scansfile: + tunnelfile.survexscans.add(scansfile) + + elif path and not re.search("\.(?:png|jpg)$(?i)", path): + name = os.path.split(path)[1] + print "ttt", tunnelfile.tunnelpath, path, name + rtunnelfilel = TunnelFile.objects.filter(tunnelname=name) + if len(rtunnelfilel): + assert len(rtunnelfilel) == 1, ("two paths with name of", path, "need more discrimination coded") + rtunnelfile = rtunnelfilel[0] + #print "ttt", tunnelfile.tunnelpath, path, name, rtunnelfile.tunnelpath + tunnelfile.tunnelcontains.add(rtunnelfile) + + tunnelfile.save() + + +def SetTunnelfileInfo(tunnelfile): + ff = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath) + tunnelfile.filesize = os.stat(ff)[stat.ST_SIZE] + fin = open(ff) + ttext = fin.read() + fin.close() + + mtype = re.search("<(fontcolours|sketch)", ttext) + assert mtype, ff + tunnelfile.bfontcolours = (mtype.group(1)=="fontcolours") + tunnelfile.npaths = len(re.findall(" + # + for path, style in re.findall('