From 24d90bae752dfc5d51cf038d3a5d4b3858765e9f Mon Sep 17 00:00:00 2001 From: goatchurch Date: Mon, 14 Sep 2009 22:52:46 +0100 Subject: [PATCH] make 2008 logbook correctly parse --- core/models_survex.py | 10 +++ core/view_surveys.py | 62 ++++++++++-------- databaseReset.py | 2 +- parsers/logbooks.py | 12 ++-- parsers/surveys.py | 103 ++++++++++++++++++++++-------- templates/survexscansfolders.html | 6 +- templates/tunnelfiles.html | 23 ++++++- urls.py | 2 + 8 files changed, 157 insertions(+), 63 deletions(-) diff --git a/core/models_survex.py b/core/models_survex.py index 2828389..7b652b7 100644 --- a/core/models_survex.py +++ b/core/models_survex.py @@ -155,6 +155,9 @@ class SurvexScansFolder(models.Model): fpath = models.CharField(max_length=200) walletname = models.CharField(max_length=200) + class Meta: + ordering = ('walletname',) + def get_absolute_url(self): return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansfolder', kwargs={"path":re.sub("#", "%23", self.walletname)})) @@ -163,18 +166,25 @@ class SurvexScanSingle(models.Model): name = models.CharField(max_length=200) survexscansfolder = models.ForeignKey("SurvexScansFolder", null=True) + class Meta: + ordering = ('name',) + def get_absolute_url(self): return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansingle', kwargs={"path":re.sub("#", "%23", self.survexscansfolder.walletname), "file":self.name})) class TunnelFile(models.Model): tunnelpath = models.CharField(max_length=200) + tunnelname = models.CharField(max_length=200) bfontcolours = models.BooleanField() + survexscansfolders = models.ManyToManyField("SurvexScansFolder") survexscans = models.ManyToManyField("SurvexScanSingle") survexblocks = models.ManyToManyField("SurvexBlock") tunnelcontains = models.ManyToManyField("TunnelFile") # case when its a frame type filesize = models.IntegerField(default=0) npaths = models.IntegerField(default=0) + survextitles = models.ManyToManyField("SurvexTitle") + class Meta: ordering = ('tunnelpath',) diff --git a/core/view_surveys.py b/core/view_surveys.py index fc93a7e..66aa1ed 100644 --- a/core/view_surveys.py +++ b/core/view_surveys.py @@ -5,6 +5,7 @@ from django.http import HttpResponse, Http404 import os, stat import re from troggle.core.models import SurvexScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile +import parsers.surveys # inline fileabstraction into here if it's not going to be useful anywhere else # keep things simple and ignore exceptions everywhere for now @@ -182,43 +183,50 @@ def tunneldata(request): tunnelfiles = TunnelFile.objects.all() return render_to_response('tunnelfiles.html', { 'tunnelfiles':tunnelfiles, 'settings': settings }) + def tunnelfile(request, path): tunnelfile = TunnelFile.objects.get(tunnelpath=path) tfile = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath) + return HttpResponse(content=open(tfile), mimetype="text/plain") - # just output the file - if not request.POST: - return HttpResponse(content=open(tfile), mimetype="text/plain") +def tunnelfileupload(request, path): + tunnelfile = TunnelFile.objects.get(tunnelpath=path) + tfile = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath) project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"] print (project, user, tunnelversion) - for uploadedfile in request.FILES.values(): - if uploadedfile.field_name != "sketch": - return HttpResponse(content="Error: non-sketch file uploaded", mimetype="text/plain") - if uploadedfile.content_type != "text/plain": - return HttpResponse(content="Error: non-plain content type", mimetype="text/plain") - - # could use this to add new files - if os.path.split(path)[1] != uploadedfile.name: - return HttpResponse(content="Error: name disagrees", mimetype="text/plain") + + + assert len(request.FILES.values()) == 1, "only one file to upload" + + uploadedfile = request.FILES.values()[0] - orgsize = tunnelfile.filesize # = os.stat(tfile)[stat.ST_SIZE] - - ttext = uploadedfile.read() + if uploadedfile.field_name != "sketch": + return HttpResponse(content="Error: non-sketch file uploaded", mimetype="text/plain") + if uploadedfile.content_type != "text/plain": + return HttpResponse(content="Error: non-plain content type", mimetype="text/plain") - # could check that the user and projects agree here + # could use this to add new files + if os.path.split(path)[1] != uploadedfile.name: + return HttpResponse(content="Error: name disagrees", mimetype="text/plain") + + orgsize = tunnelfile.filesize # = os.stat(tfile)[stat.ST_SIZE] - fout = open(tfile, "w") - fout.write(ttext) - fout.close() - - # redo its settings of - tunnelfile.filesize = os.stat(tfile)[stat.ST_SIZE] - tunnelfile.save() - - uploadedfile.close() - message = "File size %d overwritten with size %d" % (orgsize, tunnelfile.filesize) - return HttpResponse(content=message, mimetype="text/plain") + ttext = uploadedfile.read() + + # could check that the user and projects agree here + + fout = open(tfile, "w") + fout.write(ttext) + fout.close() + + # redo its settings of + parsers.surveys.SetTunnelfileInfo(tunnelfile) + tunnelfile.save() + + uploadedfile.close() + message = "File size %d overwritten with size %d" % (orgsize, tunnelfile.filesize) + return HttpResponse(content=message, mimetype="text/plain") diff --git a/databaseReset.py b/databaseReset.py index 6e049b6..bc953c9 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -63,7 +63,7 @@ def import_surveys(): def import_surveyscans(): import parsers.surveys - parsers.surveys.LoadListScans(settings.SURVEY_SCANS) + parsers.surveys.LoadListScans() def import_descriptions(): diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 7d7d7d1..9404414 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -120,7 +120,7 @@ def Parselogwikitxt(year, expedition, txt): trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt) for triphead, triptext in trippara: tripheadp = triphead.split("|") - #print tripheadp + #print "ttt", tripheadp assert len(tripheadp) == 3, (tripheadp, triptext) tripdate, tripplace, trippeople = tripheadp tripsplace = tripplace.split(" - ") @@ -135,7 +135,7 @@ def Parselogwikitxt(year, expedition, txt): tu = "" #assert tripcave == "Journey", (triphead, triptext) - print tripdate + #print tripdate ldate = ParseDate(tripdate.strip(), year) #print "\n", tripcave, "--- ppp", trippeople, len(triptext) EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, logtime_underground=0) @@ -155,7 +155,8 @@ def Parseloghtmltxt(year, expedition, txt): \s*$ ''', trippara) if not s: - print "can't parse: ", trippara # this is 2007 which needs editing + if not re.search("Rigging Guide", trippara): + print "can't parse: ", trippara # this is 2007 which needs editing #assert s, trippara continue @@ -218,7 +219,7 @@ def Parseloghtml01(year, expedition, txt): ltriptext = re.sub("", "'''", ltriptext) - print ldate, trippeople.strip() + #print ldate, trippeople.strip() # could includ the tripid (url link for cross referencing) EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0) @@ -236,8 +237,7 @@ def Parseloghtml03(year, expedition, txt): if re.match("T/U|Time underwater", sheader[-1]): tu = sheader.pop() if len(sheader) != 3: - print sheader - # continue + print "header not three pieces", sheader tripdate, triptitle, trippeople = sheader ldate = ParseDate(tripdate.strip(), year) triptitles = triptitle.split(" , ") diff --git a/parsers/surveys.py b/parsers/surveys.py index cffeb4a..8c06c9a 100644 --- a/parsers/surveys.py +++ b/parsers/surveys.py @@ -165,13 +165,42 @@ def GetListDir(sdir): res.append((f, ff, os.path.isdir(ff))) return res + + + + +def LoadListScansFile(survexscansfolder): + gld = [ ] + + # flatten out any directories in these book files + for (fyf, ffyf, fisdiryf) in GetListDir(survexscansfolder.fpath): + if fisdiryf: + gld.extend(GetListDir(ffyf)) + else: + gld.append((fyf, ffyf, fisdiryf)) + + for (fyf, ffyf, fisdiryf) in gld: + assert not fisdiryf, ffyf + if re.search("\.(?:png|jpg|jpeg)(?i)$", fyf): + survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder) + survexscansingle.save() + + # this iterates through the scans directories (either here or on the remote server) # and builds up the models we can access later -def LoadListScans(surveyscansdir): +def LoadListScans(): SurvexScanSingle.objects.all().delete() SurvexScansFolder.objects.all().delete() - for f, ff, fisdir in GetListDir(surveyscansdir): + # first do the smkhs (large kh survey scans) directory + survexscansfoldersmkhs = SurvexScansFolder(fpath=os.path.join(settings.SURVEY_SCANS, "smkhs"), walletname="smkhs") + if os.path.isdir(survexscansfoldersmkhs.fpath): + survexscansfoldersmkhs.save() + LoadListScansFile(survexscansfoldersmkhs) + + + # iterate into the surveyscans directory + for f, ff, fisdir in GetListDir(os.path.join(settings.SURVEY_SCANS, "surveyscans")): if not fisdir: continue @@ -181,28 +210,46 @@ def LoadListScans(surveyscansdir): assert fisdiry, ffy survexscansfolder = SurvexScansFolder(fpath=ffy, walletname=fy) survexscansfolder.save() - for fyf, ffyf, fisdiryf in GetListDir(ffy): - assert not fisdiryf, ffyf - survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder) - survexscansingle.save() + LoadListScansFile(survexscansfolder) + + # do the elif f != "thumbs": survexscansfolder = SurvexScansFolder(fpath=ff, walletname=f) survexscansfolder.save() - gld = [ ] - - # flatten out any directories in these book files - for (fyf, ffyf, fisdiryf) in GetListDir(ff): - if fisdiryf: - gld.extend(GetListDir(ffyf)) - else: - gld.append((fyf, ffyf, fisdiryf)) - - for (fyf, ffyf, fisdiryf) in gld: - assert not fisdiryf, ffyf - survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder) - survexscansingle.save() + LoadListScansFile(survexscansfolder) + +def FindTunnelScan(tunnelfile, path): + scansfolder, scansfile = None, None + mscansdir = re.search("(\d\d\d\d#\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg))$", path) + if mscansdir: + scansfolderl = SurvexScansFolder.objects.filter(walletname=mscansdir.group(1)) + if len(scansfolderl): + assert len(scansfolderl) == 1 + scansfolder = scansfolderl[0] + if scansfolder: + scansfilel = scansfolder.survexscansingle_set.filter(name=mscansdir.group(2)) + if len(scansfilel): + assert len(scansfilel) == 1 + scansfile = scansfilel[0] + if scansfolder: + tunnelfile.survexscansfolders.add(scansfolder) + if scansfile: + tunnelfile.survexscans.add(scansfile) + + elif path and not re.search("\.(?:png|jpg)$(?i)", path): + name = os.path.split(path)[1] + print "ttt", tunnelfile.tunnelpath, path, name + rtunnelfilel = TunnelFile.objects.filter(tunnelname=name) + if len(rtunnelfilel): + assert len(rtunnelfilel) == 1, ("two paths with name of", path, "need more discrimination coded") + rtunnelfile = rtunnelfilel[0] + #print "ttt", tunnelfile.tunnelpath, path, name, rtunnelfile.tunnelpath + tunnelfile.tunnelcontains.add(rtunnelfile) + + tunnelfile.save() + def SetTunnelfileInfo(tunnelfile): ff = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath) @@ -219,11 +266,13 @@ def SetTunnelfileInfo(tunnelfile): # # - print tunnelfile.tunnelpath, re.findall(' {{survexscansfolder.walletname}} {{survexscansfolder.survexscansingle_set.all|length}} - {{survexscansfolder.survexblock_set.all|length}} + + {% for survexblock in survexscansfolder.survexblock_set.all %} + {{survexblock}} + {% endfor %} + {% endfor %} diff --git a/templates/tunnelfiles.html b/templates/tunnelfiles.html index 8b80ade..3d49e5d 100644 --- a/templates/tunnelfiles.html +++ b/templates/tunnelfiles.html @@ -8,15 +8,34 @@

All Tunnel files

- + {% for tunnelfile in tunnelfiles %} - + + + + + + + + {% endfor %}
FileFontFrameSurvexBlocksSizePaths
FileFontSurvexBlocksSizePathsScans folderScan filesFrames
{{tunnelfile.tunnelpath}} {{tunnelfile.bfontcolours}} {{tunnelfile.filesize}} {{tunnelfile.npaths}} + {% for survexscansfolder in tunnelfile.survexscansfolders.all %} + {{survexscansfolder.walletname}} + {% endfor %} + + {% for survexscansingle in tunnelfile.survexscans.all %} + {{survexscansingle.name}} + {% endfor %} + + {% for rtunnelfile in tunnelfile.tunnelcontains.all %} + {{rtunnelfile.tunnelpath}} + {% endfor %} +
diff --git a/urls.py b/urls.py index bdaabb3..92f3c26 100644 --- a/urls.py +++ b/urls.py @@ -111,6 +111,8 @@ urlpatterns = patterns('', url(r'^tunneldata/$', view_surveys.tunneldata, name="tunneldata"), url(r'^tunneldataraw/(?P.+?\.xml)$', view_surveys.tunnelfile, name="tunnelfile"), + url(r'^tunneldataraw/(?P.+?\.xml)/upload$',view_surveys.tunnelfileupload, name="tunnelfileupload"), + #url(r'^tunneldatainfo/(?P.+?\.xml)$', view_surveys.tunnelfileinfo, name="tunnelfileinfo"), (r'^photos/(?P.*)$', 'django.views.static.serve',