From 12cf3a6d534e5038b5d78b11a03cef2b81f5f852 Mon Sep 17 00:00:00 2001 From: goatchurch Date: Fri, 11 Sep 2009 23:56:47 +0100 Subject: [PATCH] tunnelfiles scheme added --- core/admin.py | 2 +- core/models.py | 51 ++++++++++---- core/models_survex.py | 12 +++- core/view_surveys.py | 8 ++- core/views_logbooks.py | 21 +++--- core/views_other.py | 17 ----- databaseReset.py | 10 ++- media/css/main3.css | 67 +++++++++++++++--- parsers/logbooks.py | 51 ++------------ parsers/survex.py | 6 +- parsers/surveys.py | 35 +++++++++- templates/base.html | 8 +-- templates/calendar.html | 71 ------------------- templates/expedition.html | 120 +++++++++++++++++++------------- templates/logbook2005style.html | 2 +- templates/logbookentry.html | 4 +- templates/tunnelfiles.html | 20 ++++++ urls.py | 5 +- 18 files changed, 277 insertions(+), 233 deletions(-) delete mode 100644 templates/calendar.html create mode 100644 templates/tunnelfiles.html diff --git a/core/admin.py b/core/admin.py index 37afc3b..f8fe9fe 100644 --- a/core/admin.py +++ b/core/admin.py @@ -51,7 +51,7 @@ class PhotoInline(admin.TabularInline): class PersonTripInline(admin.TabularInline): model = PersonTrip exclude = ['persontrip_next','Delete'] - raw_id_fields = ('person_expedition',) + raw_id_fields = ('personexpedition',) extra = 1 #class LogbookEntryAdmin(VersionAdmin): diff --git a/core/models.py b/core/models.py index 08504c0..432eca4 100644 --- a/core/models.py +++ b/core/models.py @@ -5,6 +5,7 @@ from django.contrib import admin from django.core.files.storage import FileSystemStorage from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType +from django.db.models import Min, Max from django.conf import settings from decimal import Decimal, getcontext from django.core.urlresolvers import reverse @@ -65,21 +66,18 @@ class TroggleImageModel(ImageModel): class Expedition(TroggleModel): year = models.CharField(max_length=20, unique=True) name = models.CharField(max_length=100) - - # these will become min and max dates - date_from = models.DateField(blank=True,null=True) - date_to = models.DateField(blank=True,null=True) - + def __unicode__(self): return self.year class Meta: ordering = ('-year',) - get_latest_by = 'date_from' + get_latest_by = 'year' def get_absolute_url(self): return urlparse.urljoin(settings.URL_ROOT, reverse('expedition', args=[self.year])) + # construction function. should be moved out def get_expedition_day(self, date): expeditiondays = self.expeditionday_set.filter(date=date) if expeditiondays: @@ -89,11 +87,28 @@ class Expedition(TroggleModel): res.save() return res + def day_min(self): + res = self.expeditionday_set.all() + return res and res[0] or None + + def day_max(self): + res = self.expeditionday_set.all() + return res and res[len(res) - 1] or None + + class ExpeditionDay(TroggleModel): expedition = models.ForeignKey("Expedition") date = models.DateField() + class Meta: + ordering = ('date',) + + def GetPersonTrip(self, personexpedition): + personexpeditions = self.persontrip_set.filter(expeditionday=self) + return personexpeditions and personexpeditions[0] or None + + # # single Person, can go on many years # @@ -159,8 +174,8 @@ class Person(TroggleModel): class PersonExpedition(TroggleModel): expedition = models.ForeignKey(Expedition) person = models.ForeignKey(Person) - date_from = models.DateField(blank=True,null=True) - date_to = models.DateField(blank=True,null=True) + + is_guest = models.BooleanField(default=False) COMMITTEE_CHOICES = ( ('leader','Expo leader'), @@ -184,7 +199,6 @@ class PersonExpedition(TroggleModel): class Meta: ordering = ('-expedition',) #order_with_respect_to = 'expedition' - get_latest_by = 'expedition' def __unicode__(self): return "%s: (%s)" % (self.person, self.expedition) @@ -205,11 +219,21 @@ class PersonExpedition(TroggleModel): survexblocks = [personrole.survexblock for personrole in self.personrole_set.all() ] return sum([survexblock.totalleglength for survexblock in set(survexblocks)]) + # would prefer to return actual person trips so we could link to first and last ones + def day_min(self): + res = self.persontrip_set.aggregate(day_min=Min("expeditionday__date")) + return res["day_min"] + + def day_max(self): + res = self.persontrip_set.all().aggregate(day_max=Max("expeditionday__date")) + return res["day_max"] + # # Single parsed entry from Logbook # class LogbookEntry(TroggleModel): date = models.DateField() + expeditionday = models.ForeignKey("ExpeditionDay", null=True) expedition = models.ForeignKey(Expedition,blank=True,null=True) # yes this is double- author = models.ForeignKey(PersonExpedition,blank=True,null=True) # the person who writes it up doesn't have to have been on the trip. # Re: the above- so this field should be "typist" or something, not "author". - AC 15 jun 09 @@ -249,12 +273,14 @@ class LogbookEntry(TroggleModel): """Produces a link to a new QM with the next number filled in and this LogbookEntry set as 'found by' """ return settings.URL_ROOT + r'/admin/core/qm/add/?' + r'found_by=' + str(self.pk) +'&number=' + str(self.new_QM_number()) + def DayIndex(self): + return list(self.expeditionday.logbookentry_set.all()).index(self) # # Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry) # class PersonTrip(TroggleModel): - person_expedition = models.ForeignKey(PersonExpedition,null=True) + personexpedition = models.ForeignKey("PersonExpedition",null=True) expeditionday = models.ForeignKey("ExpeditionDay") date = models.DateField() @@ -271,9 +297,8 @@ class PersonTrip(TroggleModel): return self.logbook_entry.cave and self.logbook_entry.cave or self.logbook_entry.place def __unicode__(self): - return "%s (%s)" % (self.person_expedition, self.date) - - + return "%s (%s)" % (self.personexpedition, self.date) + ########################################## diff --git a/core/models_survex.py b/core/models_survex.py index f3fc61d..9fd23c4 100644 --- a/core/models_survex.py +++ b/core/models_survex.py @@ -113,6 +113,8 @@ class SurvexBlock(models.Model): ss.save() return ss + def DayIndex(self): + return list(self.expeditionday.survexblock_set.all()).index(self) class SurvexTitle(models.Model): @@ -143,7 +145,8 @@ class SurvexPersonRole(models.Model): person = models.ForeignKey('Person', blank=True, null=True) personexpedition = models.ForeignKey('PersonExpedition', blank=True, null=True) persontrip = models.ForeignKey('PersonTrip', blank=True, null=True) - + expeditionday = models.ForeignKey("ExpeditionDay", null=True) + def __unicode__(self): return unicode(self.person) + " - " + unicode(self.survexblock) + " - " + unicode(self.nrole) @@ -164,4 +167,11 @@ class SurvexScanSingle(models.Model): return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansingle', kwargs={"path":re.sub("#", "%23", self.survexscansfolder.walletname), "file":self.name})) +class TunnelFile(models.Model): + tunnelpath = models.CharField(max_length=200) + bfontcolours = models.BooleanField() + survexscans = models.ManyToManyField("SurvexScanSingle") + survexblocks = models.ManyToManyField("SurvexBlock") + tunnelcontains = models.ManyToManyField("TunnelFile") # case when its a frame type + \ No newline at end of file diff --git a/core/view_surveys.py b/core/view_surveys.py index ab45438..0324b24 100644 --- a/core/view_surveys.py +++ b/core/view_surveys.py @@ -4,7 +4,7 @@ from django.shortcuts import render_to_response from django.http import HttpResponse, Http404 import os import re -from troggle.core.models import SurvexScansFolder, SurvexScanSingle, SurvexBlock +from troggle.core.models import SurvexScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile # inline fileabstraction into here if it's not going to be useful anywhere else # keep things simple and ignore exceptions everywhere for now @@ -47,6 +47,7 @@ extmimetypes = {".txt": "text/plain", ".jpg": "image/jpeg", } +# dead def jgtfile(request, f): fp = os.path.join(settings.SURVEYS, f) # could also surf through SURVEX_DATA @@ -175,4 +176,9 @@ def surveyscansingle(request, path, file): def surveyscansfolders(request): survexscansfolders = SurvexScansFolder.objects.all() return render_to_response('survexscansfolders.html', { 'survexscansfolders':survexscansfolders, 'settings': settings }) + + +def tunneldata(request): + tunnelfiles = TunnelFile.objects.all() + return render_to_response('tunnelfiles.html', { 'tunnelfiles':tunnelfiles, 'settings': settings }) \ No newline at end of file diff --git a/core/views_logbooks.py b/core/views_logbooks.py index 17c88fd..1cdffff 100644 --- a/core/views_logbooks.py +++ b/core/views_logbooks.py @@ -44,16 +44,21 @@ def personindex(request): def expedition(request, expeditionname): - year = int(expeditionname) - expedition = Expedition.objects.get(year=year) - expedition_next = Expedition.objects.filter(year=year+1) and Expedition.objects.get(year=year+1) or None - expedition_prev = Expedition.objects.filter(year=year-1) and Expedition.objects.get(year=year-1) or None - message = "No message" + expedition = Expedition.objects.get(year=int(expeditionname)) + expeditions = Expedition.objects.all() + personexpeditiondays = [ ] + for personexpedition in expedition.personexpedition_set.all(): + prow = [ ] + for expeditionday in expedition.expeditionday_set.all(): + pcell = { "persontrips":expeditionday.persontrip_set.filter(personexpedition=personexpedition) } + pcell["survexblocks"] = set([survexpersonrole.survexblock for survexpersonrole in expeditionday.survexpersonrole_set.filter(personexpedition=personexpedition)]) + prow.append(pcell) + personexpeditiondays.append({"personexpedition":personexpedition, "personrow":prow}) + + message = "" if "reload" in request.GET: message = LoadLogbookForExpedition(expedition) - #message = str(GetPersonExpeditionNameLookup(expedition).keys()) - logbookentries = expedition.logbookentry_set.order_by('date') - return render_with_context(request,'expedition.html', {'expedition': expedition, 'expedition_next':expedition_next, 'expedition_prev':expedition_prev, 'logbookentries':logbookentries, 'message':message, 'settings':settings }) + return render_with_context(request,'expedition.html', {'expedition': expedition, 'expeditions':expeditions, 'personexpeditiondays':personexpeditiondays, 'message':message, 'settings':settings }) def get_absolute_url(self): return ('expedition', (expedition.year)) diff --git a/core/views_other.py b/core/views_other.py index 78d3bd1..3acb87d 100644 --- a/core/views_other.py +++ b/core/views_other.py @@ -45,23 +45,6 @@ def todo(request): totallogbookentries = LogbookEntry.objects.count() return render_with_context(request,'index.html', {'expeditions':expeditions, 'all':'all', 'totallogbookentries':totallogbookentries, "message":message}) -def calendar(request, year): - week=['S','S','M','T','W','T','F'] - expedition = Expedition.objects.get(year=year) - personexpeditions = expedition.personexpedition_set.all() - - listdays = [ ] # the columns of the table - date = expedition.date_from - while date <= expedition.date_to: - listdays.append(date) - date += datetime.timedelta(days=1) - - personexpeditiondays = [ ] - for personexpedition in personexpeditions: - pelistdays = [ (personexpedition.date_from and (personexpedition.date_from <= date < personexpedition.date_to)) for date in listdays ] - personexpeditiondays.append([personexpedition, pelistdays]) - - return render_with_context(request,'calendar.html', {"expedition":expedition, "listdays":listdays, "personexpeditiondays":personexpeditiondays}) def controlPanel(request): jobs_completed=[] diff --git a/databaseReset.py b/databaseReset.py index b104550..6c635d1 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -75,6 +75,11 @@ def parse_descriptions(): parsers.descriptions.parseDescriptions() parsers.descriptions.parseDescriptionsOnCaveObjects() +def import_tunnelfiles(): + import parsers.surveys + parsers.surveys.LoadTunnelFiles(settings.TUNNEL_DATA) + + def reset(): """ Wipe the troggle database and import everything from legacy data """ @@ -86,7 +91,8 @@ def reset(): import_survex() import_logbooks() import_QMs() - import_surveys() + import_tunnelfiles() + #import_surveys() import_descriptions() parse_descriptions() @@ -112,6 +118,8 @@ if __name__ == "__main__": resetdesc() elif "scans" in sys.argv: import_surveyscans() + elif "tunnel" in sys.argv: + import_tunnelfiles() elif "reset" in sys.argv: reset() elif "survex" in sys.argv: diff --git a/media/css/main3.css b/media/css/main3.css index 4652486..990afc7 100644 --- a/media/css/main3.css +++ b/media/css/main3.css @@ -33,6 +33,60 @@ div.logbookentry p margin:10px; } +div#content { + Zmargin-top: 50px; + Zmargin-left: 120px; + Zmargin-right: 120px; + padding-top: 10px; + padding-left: 5em; + padding-right: 5em; + padding-bottom:5em; + Zbackground:#CCC; + background-color:#f9f9f9; +} + +table.expeditionpersonlist th +{ + padding:0 0 0 0; +} + +table.expeditionpersonlist td.persondayactivity +{ + width:4em; + font-size:70%; + padding:0 0 0 0; +} +table.expeditionpersonlist td.persondayactivity-nothing +{ + background-color:#bbb; +} + +table.expeditionpersonlist a +{ + text-decoration:none; + color:#009; +} +table.expeditionpersonlist a:visited +{ + color:#509; +} + +table.expeditionpersonlist a.dayindexlog-1 +{ background-color: #440; color: white; } +table.expeditionpersonlist a.dayindexlog-2 +{ background-color: #9fa; } +table.expeditionpersonlist a.dayindexsurvex-1 +{ background-color: #000; color: #55f; } +table.expeditionpersonlist a.dayindexsurvex-2 +{ background-color: #6ff; } + +table.expeditionpersonlist a:hover +{ + color:white; + background-color:#e00; +} + + div#content div#col2 { @@ -95,7 +149,7 @@ p.indent #currentLocation { float:right; - background:#999; + Zbackground:#999; line-height: 80%; font-variant: small-caps; margin: 0px; @@ -259,22 +313,13 @@ div#editLinks { filter:alpha(opacity=75); -moz-opacity:.75; opacity:.75; + text-align:right; } div#editLinks a{ color:#FFF; } -div#content { - Zmargin-top: 50px; - Zmargin-left: 120px; - Zmargin-right: 120px; - padding-top: 10px; - padding-left: 5em; - padding-right: 5em; - padding-bottom:5em; - background:#CCC; -} .toolbarlinks { diff --git a/parsers/logbooks.py b/parsers/logbooks.py index ec314b6..7d7d7d1 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -88,14 +88,14 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_ cave=GetCaveLookup().get(lplace) #Check for an existing copy of the current entry, and save + expeditionday = expedition.get_expedition_day(date) lookupAttribs={'date':date, 'title':title} - nonLookupAttribs={'place':place, 'text':text, 'author':author, 'expedition':expedition, 'cave':cave, 'slug':slugify(title)[:50]} + nonLookupAttribs={'place':place, 'text':text, 'author':author, 'expedition':expedition, 'expeditionday':expeditionday, 'cave':cave, 'slug':slugify(title)[:50]} lbo, created=save_carefully(models.LogbookEntry, lookupAttribs, nonLookupAttribs) - expeditiondate = expedition.get_expedition_date(date) for tripperson, time_underground in trippersons: - lookupAttribs={'person_expedition':tripperson, 'logbook_entry':lbo} - nonLookupAttribs={'time_underground':time_underground, 'date':date, 'expeditiondate':expeditiondate, 'is_logbook_entry_author':(tripperson == author)} + lookupAttribs={'personexpedition':tripperson, 'logbook_entry':lbo} + nonLookupAttribs={'time_underground':time_underground, 'date':date, 'expeditionday':expeditionday, 'is_logbook_entry_author':(tripperson == author)} #print nonLookupAttribs save_carefully(models.PersonTrip, lookupAttribs, nonLookupAttribs) @@ -281,10 +281,6 @@ def SetDatesFromLogbookEntries(expedition): """ for personexpedition in expedition.personexpedition_set.all(): persontrips = personexpedition.persontrip_set.order_by('logbook_entry__date') - personexpedition.date_from = min([persontrip.logbook_entry.date for persontrip in persontrips] or [None]) - personexpedition.date_to = max([persontrip.logbook_entry.date for persontrip in persontrips] or [None]) - personexpedition.save() - # sequencing is difficult to do lprevpersontrip = None for persontrip in persontrips: @@ -296,46 +292,11 @@ def SetDatesFromLogbookEntries(expedition): lprevpersontrip = persontrip persontrip.save() - # from trips rather than logbook entries, which may include events outside the expedition - expedition.date_from = min([personexpedition.date_from for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_from] or [None]) - expedition.date_to = max([personexpedition.date_to for personexpedition in expedition.personexpedition_set.all() if personexpedition.date_to] or [None]) - expedition.save() - -# The below has been replaced with the methods get_next_by_id and get_previous_by_id -# # order by appearance in the logbook (done by id) -# lprevlogbookentry = None -# for logbookentry in expedition.logbookentry_set.order_by('id'): -# logbookentry.logbookentry_prev = lprevlogbookentry -# if lprevlogbookentry: -# lprevlogbookentry.logbookentry_next = logbookentry -# lprevlogbookentry.save() -# logbookentry.logbookentry_next = None -# logbookentry.save() -# lprevlogbookentry = logbookentry - -# This combined date / number key is a weird way of doing things. Use the primary key instead. If we are going to use the date for looking up entries, we should set it up to allow multiple results. - # order by date for setting the references -# lprevlogbookentry = None -# for logbookentry in expedition.logbookentry_set.order_by('date'): -# if lprevlogbookentry and lprevlogbookentry.date == logbookentry.date: -# mcount = re.search("_(\d+)$", lprevlogbookentry.href) -# mc = mcount and (int(mcount.group(1)) + 1) or 1 -# logbookentry.href = "%s_%d" % (logbookentry.date, mc) -# else: -# logbookentry.href = "%s" % logbookentry.date -# logbookentry.save() -# lprevlogbookentry = logbookentry - def LoadLogbookForExpedition(expedition): """ Parses all logbook entries for one expedition """ - - #We're checking for stuff that's changed in admin before deleting it now. - #print "deleting logbooks for", expedition - #expedition.logbookentry_set.all().delete() - #models.PersonTrip.objects.filter(person_expedition__expedition=expedition).delete() - + expowebbase = os.path.join(settings.EXPOWEB, "years") year = str(expedition.year) for lyear, lloc, parsefunc in yearlinks: @@ -347,7 +308,7 @@ def LoadLogbookForExpedition(expedition): fin.close() parsefunc(year, expedition, txt) SetDatesFromLogbookEntries(expedition) - return "TOLOAD: " + year + " " + str(expedition.personexpedition_set.all()[1].logbookentry_set.count()) + " " + str(models.PersonTrip.objects.filter(person_expedition__expedition=expedition).count()) + return "TOLOAD: " + year + " " + str(expedition.personexpedition_set.all()[1].logbookentry_set.count()) + " " + str(models.PersonTrip.objects.filter(personexpedition__expedition=expedition).count()) def LoadLogbooks(): diff --git a/parsers/survex.py b/parsers/survex.py index 985d4e5..f8ec6d4 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -69,7 +69,6 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): mref = comment and re.match('.*?ref.*?(\d+)\s*#\s*(\d+)', comment) if mref: refscan = "%s#%s" % (mref.group(1), mref.group(2)) - print refscan survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan) if survexscansfolders: survexblock.survexscansfolder = survexscansfolders[0] @@ -125,7 +124,9 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): if expeditions: assert len(expeditions) == 1 survexblock.expedition = expeditions[0] - survexblock.expeditiondate = survexblock.expedition.get_expedition_day(survexblock.date) + survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date) + survexblock.save() + elif re.match("team$(?i)", cmd): mteammember = re.match("(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line) if mteammember: @@ -135,6 +136,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): if (personexpedition, tm) not in teammembers: teammembers.append((personexpedition, tm)) personrole = models.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm) + personrole.expeditionday = survexblock.expeditionday if personexpedition: personrole.person=personexpedition.person personrole.save() diff --git a/parsers/surveys.py b/parsers/surveys.py index 9f7ba10..86f6405 100644 --- a/parsers/surveys.py +++ b/parsers/surveys.py @@ -24,6 +24,7 @@ def get_or_create_placeholder(year): placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs) return placeholder_logbook_entry +# dead def readSurveysFromCSV(): try: # could probably combine these two surveytab = open(os.path.join(settings.SURVEY_SCANS, "Surveys.csv")) @@ -75,6 +76,7 @@ def readSurveysFromCSV(): logging.info("added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r") +# dead def listdir(*directories): try: return os.listdir(os.path.join(settings.SURVEYS, *directories)) @@ -134,11 +136,13 @@ def parseSurveyScans(year, logfile=None): continue scanObj.save() +# dead def parseSurveys(logfile=None): readSurveysFromCSV() for year in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then parseSurveyScans(year) +# dead def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the thumbnail engine can't handle them (uses PIL) file=Image.open(filePath) print filePath @@ -148,10 +152,11 @@ def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the return False -# handles url or file +# handles url or file, so we can refer to a set of scans on another server def GetListDir(sdir): res = [ ] if sdir[:7] == "http://": + assert False, "Not written" s = urllib.urlopen(sdir) else: for f in os.listdir(sdir): @@ -198,4 +203,30 @@ def LoadListScans(surveyscansdir): survexscansingle.save() - \ No newline at end of file + +def LoadTunnelFiles(tunneldatadir): + TunnelFile.objects.all().delete() + tunneldirs = [ "" ] + while tunneldirs: + tunneldir = tunneldirs.pop() + for f in os.listdir(os.path.join(tunneldatadir, tunneldir)): + if f[0] == "." or f[-1] == "~": + continue + lf = os.path.join(tunneldir, f) + ff = os.path.join(tunneldatadir, lf) + if os.path.isdir(ff): + tunneldirs.append(lf) + elif f[-4:] == ".xml": + fin = open(ff) + mtype = re.search("<(fontcolours|sketch)", fin.read(200)) + assert mtype, lf + fin.close() + tunnelfile = TunnelFile(tunnelpath=lf, bfontcolours=(mtype.group(1)=="fontcolours")) + tunnelfile.save() + + +# survexscans = models.ManyToManyField("SurvexScanSingle") +# survexblocks = models.ManyToManyField("SurvexBlock") +# tunnelcontains = models.ManyToManyField("TunnelFile") # case when its a frame type + + diff --git a/templates/base.html b/templates/base.html index 10e670c..302aa25 100644 --- a/templates/base.html +++ b/templates/base.html @@ -18,6 +18,7 @@