working maps: cave -> desc, survey -> cave, expedition -> person. Added /millnialpeople/ page.

core/methods_millenial.py

@@ -0,0 +1,24 @@
+import utm
+import math
+from django.conf import settings
+
+def lat_lon_entrance(utmstring):
+    try:    
+        x = float(utmstring.split()[0])
+        y = float(utmstring.split()[1])
+        #return ' '+str(x+y)+' '+str(y)
+        q = utm.to_latlon(x, y, 33, 'U')
+        return "{:.5f} {:.5f}".format(q[0],q[1])
+    except:
+        return 'Not found'
+
+def top_camp_distance(utmstring):
+    try:    
+        x = float(utmstring.split()[0])
+        y = float(utmstring.split()[1])
+        tx = settings.TOPCAMPX
+        ty = settings.TOPCAMPY
+        dist = math.sqrt( (tx-x)*(tx-x) + (ty-y)*(ty-y) )
+        return "{:.1f}".format(dist)
+    except:
+        return 'Not found'

core/models.py

@@ -15,849 +15,8 @@ from django.template import Context, loader
 import settings
 getcontext().prec=2 #use 2 significant figures for decimal calculations
 
-from troggle.core.models_survex import *
+from troggle.core.models_survex import * #ancient models for both survex and other things
+from troggle.core.models_old import *
 
 
-def get_related_by_wikilinks(wiki_text):
-    found=re.findall(settings.QM_PATTERN,wiki_text)
-    res=[]
-    for wikilink in found:
-        qmdict={'urlroot':settings.URL_ROOT,'cave':wikilink[2],'year':wikilink[1],'number':wikilink[3]}
-        try:
-            cave_slugs = CaveSlug.objects.filter(cave__kataster_number = qmdict['cave'])
-            qm=QM.objects.get(found_by__cave_slug__in = cave_slugs,
-                              found_by__date__year = qmdict['year'],
-                              number = qmdict['number'])
-            res.append(qm)         
-        except QM.DoesNotExist:
-            print('fail on '+str(wikilink))
-    
-    return res
-
-try:
-      logging.basicConfig(level=logging.DEBUG,
-                           filename=settings.LOGFILE,
-                           filemode='w')
-except:
-      subprocess.call(settings.FIX_PERMISSIONS) 
-      logging.basicConfig(level=logging.DEBUG,
-                           filename=settings.LOGFILE,
-                           filemode='w')
-
-#This class is for adding fields and methods which all of our models will have.
-class TroggleModel(models.Model):
-    new_since_parsing = models.BooleanField(default=False, editable=False)
-    non_public = models.BooleanField(default=False)
-    def object_name(self):
-        return self._meta.object_name
-
-    def get_admin_url(self):
-        return urlparse.urljoin(settings.URL_ROOT, "/admin/core/" + self.object_name().lower() + "/" + str(self.pk))
-
-    class Meta:
-        abstract = True
-
-class TroggleImageModel(ImageModel):
-    new_since_parsing = models.BooleanField(default=False, editable=False)
-    
-    def object_name(self):
-        return self._meta.object_name
-
-    def get_admin_url(self):
-        return urlparse.urljoin(settings.URL_ROOT, "/admin/core/" + self.object_name().lower() + "/" + str(self.pk))
-
-
-    class Meta:
-        abstract = True
-
-# 
-# single Expedition, usually seen by year
-#
-class Expedition(TroggleModel):
-    year        = models.CharField(max_length=20, unique=True)
-    name        = models.CharField(max_length=100)
-        
-    def __unicode__(self):
-        return self.year
-
-    class Meta:
-        ordering = ('-year',)
-        get_latest_by = 'year'
-    
-    def get_absolute_url(self):
-        return urlparse.urljoin(settings.URL_ROOT, reverse('expedition', args=[self.year]))
-    
-    # construction function.  should be moved out
-    def get_expedition_day(self, date):
-        expeditiondays = self.expeditionday_set.filter(date=date)
-        if expeditiondays:
-            assert len(expeditiondays) == 1
-            return expeditiondays[0]
-        res = ExpeditionDay(expedition=self, date=date)
-        res.save()
-        return res
-        
-    def day_min(self):
-        res = self.expeditionday_set.all()
-        return res and res[0] or None
-    
-    def day_max(self):
-        res = self.expeditionday_set.all()
-        return res and res[len(res) - 1] or None
-        
-        
-
-class ExpeditionDay(TroggleModel):
-    expedition  = models.ForeignKey("Expedition")
-    date        = models.DateField()
-
-    class Meta:
-        ordering = ('date',)  
-
-    def GetPersonTrip(self, personexpedition):
-        personexpeditions = self.persontrip_set.filter(expeditionday=self)
-        return personexpeditions and personexpeditions[0] or None
-    
-        
-#
-# single Person, can go on many years
-#
-class Person(TroggleModel):
-    first_name  = models.CharField(max_length=100)
-    last_name   = models.CharField(max_length=100)
-    is_vfho     = models.BooleanField(help_text="VFHO is the Vereines für Höhlenkunde in Obersteier, a nearby Austrian caving club.", default=False)
-    mug_shot    = models.CharField(max_length=100, blank=True,null=True)
-    blurb = models.TextField(blank=True,null=True)
-    
-    #href        = models.CharField(max_length=200)
-    orderref    = models.CharField(max_length=200)  # for alphabetic 
-    
-    #the below have been removed and made methods. I'm not sure what the b in bisnotable stands for. - AC 16 Feb
-    #notability  = models.FloatField()               # for listing the top 20 people
-    #bisnotable  = models.BooleanField(default=False)
-    user	= models.OneToOneField(User, null=True, blank=True)
-    def get_absolute_url(self):
-        return urlparse.urljoin(settings.URL_ROOT,reverse('person',kwargs={'first_name':self.first_name,'last_name':self.last_name}))
-
-    class Meta:
-        verbose_name_plural = "People"
-        ordering = ('orderref',)  # "Wookey" makes too complex for: ('last_name', 'first_name') 
-    
-    def __unicode__(self):
-        if self.last_name:
-            return "%s %s" % (self.first_name, self.last_name)
-        return self.first_name
-
-    
-    def notability(self):
-        notability = Decimal(0)
-        for personexpedition in self.personexpedition_set.all():
-             if not personexpedition.is_guest:
-                notability += Decimal(1) / (2012 - int(personexpedition.expedition.year))
-        return notability
-
-    def bisnotable(self):
-        return self.notability() > Decimal(1)/Decimal(3)
-    
-    def surveyedleglength(self):
-        return sum([personexpedition.surveyedleglength()  for personexpedition in self.personexpedition_set.all()])
-    
-    def first(self):
-        return self.personexpedition_set.order_by('-expedition')[0]
-    def last(self):
-        return self.personexpedition_set.order_by('expedition')[0]
-    
-    #def Sethref(self):
-        #if self.last_name:
-            #self.href = self.first_name.lower() + "_" + self.last_name.lower()
-            #self.orderref = self.last_name + " " + self.first_name
-        #else:
-          #  self.href = self.first_name.lower()
-            #self.orderref = self.first_name
-        #self.notability = 0.0  # set temporarily
-        
-
-#
-# Person's attenance to one Expo
-#
-class PersonExpedition(TroggleModel):
-    expedition  = models.ForeignKey(Expedition)
-    person      = models.ForeignKey(Person)
-    slugfield   = models.SlugField(max_length=50,blank=True,null=True)
-    
-    is_guest    = models.BooleanField(default=False)  
-    COMMITTEE_CHOICES = (
-        ('leader','Expo leader'),
-        ('medical','Expo medical officer'),
-        ('treasurer','Expo treasurer'),
-        ('sponsorship','Expo sponsorship coordinator'),
-        ('research','Expo research coordinator'),
-        )
-    expo_committee_position = models.CharField(blank=True,null=True,choices=COMMITTEE_CHOICES,max_length=200)
-    nickname    = models.CharField(max_length=100,blank=True,null=True)
-    
-    def GetPersonroles(self):
-        res = [ ]
-        for personrole in self.personrole_set.order_by('survexblock'):
-            if res and res[-1]['survexpath'] == personrole.survexblock.survexpath:
-                res[-1]['roles'] += ", " + str(personrole.role)
-            else:
-                res.append({'date':personrole.survexblock.date, 'survexpath':personrole.survexblock.survexpath, 'roles':str(personrole.role)})
-        return res
-
-    class Meta:
-        ordering = ('-expedition',)
-        #order_with_respect_to = 'expedition'
-
-    def __unicode__(self):
-        return "%s: (%s)" % (self.person, self.expedition)
-    
-    
-    #why is the below a function in personexpedition, rather than in person? - AC 14 Feb 09
-    def name(self):
-        if self.nickname:
-            return "%s (%s) %s" % (self.person.first_name, self.nickname, self.person.last_name)
-        if self.person.last_name:
-            return "%s %s" % (self.person.first_name, self.person.last_name)
-        return self.person.first_name
-
-    def get_absolute_url(self):
-        return urlparse.urljoin(settings.URL_ROOT, reverse('personexpedition',kwargs={'first_name':self.person.first_name,'last_name':self.person.last_name,'year':self.expedition.year}))
-	
-    def surveyedleglength(self):
-        survexblocks = [personrole.survexblock  for personrole in self.personrole_set.all() ]
-        return sum([survexblock.totalleglength  for survexblock in set(survexblocks)])
-    
-    # would prefer to return actual person trips so we could link to first and last ones
-    def day_min(self):
-        res = self.persontrip_set.aggregate(day_min=Min("expeditionday__date"))
-        return res["day_min"]
-
-    def day_max(self):
-        res = self.persontrip_set.all().aggregate(day_max=Max("expeditionday__date"))
-        return res["day_max"]
-
-#
-# Single parsed entry from Logbook
-#    
-class LogbookEntry(TroggleModel):
-    date    = models.DateField()#MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.
-    expeditionday = models.ForeignKey("ExpeditionDay", null=True)#MJG wants to KILL THIS (redundant information)
-    expedition  = models.ForeignKey(Expedition,blank=True,null=True)  # yes this is double-
-    #author  = models.ForeignKey(PersonExpedition,blank=True,null=True)  # the person who writes it up doesn't have to have been on the trip.
-    # Re: the above- so this field should be "typist" or something, not "author". - AC 15 jun 09
-    #MJG wants to KILL THIS, as it is typically redundant with PersonTrip.is_logbook_entry_author, in the rare it was not redundanty and of actually interest it could be added to the text.
-    title     = models.CharField(max_length=settings.MAX_LOGBOOK_ENTRY_TITLE_LENGTH)
-    cave_slug = models.SlugField(max_length=50)
-    place     = models.CharField(max_length=100,blank=True,null=True,help_text="Only use this if you haven't chosen a cave")
-    text      = models.TextField()
-    slug      = models.SlugField(max_length=50)
-    filename  = models.CharField(max_length=200,null=True)
-
-    class Meta:
-        verbose_name_plural = "Logbook Entries"
-        # several PersonTrips point in to this object
-        ordering = ('-date',)
-        
-    def __getattribute__(self, item):
-      if item == "cave":  #Allow a logbookentries cave to be directly accessed despite not having a proper foreignkey
-        return CaveSlug.objects.get(slug = self.cave_slug).cave
-      return super(LogbookEntry, self).__getattribute__(item)
-
-    def __init__(self, *args, **kwargs):
-        if "cave" in kwargs.keys():
-            if kwargs["cave"] is not None:
-                kwargs["cave_slug"] = CaveSlug.objects.get(cave=kwargs["cave"], primary=True).slug
-            kwargs.pop("cave")
-        return super(LogbookEntry, self).__init__(*args, **kwargs)
-
-    def isLogbookEntry(self): # Function used in templates
-        return True
-
-    def get_absolute_url(self):
-        return urlparse.urljoin(settings.URL_ROOT, reverse('logbookentry',kwargs={'date':self.date,'slug':self.slug}))
-
-    def __unicode__(self):
-        return "%s: (%s)" % (self.date, self.title)
-
-    def get_next_by_id(self):
-        LogbookEntry.objects.get(id=self.id+1)
-
-    def get_previous_by_id(self):
-        LogbookEntry.objects.get(id=self.id-1)
-
-    def new_QM_number(self):
-        """Returns  """
-        if self.cave:
-            nextQMnumber=self.cave.new_QM_number(self.date.year)
-        else:
-            return none
-        return nextQMnumber
-
-    def new_QM_found_link(self):
-        """Produces a link to a new QM with the next number filled in and this LogbookEntry set as 'found by' """
-        return settings.URL_ROOT + r'/admin/core/qm/add/?' + r'found_by=' + str(self.pk) +'&number=' + str(self.new_QM_number())
-
-    def DayIndex(self):
-        return list(self.expeditionday.logbookentry_set.all()).index(self)
-
-#
-# Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry)
-#
-class PersonTrip(TroggleModel):
-    personexpedition = models.ForeignKey("PersonExpedition",null=True)
-    
-    #expeditionday    = models.ForeignKey("ExpeditionDay")#MJG wants to KILL THIS (redundant information)
-    #date             = models.DateField() #MJG wants to KILL THIS (redundant information)
-    time_underground = models.FloatField(help_text="In decimal hours")
-    logbook_entry    = models.ForeignKey(LogbookEntry)
-    is_logbook_entry_author = models.BooleanField(default=False)
-    
-    
-    # sequencing by person (difficult to solve locally)
-    #persontrip_next  = models.ForeignKey('PersonTrip', related_name='pnext', blank=True,null=True)#MJG wants to KILL THIS (and use funstion persontrip_next_auto)
-    #persontrip_prev  = models.ForeignKey('PersonTrip', related_name='pprev', blank=True,null=True)#MJG wants to KILL THIS(and use funstion persontrip_prev_auto)
-    
-    def persontrip_next(self):
-        futurePTs = PersonTrip.objects.filter(personexpedition = self.personexpedition, logbook_entry__date__gt = self.logbook_entry.date).order_by('logbook_entry__date').all()
-        if len(futurePTs) > 0:
-            return futurePTs[0]
-        else:
-            return None
-
-    def persontrip_prev(self):
-        pastPTs = PersonTrip.objects.filter(personexpedition = self.personexpedition, logbook_entry__date__lt = self.logbook_entry.date).order_by('-logbook_entry__date').all()
-        if len(pastPTs) > 0:
-            return pastPTs[0]
-        else:
-            return None
-
-    def place(self):
-        return self.logbook_entry.cave and self.logbook_entry.cave or self.logbook_entry.place
-
-    def __unicode__(self):
-        return "%s (%s)" % (self.personexpedition, self.logbook_entry.date)
-    
-
-
-##########################################
-# move following classes into models_cave
-##########################################
-
-class Area(TroggleModel):
-    short_name = models.CharField(max_length=100)
-    name = models.CharField(max_length=200, blank=True, null=True)
-    description = models.TextField(blank=True,null=True)
-    parent = models.ForeignKey('Area', blank=True, null=True)
-    def __unicode__(self):
-        if self.parent:
-            return unicode(self.parent) + u" - " + unicode(self.short_name)
-        else:
-            return unicode(self.short_name)
-    def kat_area(self):
-        if self.short_name in ["1623", "1626"]:
-            return self.short_name
-        elif self.parent:
-            return self.parent.kat_area()
-
-class CaveAndEntrance(models.Model):
-    cave = models.ForeignKey('Cave')
-    entrance = models.ForeignKey('Entrance')
-    entrance_letter = models.CharField(max_length=20,blank=True,null=True)
-    def __unicode__(self):
-        return unicode(self.cave) + unicode(self.entrance_letter)
-        
-class CaveSlug(models.Model):
-    cave = models.ForeignKey('Cave')
-    slug = models.SlugField(max_length=50, unique = True)
-    primary = models.BooleanField(default=False)
-    
-
-class Cave(TroggleModel):
-    # too much here perhaps, 
-    official_name = models.CharField(max_length=160)
-    area = models.ManyToManyField(Area, blank=True, null=True)
-    kataster_code = models.CharField(max_length=20,blank=True,null=True)
-    kataster_number = models.CharField(max_length=10,blank=True, null=True)
-    unofficial_number = models.CharField(max_length=60,blank=True, null=True)
-    entrances = models.ManyToManyField('Entrance', through='CaveAndEntrance')
-    explorers = models.TextField(blank=True,null=True)
-    underground_description = models.TextField(blank=True,null=True)
-    equipment = models.TextField(blank=True,null=True)
-    references = models.TextField(blank=True,null=True)
-    survey = models.TextField(blank=True,null=True)
-    kataster_status = models.TextField(blank=True,null=True)
-    underground_centre_line = models.TextField(blank=True,null=True)
-    notes = models.TextField(blank=True,null=True)
-    length = models.CharField(max_length=100,blank=True,null=True)
-    depth = models.CharField(max_length=100,blank=True,null=True)
-    extent = models.CharField(max_length=100,blank=True,null=True)
-    survex_file = models.CharField(max_length=100,blank=True,null=True)
-    description_file = models.CharField(max_length=200,blank=True,null=True)
-    url = models.CharField(max_length=200,blank=True,null=True)
-    filename = models.CharField(max_length=200)
-
-
-    #class Meta:
-    #    unique_together = (("area", "kataster_number"), ("area", "unofficial_number"))
-    # FIXME Kataster Areas and CUCC defined sub areas need seperating    
-
-
-    #href    = models.CharField(max_length=100)
-    
-    class Meta:
-        ordering = ('kataster_code', 'unofficial_number')  
-
-    def hassurvey(self):
-        if not self.underground_centre_line:
-            return "No"
-        if (self.survey.find("<img") > -1 or self.survey.find("<a") > -1 or self.survey.find("<IMG") > -1 or self.survey.find("<A") > -1):
-            return "Yes"
-        return "Missing"
-
-    def hassurveydata(self):
-        if not self.underground_centre_line:
-            return "No"
-        if self.survex_file:
-            return "Yes"
-        return "Missing"
-         
-    def slug(self):
-        primarySlugs = self.caveslug_set.filter(primary = True)
-        if primarySlugs:
-            return primarySlugs[0].slug
-        else:
-            slugs = self.caveslug_set.filter()
-            if slugs:
-                return slugs[0].slug
-
-    def ours(self):
-        return bool(re.search(r'CUCC', self.explorers))
-
-    def reference(self):
-        if self.kataster_number:
-            return "%s-%s" % (self.kat_area(), self.kataster_number)
-        else:
-            return "%s-%s" % (self.kat_area(), self.unofficial_number)
-    
-    def get_absolute_url(self):
-        if self.kataster_number:
-            href = self.kataster_number
-        elif self.unofficial_number:
-            href = self.unofficial_number
-        else:
-            href = official_name.lower()
-        #return settings.URL_ROOT + '/cave/' + href + '/'
-        return urlparse.urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,}))
-
-    def __unicode__(self, sep = u": "):
-        return unicode(self.slug())
-
-    def get_QMs(self):
-        return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all())	
-
-    def new_QM_number(self, year=datetime.date.today().year):
-            """Given a cave and the current year, returns the next QM number."""
-            try:
-                res=QM.objects.filter(found_by__date__year=year, found_by__cave=self).order_by('-number')[0]
-            except IndexError:
-                return 1
-            return res.number+1
-
-    def kat_area(self):
-        for a in self.area.all():
-            if a.kat_area():
-                return a.kat_area()
-    
-    def entrances(self):
-        return CaveAndEntrance.objects.filter(cave=self)
-
-    def singleentrance(self):
-        return len(CaveAndEntrance.objects.filter(cave=self)) == 1
-    
-    def entrancelist(self):
-        rs = []
-        res = ""
-        for e in CaveAndEntrance.objects.filter(cave=self):
-            rs.append(e.entrance_letter)
-        rs.sort()
-        prevR = None
-        n = 0
-        for r in rs:
-            if prevR:
-                if chr(ord(prevR) + 1 ) == r:
-                    prevR = r
-                    n += 1
-                else:
-                    if n == 0:
-                        res += ", " + prevR
-                    else:
-                        res += "&ndash;" + prevR
-            else:
-                prevR = r
-                n = 0
-                res += r
-        if n == 0:
-            res += ", " + prevR
-        else:
-            res += "&ndash;" + prevR
-        return res
-    
-    def writeDataFile(self):
-        try:
-            f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "w")
-        except:
-            subprocess.call(settings.FIX_PERMISSIONS) 
-            f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "w")
-        t = loader.get_template('dataformat/cave.xml')
-        c = Context({'cave': self})
-        u = t.render(c)
-        u8 = u.encode("utf-8")
-        f.write(u8)
-        f.close()
-        
-    def getArea(self):
-        areas = self.area.all()
-        lowestareas = list(areas)
-        for area in areas:
-            if area.parent in areas:
-                try:
-                    lowestareas.remove(area.parent)
-                except:
-                    pass
-        return lowestareas[0]
-
-def getCaveByReference(reference):
-    areaname, code = reference.split("-", 1)
-    print(areaname, code)
-    area = Area.objects.get(short_name = areaname)
-    print(area)
-    foundCaves = list(Cave.objects.filter(area = area,  kataster_number = code).all()) + list(Cave.objects.filter(area = area,  unofficial_number = code).all()) 
-    print(list(foundCaves))
-    assert len(foundCaves) == 1
-    return foundCaves[0]
-
-class OtherCaveName(TroggleModel):
-    name = models.CharField(max_length=160)
-    cave = models.ForeignKey(Cave)
-    def __unicode__(self):
-        return unicode(self.name)
-            
-class EntranceSlug(models.Model):
-    entrance = models.ForeignKey('Entrance')
-    slug = models.SlugField(max_length=50, unique = True)
-    primary = models.BooleanField(default=False)
-
-class Entrance(TroggleModel):
-    name = models.CharField(max_length=100, blank=True,null=True)
-    entrance_description = models.TextField(blank=True,null=True)
-    explorers = models.TextField(blank=True,null=True)
-    map_description = models.TextField(blank=True,null=True)
-    location_description = models.TextField(blank=True,null=True)
-    approach = models.TextField(blank=True,null=True)
-    underground_description = models.TextField(blank=True,null=True)
-    photo = models.TextField(blank=True,null=True)
-    MARKING_CHOICES = (
-        ('P', 'Paint'),
-        ('P?', 'Paint (?)'),
-        ('T', 'Tag'),
-        ('T?', 'Tag (?)'),
-        ('R', 'Needs Retag'),
-        ('S', 'Spit'),
-        ('S?', 'Spit (?)'),
-        ('U', 'Unmarked'),
-        ('?', 'Unknown'))
-    marking = models.CharField(max_length=2, choices=MARKING_CHOICES)
-    marking_comment = models.TextField(blank=True,null=True)
-    FINDABLE_CHOICES = (
-        ('?', 'To be confirmed ...'),
-        ('S', 'Coordinates'),
-        ('L', 'Lost'),
-        ('R', 'Refindable'))
-    findability = models.CharField(max_length=1, choices=FINDABLE_CHOICES, blank=True, null=True)
-    findability_description = models.TextField(blank=True,null=True)
-    alt = models.TextField(blank=True, null=True)
-    northing = models.TextField(blank=True, null=True)
-    easting = models.TextField(blank=True, null=True)
-    tag_station = models.TextField(blank=True, null=True)
-    exact_station = models.TextField(blank=True, null=True)
-    other_station = models.TextField(blank=True, null=True)
-    other_description = models.TextField(blank=True,null=True)
-    bearings = models.TextField(blank=True,null=True)
-    url = models.CharField(max_length=200,blank=True,null=True)
-    filename = models.CharField(max_length=200)
-    cached_primary_slug = models.CharField(max_length=200,blank=True,null=True)
-
-    def __unicode__(self):
-        return unicode(self.slug())
-
-    def exact_location(self):
-        return SurvexStation.objects.lookup(self.exact_station)
-    def other_location(self):
-        return SurvexStation.objects.lookup(self.other_station)
-
-
-    def find_location(self):
-        r = {'':  'To be entered ',
-			 '?': 'To be confirmed:',
-             'S': '',
-             'L': 'Lost:',
-             'R': 'Refindable:'}[self.findability]
-        if self.tag_station:
-            try:
-                s = SurvexStation.objects.lookup(self.tag_station)
-                return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z)
-            except:
-                return r + "%s Tag Station not in dataset" % self.tag_station
-        if self.exact_station:
-            try:
-                s = SurvexStation.objects.lookup(self.exact_station)
-                return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z)
-            except:
-                return r + "%s Exact Station not in dataset" % self.tag_station
-        if self.other_station:
-            try:
-                s = SurvexStation.objects.lookup(self.other_station)
-                return r + "%0.0fE %0.0fN %0.0fAlt %s" % (s.x, s.y, s.z, self.other_description)
-            except:
-                return r + "%s Other Station not in dataset" % self.tag_station
-        if self.FINDABLE_CHOICES == "S":
-            r += "ERROR, Entrance has been surveyed but has no survex point"
-        if self.bearings:
-            return r + self.bearings
-        return r
-
-    def best_station(self):
-        if self.tag_station:
-            return self.tag_station
-        if self.exact_station:
-            return self.exact_station
-        if self.other_station:
-            return self.other_station
-
-    def has_photo(self):
-        if self.photo:
-            if (self.photo.find("<img") > -1 or self.photo.find("<a") > -1 or self.photo.find("<IMG") > -1 or self.photo.find("<A") > -1):
-                return "Yes"
-            else:
-                return "Missing"
-        else:
-            return "No"
-
-    def marking_val(self):
-        for m in self.MARKING_CHOICES:
-            if m[0] == self.marking:
-                return m[1]
-    def findability_val(self):
-        for f in self.FINDABLE_CHOICES:
-            if f[0] == self.findability:
-                return f[1]
-                
-    def tag(self):
-        return SurvexStation.objects.lookup(self.tag_station)
-        
-    def needs_surface_work(self):
-        return self.findability != "S" or not self.has_photo or self.marking != "T"
-
-    def get_absolute_url(self):
-        
-        ancestor_titles='/'.join([subcave.title for subcave in self.get_ancestors()])
-        if ancestor_titles:
-            res = '/'.join((self.get_root().cave.get_absolute_url(), ancestor_titles, self.title))
-        
-        else:
-            res = '/'.join((self.get_root().cave.get_absolute_url(), self.title))
-            
-        return res
-
-    def slug(self):
-        if not self.cached_primary_slug:
-            primarySlugs = self.entranceslug_set.filter(primary = True)
-            if primarySlugs:    
-                self.cached_primary_slug = primarySlugs[0].slug
-                self.save()
-            else:
-                slugs = self.entranceslug_set.filter()
-                if slugs:
-                    self.cached_primary_slug = slugs[0].slug
-                    self.save()
-        return self.cached_primary_slug
-
-    def writeDataFile(self):
-        try:
-            f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w")
-        except:
-            subprocess.call(settings.FIX_PERMISSIONS) 
-            f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w")
-        t = loader.get_template('dataformat/entrance.xml')
-        c = Context({'entrance': self})
-        u = t.render(c)
-        u8 = u.encode("utf-8")
-        f.write(u8)
-        f.close()
-
-class CaveDescription(TroggleModel):
-    short_name = models.CharField(max_length=50, unique = True)
-    long_name = models.CharField(max_length=200, blank=True, null=True)
-    description = models.TextField(blank=True,null=True)
-    linked_subcaves = models.ManyToManyField("NewSubCave", blank=True,null=True)
-    linked_entrances = models.ManyToManyField("Entrance", blank=True,null=True)
-    linked_qms = models.ManyToManyField("QM", blank=True,null=True)
-
-    def __unicode__(self):
-        if self.long_name:
-            return unicode(self.long_name)
-        else:
-            return unicode(self.short_name)
-    
-    def get_absolute_url(self):
-        return urlparse.urljoin(settings.URL_ROOT, reverse('cavedescription', args=(self.short_name,)))
-    
-    def save(self):
-        """
-        Overridden save method which stores wikilinks in text as links in database.
-        """
-        super(CaveDescription, self).save()
-        qm_list=get_related_by_wikilinks(self.description)
-        for qm in qm_list:
-            self.linked_qms.add(qm)
-        super(CaveDescription, self).save()
-
-class NewSubCave(TroggleModel):
-    name = models.CharField(max_length=200, unique = True)
-    def __unicode__(self):
-        return unicode(self.name)
-
-class QM(TroggleModel):
-    #based on qm.csv in trunk/expoweb/1623/204 which has the fields:
-    #"Number","Grade","Area","Description","Page reference","Nearest station","Completion description","Comment"
-    found_by = models.ForeignKey(LogbookEntry, related_name='QMs_found',blank=True, null=True )
-    ticked_off_by = models.ForeignKey(LogbookEntry, related_name='QMs_ticked_off',null=True,blank=True)
-    #cave = models.ForeignKey(Cave)
-    #expedition = models.ForeignKey(Expedition)
-
-    number = models.IntegerField(help_text="this is the sequential number in the year", )
-    GRADE_CHOICES=(
-	('A', 'A: Large obvious lead'),
-	('B', 'B: Average lead'),
-	('C', 'C: Tight unpromising lead'),
-	('D', 'D: Dig'),
-	('X', 'X: Unclimbable aven')
-    )
-    grade = models.CharField(max_length=1, choices=GRADE_CHOICES)
-    location_description = models.TextField(blank=True)
-    #should be a foreignkey to surveystation
-    nearest_station_description = models.CharField(max_length=400,null=True,blank=True)
-    nearest_station = models.CharField(max_length=200,blank=True,null=True)
-    area = models.CharField(max_length=100,blank=True,null=True)
-    completion_description = models.TextField(blank=True,null=True)
-    comment=models.TextField(blank=True,null=True)
-
-    def __unicode__(self):
-        return u"%s %s" % (self.code(), self.grade)
-
-    def code(self):
-        return u"%s-%s-%s" % (unicode(self.found_by.cave)[6:], self.found_by.date.year, self.number)
-
-    def get_absolute_url(self):
-        #return settings.URL_ROOT + '/cave/' + self.found_by.cave.kataster_number + '/' + str(self.found_by.date.year) + '-' + '%02d' %self.number
-        return urlparse.urljoin(settings.URL_ROOT, reverse('qm',kwargs={'cave_id':self.found_by.cave.kataster_number,'year':self.found_by.date.year,'qm_id':self.number,'grade':self.grade}))
-
-    def get_next_by_id(self):
-        return QM.objects.get(id=self.id+1)
-
-    def get_previous_by_id(self):
-        return QM.objects.get(id=self.id-1)
-
-    def wiki_link(self):
-        return u"%s%s%s" % ('[[QM:',self.code(),']]')
-
-photoFileStorage = FileSystemStorage(location=settings.PHOTOS_ROOT, base_url=settings.PHOTOS_URL)
-class DPhoto(TroggleImageModel): 
-    caption = models.CharField(max_length=1000,blank=True,null=True)
-    contains_logbookentry = models.ForeignKey(LogbookEntry,blank=True,null=True)
-    contains_person = models.ManyToManyField(Person,blank=True,null=True)
-    file = models.ImageField(storage=photoFileStorage, upload_to='.',)
-    is_mugshot = models.BooleanField(default=False)
-    contains_cave = models.ForeignKey(Cave,blank=True,null=True)
-    contains_entrance = models.ForeignKey(Entrance, related_name="photo_file",blank=True,null=True)
-    #nearest_survey_point = models.ForeignKey(SurveyStation,blank=True,null=True)
-    nearest_QM = models.ForeignKey(QM,blank=True,null=True)
-    lon_utm = models.FloatField(blank=True,null=True)
-    lat_utm = models.FloatField(blank=True,null=True)
-    
-    class IKOptions:
-        spec_module = 'core.imagekit_specs'
-        cache_dir = 'thumbs'
-        image_field = 'file'
-        
-    #content_type = models.ForeignKey(ContentType)
-    #object_id = models.PositiveIntegerField()
-    #location = generic.GenericForeignKey('content_type', 'object_id')
-
-    def __unicode__(self):
-        return self.caption
-
-scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL)
-def get_scan_path(instance, filename):
-    year=instance.survey.expedition.year
-    #print("WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number, instance.survey.wallet_letter)
-    number=str(instance.survey.wallet_number)
-    if str(instance.survey.wallet_letter) != "None":
-        number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01
-    return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg')
-
-class ScannedImage(TroggleImageModel): 
-    file = models.ImageField(storage=scansFileStorage, upload_to=get_scan_path)
-    scanned_by = models.ForeignKey(Person,blank=True, null=True)
-    scanned_on = models.DateField(null=True)
-    survey = models.ForeignKey('Survey')
-    contents = models.CharField(max_length=20,choices=(('notes','notes'),('plan','plan_sketch'),('elevation','elevation_sketch')))
-    number_in_wallet = models.IntegerField(null=True)
-    lon_utm = models.FloatField(blank=True,null=True)
-    lat_utm = models.FloatField(blank=True,null=True)
-
-    class IKOptions:
-        spec_module = 'core.imagekit_specs'
-        cache_dir = 'thumbs'
-        image_field = 'file'
-    #content_type = models.ForeignKey(ContentType)
-    #object_id = models.PositiveIntegerField()
-    #location = generic.GenericForeignKey('content_type', 'object_id')
-
-    #This is an ugly hack to deal with the #s in our survey scan paths. The correct thing is to write a custom file storage backend which calls urlencode on the name for making file.url but not file.path.
-    def correctURL(self):
-        return string.replace(self.file.url,r'#',r'%23')
-    
-    def __unicode__(self):
-        return get_scan_path(self,'')
-
-class Survey(TroggleModel):
-    expedition = models.ForeignKey('Expedition') #REDUNDANT (logbook_entry)
-    wallet_number = models.IntegerField(blank=True,null=True)
-    wallet_letter = models.CharField(max_length=1,blank=True,null=True)
-    comments = models.TextField(blank=True,null=True)
-    location = models.CharField(max_length=400,blank=True,null=True) #REDUNDANT
-    subcave = models.ForeignKey('NewSubCave', blank=True, null=True)
-    #notes_scan = models.ForeignKey('ScannedImage',related_name='notes_scan',blank=True, null=True)  	#Replaced by contents field of ScannedImage model
-    survex_block  = models.OneToOneField('SurvexBlock',blank=True, null=True)
-    logbook_entry = models.ForeignKey('LogbookEntry')
-    centreline_printed_on = models.DateField(blank=True, null=True)
-    centreline_printed_by = models.ForeignKey('Person',related_name='centreline_printed_by',blank=True,null=True)
-    #sketch_scan = models.ForeignKey(ScannedImage,blank=True, null=True) 					#Replaced by contents field of ScannedImage model
-    tunnel_file = models.FileField(upload_to='surveyXMLfiles',blank=True, null=True)
-    tunnel_main_sketch = models.ForeignKey('Survey',blank=True,null=True)
-    integrated_into_main_sketch_on = models.DateField(blank=True,null=True)
-    integrated_into_main_sketch_by = models.ForeignKey('Person' ,related_name='integrated_into_main_sketch_by', blank=True,null=True)
-    rendered_image = models.ImageField(upload_to='renderedSurveys',blank=True,null=True)
-    def __unicode__(self):
-        return self.expedition.year+"#"+"%02d" % int(self.wallet_number)
-
-    def notes(self):
-        return self.scannedimage_set.filter(contents='notes')
-
-    def plans(self):
-        return self.scannedimage_set.filter(contents='plan')
-
-    def elevations(self):
-        return self.scannedimage_set.filter(contents='elevation')
+from troggle.core.models_millenial import * #updated models are here

core/models.py.old

@@ -0,0 +1,864 @@
+import urllib, urlparse, string, os, datetime, logging, re
+import subprocess
+from django.forms import ModelForm
+from django.db import models
+from django.contrib import admin
+from django.core.files.storage import FileSystemStorage
+from django.contrib.auth.models import User
+from django.contrib.contenttypes.models import ContentType
+from django.db.models import Min, Max
+from django.conf import settings
+from decimal import Decimal, getcontext
+from django.core.urlresolvers import reverse
+from imagekit.models import ImageModel
+from django.template import Context, loader
+import settings
+getcontext().prec=2 #use 2 significant figures for decimal calculations
+
+from troggle.core.models_survex import *
+
+from troggle.core.models_millenial import *
+
+def get_related_by_wikilinks(wiki_text):
+    found=re.findall(settings.QM_PATTERN,wiki_text)
+    res=[]
+    for wikilink in found:
+        qmdict={'urlroot':settings.URL_ROOT,'cave':wikilink[2],'year':wikilink[1],'number':wikilink[3]}
+        try:
+            cave_slugs = CaveSlug.objects.filter(cave__kataster_number = qmdict['cave'])
+            qm=QM.objects.get(found_by__cave_slug__in = cave_slugs,
+                              found_by__date__year = qmdict['year'],
+                              number = qmdict['number'])
+            res.append(qm)         
+        except QM.DoesNotExist:
+            print('fail on '+str(wikilink))
+    
+    return res
+
+try:
+      logging.basicConfig(level=logging.DEBUG,
+                           filename=settings.LOGFILE,
+                           filemode='w')
+except:
+      subprocess.call(settings.FIX_PERMISSIONS) 
+      logging.basicConfig(level=logging.DEBUG,
+                           filename=settings.LOGFILE,
+                           filemode='w')
+
+#This class is for adding fields and methods which all of our models will have.
+class TroggleModel(models.Model):
+    new_since_parsing = models.BooleanField(default=False, editable=False)
+    non_public = models.BooleanField(default=False)
+    def object_name(self):
+        return self._meta.object_name
+
+    def get_admin_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, "/admin/core/" + self.object_name().lower() + "/" + str(self.pk))
+
+    class Meta:
+        abstract = True
+
+class TroggleImageModel(ImageModel):
+    new_since_parsing = models.BooleanField(default=False, editable=False)
+    
+    def object_name(self):
+        return self._meta.object_name
+
+    def get_admin_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, "/admin/core/" + self.object_name().lower() + "/" + str(self.pk))
+
+
+    class Meta:
+        abstract = True
+
+# 
+# single Expedition, usually seen by year
+#
+class Expedition(TroggleModel):
+    year        = models.CharField(max_length=20, unique=True)
+    name        = models.CharField(max_length=100)
+        
+    def __unicode__(self):
+        return self.year
+
+    class Meta:
+        ordering = ('-year',)
+        get_latest_by = 'year'
+    
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, reverse('expedition', args=[self.year]))
+    
+    # construction function.  should be moved out
+    def get_expedition_day(self, date):
+        expeditiondays = self.expeditionday_set.filter(date=date)
+        if expeditiondays:
+            assert len(expeditiondays) == 1
+            return expeditiondays[0]
+        res = ExpeditionDay(expedition=self, date=date)
+        res.save()
+        return res
+        
+    def day_min(self):
+        res = self.expeditionday_set.all()
+        return res and res[0] or None
+    
+    def day_max(self):
+        res = self.expeditionday_set.all()
+        return res and res[len(res) - 1] or None
+        
+        
+
+class ExpeditionDay(TroggleModel):
+    expedition  = models.ForeignKey("Expedition")
+    date        = models.DateField()
+
+    class Meta:
+        ordering = ('date',)  
+
+    def GetPersonTrip(self, personexpedition):
+        personexpeditions = self.persontrip_set.filter(expeditionday=self)
+        return personexpeditions and personexpeditions[0] or None
+    
+        
+#
+# single Person, can go on many years
+#
+class Person(TroggleModel):
+    first_name  = models.CharField(max_length=100)
+    last_name   = models.CharField(max_length=100)
+    is_vfho     = models.BooleanField(help_text="VFHO is the Vereines für Höhlenkunde in Obersteier, a nearby Austrian caving club.", default=False)
+    mug_shot    = models.CharField(max_length=100, blank=True,null=True)
+    blurb = models.TextField(blank=True,null=True)
+    
+    #href        = models.CharField(max_length=200)
+    orderref    = models.CharField(max_length=200)  # for alphabetic 
+    
+    #the below have been removed and made methods. I'm not sure what the b in bisnotable stands for. - AC 16 Feb
+    #notability  = models.FloatField()               # for listing the top 20 people
+    #bisnotable  = models.BooleanField(default=False)
+    user	= models.OneToOneField(User, null=True, blank=True)
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT,reverse('person',kwargs={'first_name':self.first_name,'last_name':self.last_name}))
+
+    class Meta:
+        verbose_name_plural = "People"
+        ordering = ('orderref',)  # "Wookey" makes too complex for: ('last_name', 'first_name') 
+    
+    def __unicode__(self):
+        if self.last_name:
+            return "%s %s" % (self.first_name, self.last_name)
+        return self.first_name
+
+    
+    def notability(self):
+        notability = Decimal(0)
+        for personexpedition in self.personexpedition_set.all():
+             if not personexpedition.is_guest:
+                notability += Decimal(1) / (2012 - int(personexpedition.expedition.year))
+        return notability
+
+    def bisnotable(self):
+        return self.notability() > Decimal(1)/Decimal(3)
+    
+    def surveyedleglength(self):
+        return sum([personexpedition.surveyedleglength()  for personexpedition in self.personexpedition_set.all()])
+    
+    def first(self):
+        return self.personexpedition_set.order_by('-expedition')[0]
+    def last(self):
+        return self.personexpedition_set.order_by('expedition')[0]
+    
+    #def Sethref(self):
+        #if self.last_name:
+            #self.href = self.first_name.lower() + "_" + self.last_name.lower()
+            #self.orderref = self.last_name + " " + self.first_name
+        #else:
+          #  self.href = self.first_name.lower()
+            #self.orderref = self.first_name
+        #self.notability = 0.0  # set temporarily
+        
+
+#
+# Person's attenance to one Expo
+#
+class PersonExpedition(TroggleModel):
+    expedition  = models.ForeignKey(Expedition)
+    person      = models.ForeignKey(Person)
+    slugfield   = models.SlugField(max_length=50,blank=True,null=True)
+    
+    is_guest    = models.BooleanField(default=False)  
+    COMMITTEE_CHOICES = (
+        ('leader','Expo leader'),
+        ('medical','Expo medical officer'),
+        ('treasurer','Expo treasurer'),
+        ('sponsorship','Expo sponsorship coordinator'),
+        ('research','Expo research coordinator'),
+        )
+    expo_committee_position = models.CharField(blank=True,null=True,choices=COMMITTEE_CHOICES,max_length=200)
+    nickname    = models.CharField(max_length=100,blank=True,null=True)
+    
+    def GetPersonroles(self):
+        res = [ ]
+        for personrole in self.personrole_set.order_by('survexblock'):
+            if res and res[-1]['survexpath'] == personrole.survexblock.survexpath:
+                res[-1]['roles'] += ", " + str(personrole.role)
+            else:
+                res.append({'date':personrole.survexblock.date, 'survexpath':personrole.survexblock.survexpath, 'roles':str(personrole.role)})
+        return res
+
+    class Meta:
+        ordering = ('-expedition',)
+        #order_with_respect_to = 'expedition'
+
+    def __unicode__(self):
+        return "%s: (%s)" % (self.person, self.expedition)
+    
+    
+    #why is the below a function in personexpedition, rather than in person? - AC 14 Feb 09
+    def name(self):
+        if self.nickname:
+            return "%s (%s) %s" % (self.person.first_name, self.nickname, self.person.last_name)
+        if self.person.last_name:
+            return "%s %s" % (self.person.first_name, self.person.last_name)
+        return self.person.first_name
+
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, reverse('personexpedition',kwargs={'first_name':self.person.first_name,'last_name':self.person.last_name,'year':self.expedition.year}))
+	
+    def surveyedleglength(self):
+        survexblocks = [personrole.survexblock  for personrole in self.personrole_set.all() ]
+        return sum([survexblock.totalleglength  for survexblock in set(survexblocks)])
+    
+    # would prefer to return actual person trips so we could link to first and last ones
+    def day_min(self):
+        res = self.persontrip_set.aggregate(day_min=Min("expeditionday__date"))
+        return res["day_min"]
+
+    def day_max(self):
+        res = self.persontrip_set.all().aggregate(day_max=Max("expeditionday__date"))
+        return res["day_max"]
+
+#
+# Single parsed entry from Logbook
+#    
+class LogbookEntry(TroggleModel):
+    date    = models.DateField()#MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.
+    expeditionday = models.ForeignKey("ExpeditionDay", null=True)#MJG wants to KILL THIS (redundant information)
+    expedition  = models.ForeignKey(Expedition,blank=True,null=True)  # yes this is double-
+    #author  = models.ForeignKey(PersonExpedition,blank=True,null=True)  # the person who writes it up doesn't have to have been on the trip.
+    # Re: the above- so this field should be "typist" or something, not "author". - AC 15 jun 09
+    #MJG wants to KILL THIS, as it is typically redundant with PersonTrip.is_logbook_entry_author, in the rare it was not redundanty and of actually interest it could be added to the text.
+    title     = models.CharField(max_length=settings.MAX_LOGBOOK_ENTRY_TITLE_LENGTH)
+    cave_slug = models.SlugField(max_length=50)
+    place     = models.CharField(max_length=100,blank=True,null=True,help_text="Only use this if you haven't chosen a cave")
+    text      = models.TextField()
+    slug      = models.SlugField(max_length=50)
+    filename  = models.CharField(max_length=200,null=True)
+
+    class Meta:
+        verbose_name_plural = "Logbook Entries"
+        # several PersonTrips point in to this object
+        ordering = ('-date',)
+        
+    def __getattribute__(self, item):
+      if item == "cave":  #Allow a logbookentries cave to be directly accessed despite not having a proper foreignkey
+        return CaveSlug.objects.get(slug = self.cave_slug).cave
+      return super(LogbookEntry, self).__getattribute__(item)
+
+    def __init__(self, *args, **kwargs):
+        if "cave" in kwargs.keys():
+            if kwargs["cave"] is not None:
+                kwargs["cave_slug"] = CaveSlug.objects.get(cave=kwargs["cave"], primary=True).slug
+            kwargs.pop("cave")
+        return super(LogbookEntry, self).__init__(*args, **kwargs)
+
+    def isLogbookEntry(self): # Function used in templates
+        return True
+
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, reverse('logbookentry',kwargs={'date':self.date,'slug':self.slug}))
+
+    def __unicode__(self):
+        return "%s: (%s)" % (self.date, self.title)
+
+    def get_next_by_id(self):
+        LogbookEntry.objects.get(id=self.id+1)
+
+    def get_previous_by_id(self):
+        LogbookEntry.objects.get(id=self.id-1)
+
+    def new_QM_number(self):
+        """Returns  """
+        if self.cave:
+            nextQMnumber=self.cave.new_QM_number(self.date.year)
+        else:
+            return none
+        return nextQMnumber
+
+    def new_QM_found_link(self):
+        """Produces a link to a new QM with the next number filled in and this LogbookEntry set as 'found by' """
+        return settings.URL_ROOT + r'/admin/core/qm/add/?' + r'found_by=' + str(self.pk) +'&number=' + str(self.new_QM_number())
+
+    def DayIndex(self):
+        return list(self.expeditionday.logbookentry_set.all()).index(self)
+
+#
+# Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry)
+#
+class PersonTrip(TroggleModel):
+    personexpedition = models.ForeignKey("PersonExpedition",null=True)
+    
+    #expeditionday    = models.ForeignKey("ExpeditionDay")#MJG wants to KILL THIS (redundant information)
+    #date             = models.DateField() #MJG wants to KILL THIS (redundant information)
+    time_underground = models.FloatField(help_text="In decimal hours")
+    logbook_entry    = models.ForeignKey(LogbookEntry)
+    is_logbook_entry_author = models.BooleanField(default=False)
+    
+    
+    # sequencing by person (difficult to solve locally)
+    #persontrip_next  = models.ForeignKey('PersonTrip', related_name='pnext', blank=True,null=True)#MJG wants to KILL THIS (and use funstion persontrip_next_auto)
+    #persontrip_prev  = models.ForeignKey('PersonTrip', related_name='pprev', blank=True,null=True)#MJG wants to KILL THIS(and use funstion persontrip_prev_auto)
+    
+    def persontrip_next(self):
+        futurePTs = PersonTrip.objects.filter(personexpedition = self.personexpedition, logbook_entry__date__gt = self.logbook_entry.date).order_by('logbook_entry__date').all()
+        if len(futurePTs) > 0:
+            return futurePTs[0]
+        else:
+            return None
+
+    def persontrip_prev(self):
+        pastPTs = PersonTrip.objects.filter(personexpedition = self.personexpedition, logbook_entry__date__lt = self.logbook_entry.date).order_by('-logbook_entry__date').all()
+        if len(pastPTs) > 0:
+            return pastPTs[0]
+        else:
+            return None
+
+    def place(self):
+        return self.logbook_entry.cave and self.logbook_entry.cave or self.logbook_entry.place
+
+    def __unicode__(self):
+        return "%s (%s)" % (self.personexpedition, self.logbook_entry.date)
+    
+
+
+##########################################
+# move following classes into models_cave
+##########################################
+
+class Area(TroggleModel):
+    short_name = models.CharField(max_length=100)
+    name = models.CharField(max_length=200, blank=True, null=True)
+    description = models.TextField(blank=True,null=True)
+    parent = models.ForeignKey('Area', blank=True, null=True)
+    def __unicode__(self):
+        if self.parent:
+            return unicode(self.parent) + u" - " + unicode(self.short_name)
+        else:
+            return unicode(self.short_name)
+    def kat_area(self):
+        if self.short_name in ["1623", "1626"]:
+            return self.short_name
+        elif self.parent:
+            return self.parent.kat_area()
+
+class CaveAndEntrance(models.Model):
+    cave = models.ForeignKey('Cave')
+    entrance = models.ForeignKey('Entrance')
+    entrance_letter = models.CharField(max_length=20,blank=True,null=True)
+    def __unicode__(self):
+        return unicode(self.cave) + unicode(self.entrance_letter)
+        
+class CaveSlug(models.Model):
+    cave = models.ForeignKey('Cave')
+    slug = models.SlugField(max_length=50, unique = True)
+    primary = models.BooleanField(default=False)
+    
+
+class Cave(TroggleModel):
+    # too much here perhaps, 
+    official_name = models.CharField(max_length=160)
+    area = models.ManyToManyField(Area, blank=True, null=True)
+    kataster_code = models.CharField(max_length=20,blank=True,null=True)
+    kataster_number = models.CharField(max_length=10,blank=True, null=True)
+    unofficial_number = models.CharField(max_length=60,blank=True, null=True)
+    entrances = models.ManyToManyField('Entrance', through='CaveAndEntrance')
+    explorers = models.TextField(blank=True,null=True)
+    underground_description = models.TextField(blank=True,null=True)
+    equipment = models.TextField(blank=True,null=True)
+    references = models.TextField(blank=True,null=True)
+    survey = models.TextField(blank=True,null=True)
+    kataster_status = models.TextField(blank=True,null=True)
+    underground_centre_line = models.TextField(blank=True,null=True)
+    notes = models.TextField(blank=True,null=True)
+    length = models.CharField(max_length=100,blank=True,null=True)
+    depth = models.CharField(max_length=100,blank=True,null=True)
+    extent = models.CharField(max_length=100,blank=True,null=True)
+    survex_file = models.CharField(max_length=100,blank=True,null=True)
+    description_file = models.CharField(max_length=200,blank=True,null=True)
+    url = models.CharField(max_length=200,blank=True,null=True)
+    filename = models.CharField(max_length=200)
+
+
+    #class Meta:
+    #    unique_together = (("area", "kataster_number"), ("area", "unofficial_number"))
+    # FIXME Kataster Areas and CUCC defined sub areas need seperating    
+
+
+    #href    = models.CharField(max_length=100)
+    
+    class Meta:
+        ordering = ('kataster_code', 'unofficial_number')  
+
+    def hassurvey(self):
+        if not self.underground_centre_line:
+            return "No"
+        if (self.survey.find("<img") > -1 or self.survey.find("<a") > -1 or self.survey.find("<IMG") > -1 or self.survey.find("<A") > -1):
+            return "Yes"
+        return "Missing"
+
+    def hassurveydata(self):
+        if not self.underground_centre_line:
+            return "No"
+        if self.survex_file:
+            return "Yes"
+        return "Missing"
+         
+    def slug(self):
+        primarySlugs = self.caveslug_set.filter(primary = True)
+        if primarySlugs:
+            return primarySlugs[0].slug
+        else:
+            slugs = self.caveslug_set.filter()
+            if slugs:
+                return slugs[0].slug
+
+    def ours(self):
+        return bool(re.search(r'CUCC', self.explorers))
+
+    def reference(self):
+        if self.kataster_number:
+            return "%s-%s" % (self.kat_area(), self.kataster_number)
+        else:
+            return "%s-%s" % (self.kat_area(), self.unofficial_number)
+    
+    def get_absolute_url(self):
+        if self.kataster_number:
+            href = self.kataster_number
+        elif self.unofficial_number:
+            href = self.unofficial_number
+        else:
+            href = official_name.lower()
+        #return settings.URL_ROOT + '/cave/' + href + '/'
+        return urlparse.urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,}))
+
+    def __unicode__(self, sep = u": "):
+        return unicode(self.slug())
+
+    def get_QMs(self):
+        return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all())	
+
+    def new_QM_number(self, year=datetime.date.today().year):
+            """Given a cave and the current year, returns the next QM number."""
+            try:
+                res=QM.objects.filter(found_by__date__year=year, found_by__cave=self).order_by('-number')[0]
+            except IndexError:
+                return 1
+            return res.number+1
+
+    def kat_area(self):
+        for a in self.area.all():
+            if a.kat_area():
+                return a.kat_area()
+    
+    def entrances(self):
+        return CaveAndEntrance.objects.filter(cave=self)
+
+    def singleentrance(self):
+        return len(CaveAndEntrance.objects.filter(cave=self)) == 1
+    
+    def entrancelist(self):
+        rs = []
+        res = ""
+        for e in CaveAndEntrance.objects.filter(cave=self):
+            rs.append(e.entrance_letter)
+        rs.sort()
+        prevR = None
+        n = 0
+        for r in rs:
+            if prevR:
+                if chr(ord(prevR) + 1 ) == r:
+                    prevR = r
+                    n += 1
+                else:
+                    if n == 0:
+                        res += ", " + prevR
+                    else:
+                        res += "&ndash;" + prevR
+            else:
+                prevR = r
+                n = 0
+                res += r
+        if n == 0:
+            res += ", " + prevR
+        else:
+            res += "&ndash;" + prevR
+        return res
+    
+    def writeDataFile(self):
+        try:
+            f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "w")
+        except:
+            subprocess.call(settings.FIX_PERMISSIONS) 
+            f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "w")
+        t = loader.get_template('dataformat/cave.xml')
+        c = Context({'cave': self})
+        u = t.render(c)
+        u8 = u.encode("utf-8")
+        f.write(u8)
+        f.close()
+        
+    def getArea(self):
+        areas = self.area.all()
+        lowestareas = list(areas)
+        for area in areas:
+            if area.parent in areas:
+                try:
+                    lowestareas.remove(area.parent)
+                except:
+                    pass
+        return lowestareas[0]
+
+def getCaveByReference(reference):
+    areaname, code = reference.split("-", 1)
+    print(areaname, code)
+    area = Area.objects.get(short_name = areaname)
+    print(area)
+    foundCaves = list(Cave.objects.filter(area = area,  kataster_number = code).all()) + list(Cave.objects.filter(area = area,  unofficial_number = code).all()) 
+    print(list(foundCaves))
+    assert len(foundCaves) == 1
+    return foundCaves[0]
+
+class OtherCaveName(TroggleModel):
+    name = models.CharField(max_length=160)
+    cave = models.ForeignKey(Cave)
+    def __unicode__(self):
+        return unicode(self.name)
+            
+class EntranceSlug(models.Model):
+    entrance = models.ForeignKey('Entrance')
+    slug = models.SlugField(max_length=50, unique = True)
+    primary = models.BooleanField(default=False)
+
+class Entrance(TroggleModel):
+    name = models.CharField(max_length=100, blank=True,null=True)
+    entrance_description = models.TextField(blank=True,null=True)
+    explorers = models.TextField(blank=True,null=True)
+    map_description = models.TextField(blank=True,null=True)
+    location_description = models.TextField(blank=True,null=True)
+    approach = models.TextField(blank=True,null=True)
+    underground_description = models.TextField(blank=True,null=True)
+    photo = models.TextField(blank=True,null=True)
+    MARKING_CHOICES = (
+        ('P', 'Paint'),
+        ('P?', 'Paint (?)'),
+        ('T', 'Tag'),
+        ('T?', 'Tag (?)'),
+        ('R', 'Needs Retag'),
+        ('S', 'Spit'),
+        ('S?', 'Spit (?)'),
+        ('U', 'Unmarked'),
+        ('?', 'Unknown'))
+    marking = models.CharField(max_length=2, choices=MARKING_CHOICES)
+    marking_comment = models.TextField(blank=True,null=True)
+    FINDABLE_CHOICES = (
+        ('?', 'To be confirmed ...'),
+        ('S', 'Coordinates'),
+        ('L', 'Lost'),
+        ('R', 'Refindable'))
+    findability = models.CharField(max_length=1, choices=FINDABLE_CHOICES, blank=True, null=True)
+    findability_description = models.TextField(blank=True,null=True)
+    alt = models.TextField(blank=True, null=True)
+    northing = models.TextField(blank=True, null=True)
+    easting = models.TextField(blank=True, null=True)
+    tag_station = models.TextField(blank=True, null=True)
+    exact_station = models.TextField(blank=True, null=True)
+    other_station = models.TextField(blank=True, null=True)
+    other_description = models.TextField(blank=True,null=True)
+    bearings = models.TextField(blank=True,null=True)
+    url = models.CharField(max_length=200,blank=True,null=True)
+    filename = models.CharField(max_length=200)
+    cached_primary_slug = models.CharField(max_length=200,blank=True,null=True)
+
+    def __unicode__(self):
+        return unicode(self.slug())
+
+    def exact_location(self):
+        return SurvexStation.objects.lookup(self.exact_station)
+    def other_location(self):
+        return SurvexStation.objects.lookup(self.other_station)
+
+
+    def find_location(self):
+        r = {'':  'To be entered ',
+			 '?': 'To be confirmed:',
+             'S': '',
+             'L': 'Lost:',
+             'R': 'Refindable:'}[self.findability]
+        if self.tag_station:
+            try:
+                s = SurvexStation.objects.lookup(self.tag_station)
+                return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z)
+            except:
+                return r + "%s Tag Station not in dataset" % self.tag_station
+        if self.exact_station:
+            try:
+                s = SurvexStation.objects.lookup(self.exact_station)
+                return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z)
+            except:
+                return r + "%s Exact Station not in dataset" % self.tag_station
+        if self.other_station:
+            try:
+                s = SurvexStation.objects.lookup(self.other_station)
+                return r + "%0.0fE %0.0fN %0.0fAlt %s" % (s.x, s.y, s.z, self.other_description)
+            except:
+                return r + "%s Other Station not in dataset" % self.tag_station
+        if self.FINDABLE_CHOICES == "S":
+            r += "ERROR, Entrance has been surveyed but has no survex point"
+        if self.bearings:
+            return r + self.bearings
+        return r
+
+    def best_station(self):
+        if self.tag_station:
+            return self.tag_station
+        if self.exact_station:
+            return self.exact_station
+        if self.other_station:
+            return self.other_station
+
+    def has_photo(self):
+        if self.photo:
+            if (self.photo.find("<img") > -1 or self.photo.find("<a") > -1 or self.photo.find("<IMG") > -1 or self.photo.find("<A") > -1):
+                return "Yes"
+            else:
+                return "Missing"
+        else:
+            return "No"
+
+    def marking_val(self):
+        for m in self.MARKING_CHOICES:
+            if m[0] == self.marking:
+                return m[1]
+    def findability_val(self):
+        for f in self.FINDABLE_CHOICES:
+            if f[0] == self.findability:
+                return f[1]
+                
+    def tag(self):
+        return SurvexStation.objects.lookup(self.tag_station)
+        
+    def needs_surface_work(self):
+        return self.findability != "S" or not self.has_photo or self.marking != "T"
+
+    def get_absolute_url(self):
+        
+        ancestor_titles='/'.join([subcave.title for subcave in self.get_ancestors()])
+        if ancestor_titles:
+            res = '/'.join((self.get_root().cave.get_absolute_url(), ancestor_titles, self.title))
+        
+        else:
+            res = '/'.join((self.get_root().cave.get_absolute_url(), self.title))
+            
+        return res
+
+    def slug(self):
+        if not self.cached_primary_slug:
+            primarySlugs = self.entranceslug_set.filter(primary = True)
+            if primarySlugs:    
+                self.cached_primary_slug = primarySlugs[0].slug
+                self.save()
+            else:
+                slugs = self.entranceslug_set.filter()
+                if slugs:
+                    self.cached_primary_slug = slugs[0].slug
+                    self.save()
+        return self.cached_primary_slug
+
+    def writeDataFile(self):
+        try:
+            f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w")
+        except:
+            subprocess.call(settings.FIX_PERMISSIONS) 
+            f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w")
+        t = loader.get_template('dataformat/entrance.xml')
+        c = Context({'entrance': self})
+        u = t.render(c)
+        u8 = u.encode("utf-8")
+        f.write(u8)
+        f.close()
+
+class CaveDescription(TroggleModel):
+    short_name = models.CharField(max_length=50, unique = True)
+    long_name = models.CharField(max_length=200, blank=True, null=True)
+    description = models.TextField(blank=True,null=True)
+    linked_subcaves = models.ManyToManyField("NewSubCave", blank=True,null=True)
+    linked_entrances = models.ManyToManyField("Entrance", blank=True,null=True)
+    linked_qms = models.ManyToManyField("QM", blank=True,null=True)
+
+    def __unicode__(self):
+        if self.long_name:
+            return unicode(self.long_name)
+        else:
+            return unicode(self.short_name)
+    
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, reverse('cavedescription', args=(self.short_name,)))
+    
+    def save(self):
+        """
+        Overridden save method which stores wikilinks in text as links in database.
+        """
+        super(CaveDescription, self).save()
+        qm_list=get_related_by_wikilinks(self.description)
+        for qm in qm_list:
+            self.linked_qms.add(qm)
+        super(CaveDescription, self).save()
+
+class NewSubCave(TroggleModel):
+    name = models.CharField(max_length=200, unique = True)
+    def __unicode__(self):
+        return unicode(self.name)
+
+class QM(TroggleModel):
+    #based on qm.csv in trunk/expoweb/1623/204 which has the fields:
+    #"Number","Grade","Area","Description","Page reference","Nearest station","Completion description","Comment"
+    found_by = models.ForeignKey(LogbookEntry, related_name='QMs_found',blank=True, null=True )
+    ticked_off_by = models.ForeignKey(LogbookEntry, related_name='QMs_ticked_off',null=True,blank=True)
+    #cave = models.ForeignKey(Cave)
+    #expedition = models.ForeignKey(Expedition)
+
+    number = models.IntegerField(help_text="this is the sequential number in the year", )
+    GRADE_CHOICES=(
+	('A', 'A: Large obvious lead'),
+	('B', 'B: Average lead'),
+	('C', 'C: Tight unpromising lead'),
+	('D', 'D: Dig'),
+	('X', 'X: Unclimbable aven')
+    )
+    grade = models.CharField(max_length=1, choices=GRADE_CHOICES)
+    location_description = models.TextField(blank=True)
+    #should be a foreignkey to surveystation
+    nearest_station_description = models.CharField(max_length=400,null=True,blank=True)
+    nearest_station = models.CharField(max_length=200,blank=True,null=True)
+    area = models.CharField(max_length=100,blank=True,null=True)
+    completion_description = models.TextField(blank=True,null=True)
+    comment=models.TextField(blank=True,null=True)
+
+    def __unicode__(self):
+        return u"%s %s" % (self.code(), self.grade)
+
+    def code(self):
+        return u"%s-%s-%s" % (unicode(self.found_by.cave)[6:], self.found_by.date.year, self.number)
+
+    def get_absolute_url(self):
+        #return settings.URL_ROOT + '/cave/' + self.found_by.cave.kataster_number + '/' + str(self.found_by.date.year) + '-' + '%02d' %self.number
+        return urlparse.urljoin(settings.URL_ROOT, reverse('qm',kwargs={'cave_id':self.found_by.cave.kataster_number,'year':self.found_by.date.year,'qm_id':self.number,'grade':self.grade}))
+
+    def get_next_by_id(self):
+        return QM.objects.get(id=self.id+1)
+
+    def get_previous_by_id(self):
+        return QM.objects.get(id=self.id-1)
+
+    def wiki_link(self):
+        return u"%s%s%s" % ('[[QM:',self.code(),']]')
+
+photoFileStorage = FileSystemStorage(location=settings.PHOTOS_ROOT, base_url=settings.PHOTOS_URL)
+class DPhoto(TroggleImageModel): 
+    caption = models.CharField(max_length=1000,blank=True,null=True)
+    contains_logbookentry = models.ForeignKey(LogbookEntry,blank=True,null=True)
+    contains_person = models.ManyToManyField(Person,blank=True,null=True)
+    file = models.ImageField(storage=photoFileStorage, upload_to='.',)
+    is_mugshot = models.BooleanField(default=False)
+    contains_cave = models.ForeignKey(Cave,blank=True,null=True)
+    contains_entrance = models.ForeignKey(Entrance, related_name="photo_file",blank=True,null=True)
+    #nearest_survey_point = models.ForeignKey(SurveyStation,blank=True,null=True)
+    nearest_QM = models.ForeignKey(QM,blank=True,null=True)
+    lon_utm = models.FloatField(blank=True,null=True)
+    lat_utm = models.FloatField(blank=True,null=True)
+    
+    class IKOptions:
+        spec_module = 'core.imagekit_specs'
+        cache_dir = 'thumbs'
+        image_field = 'file'
+        
+    #content_type = models.ForeignKey(ContentType)
+    #object_id = models.PositiveIntegerField()
+    #location = generic.GenericForeignKey('content_type', 'object_id')
+
+    def __unicode__(self):
+        return self.caption
+
+scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL)
+def get_scan_path(instance, filename):
+    year=instance.survey.expedition.year
+    #print("WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number, instance.survey.wallet_letter)
+    number=str(instance.survey.wallet_number)
+    if str(instance.survey.wallet_letter) != "None":
+        number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01
+    return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg')
+
+class ScannedImage(TroggleImageModel): 
+    file = models.ImageField(storage=scansFileStorage, upload_to=get_scan_path)
+    scanned_by = models.ForeignKey(Person,blank=True, null=True)
+    scanned_on = models.DateField(null=True)
+    survey = models.ForeignKey('Survey')
+    contents = models.CharField(max_length=20,choices=(('notes','notes'),('plan','plan_sketch'),('elevation','elevation_sketch')))
+    number_in_wallet = models.IntegerField(null=True)
+    lon_utm = models.FloatField(blank=True,null=True)
+    lat_utm = models.FloatField(blank=True,null=True)
+
+    class IKOptions:
+        spec_module = 'core.imagekit_specs'
+        cache_dir = 'thumbs'
+        image_field = 'file'
+    #content_type = models.ForeignKey(ContentType)
+    #object_id = models.PositiveIntegerField()
+    #location = generic.GenericForeignKey('content_type', 'object_id')
+
+    #This is an ugly hack to deal with the #s in our survey scan paths. The correct thing is to write a custom file storage backend which calls urlencode on the name for making file.url but not file.path.
+    def correctURL(self):
+        return string.replace(self.file.url,r'#',r'%23')
+    
+    def __unicode__(self):
+        return get_scan_path(self,'')
+
+class Survey(TroggleModel):
+    expedition = models.ForeignKey('Expedition') #REDUNDANT (logbook_entry)
+    wallet_number = models.IntegerField(blank=True,null=True)
+    wallet_letter = models.CharField(max_length=1,blank=True,null=True)
+    comments = models.TextField(blank=True,null=True)
+    location = models.CharField(max_length=400,blank=True,null=True) #REDUNDANT
+    subcave = models.ForeignKey('NewSubCave', blank=True, null=True)
+    #notes_scan = models.ForeignKey('ScannedImage',related_name='notes_scan',blank=True, null=True)  	#Replaced by contents field of ScannedImage model
+    survex_block  = models.OneToOneField('SurvexBlock',blank=True, null=True)
+    logbook_entry = models.ForeignKey('LogbookEntry')
+    centreline_printed_on = models.DateField(blank=True, null=True)
+    centreline_printed_by = models.ForeignKey('Person',related_name='centreline_printed_by',blank=True,null=True)
+    #sketch_scan = models.ForeignKey(ScannedImage,blank=True, null=True) 					#Replaced by contents field of ScannedImage model
+    tunnel_file = models.FileField(upload_to='surveyXMLfiles',blank=True, null=True)
+    tunnel_main_sketch = models.ForeignKey('Survey',blank=True,null=True)
+    integrated_into_main_sketch_on = models.DateField(blank=True,null=True)
+    integrated_into_main_sketch_by = models.ForeignKey('Person' ,related_name='integrated_into_main_sketch_by', blank=True,null=True)
+    rendered_image = models.ImageField(upload_to='renderedSurveys',blank=True,null=True)
+    def __unicode__(self):
+        return self.expedition.year+"#"+"%02d" % int(self.wallet_number)
+
+    def notes(self):
+        return self.scannedimage_set.filter(contents='notes')
+
+    def plans(self):
+        return self.scannedimage_set.filter(contents='plan')
+
+    def elevations(self):
+        return self.scannedimage_set.filter(contents='elevation')

core/models_millenial.py

@@ -0,0 +1,83 @@
+from django.db import models
+from django.conf import settings
+
+import troggle.core.methods_millenial as  methods_millenial
+
+#
+#   This file was created in 2019
+#   It's a result of massive frustration with cluttered database of troggle
+#   Maximal clarity of code was primary goal (previous code had very little comments)
+#   Maximal speed of database rebuild was secondary goal
+#
+
+#
+#   The following file will tell you what fields and methods are avaliable inside this database
+#   be carefull you might miss some! ManyToMany fields can be used from the far end as well
+#
+
+
+#
+#  Naming conventions:
+#  (Upper/lower convention)
+#  Class names are writen Udddd_ddd_dddM - they finish with M for backwards compatibility
+#  Fields/methods are written lower_lower_lower
+#
+
+class PersonM(models.Model): #instance of this class corresponds to one physical peson
+    name = models.CharField(max_length=100) #just name, talk to wookey if you diagree
+    surveys_made = models.ManyToManyField('SurveyM', related_name='people_surveyed') #links to survey objects that this person made (made=:survex says so)
+    expos_attended = models.ManyToManyField('ExpeditionM', related_name='people_attended') #expos attended by this person (attended=:folk.csv says so)
+    logbook_entries_written = models.ManyToManyField('Logbook_entryM', related_name='people_wrote') #links to logbook chuncks created by a person
+
+class CaveM(models.Model): #instance of this class corresponds to one 'thing' that people call cave
+    entrance = models.CharField(max_length=100) #UTM string describing ONE(!) entrance. Purpose = findability
+    title = models.TextField() #title given to the topmost survey in survex, numeric name otherwise c.f. name (e.g. 'Fishface')
+    name = models.TextField() #name given to the topmost survey in survex (e.g. '2017-cucc-28')
+    surveys = models.ManyToManyField('SurveyM', related_name='cave_parent') #links to surveys objects that this cave contains
+    survex_file = models.TextField() #gives path to top level survex file
+    total_length = models.FloatField() #holds total length of this cave (as given by cavern)
+    total_depth = models.FloatField() #holds total depth of this cave (as given by cavern)
+    description = models.TextField() #holds link to description
+    date = models.TextField() #holds date of last visit
+    def top_camp_distance(self): #returns distance of this cave from topcamp
+        return methods_millenial.top_camp_distance(self.entrance)
+    def top_camp_bearing(self): #returns bearing to this cave from topcamp in format 235.5 (float north-based azimuth)
+        return methods_millenial.top_camp_bearing(self.entrance)
+    def top_camp_bearing_letter(self): #returns bearing to this cave from topcamp in format e.g. 'NE'
+        return methods_millenial.top_camp_bearing_letter(self.entrance)
+    def lat_lon_entrance(self): #lat_lon entrance location
+        return methods_millenial.lat_lon_entrance(self.entrance)
+
+
+class Cave_descriptionM(models.Model): #instance of this class corresponds to each of the .html files in descriptions
+    #each of those holds one XML field
+    slug = models.TextField()
+    explorers = models.TextField()
+    underground_description = models.TextField()
+    equipment = models.TextField()
+    references = models.TextField()
+    survey = models.TextField()
+    kataster_status = models.TextField()
+    underground_centre_line = models.TextField()
+    survex_file = models.TextField() #as given in .html file
+    notes = models.TextField()
+
+    
+
+class ExpeditionM(models.Model): #instance of this class corresponds to one expo (usually one year)
+    date = models.CharField(max_length=100) #date in format YYYY.MM.DD-YYYY.MM.DD
+
+
+class SurveyM(models.Model): #instance of this class corresponds to one .svx file - one trip
+    date = models.CharField(max_length=100) #date of the trip in format YYYY.MM.DD (dated:=date given by .svx file) 
+    survex_file = models.TextField()
+    
+class Logbook_entryM(models.Model): #instance of this class corresponds to one bit of logbook (c.f. expo.survex.com/years/2015/logbook.html or simil)
+    date = models.CharField(max_length=100) #date as typed into logbook
+    contents = models.TextField() #contents of the logbook chunk
+
+class Parser_messageM(models.Model): #instance of this class contains one error or warining message produce by any of the parsers
+    parsername = models.CharField(max_length = 20) #name of parser
+    content = models.TextField() #content of message
+    message_type = models.CharField(max_length = 10) # [Error,Info] or similar
+

core/models_old.py

@@ -0,0 +1,862 @@
+import urllib, urlparse, string, os, datetime, logging, re
+import subprocess
+from django.forms import ModelForm
+from django.db import models
+from django.contrib import admin
+from django.core.files.storage import FileSystemStorage
+from django.contrib.auth.models import User
+from django.contrib.contenttypes.models import ContentType
+from django.db.models import Min, Max
+from django.conf import settings
+from decimal import Decimal, getcontext
+from django.core.urlresolvers import reverse
+from imagekit.models import ImageModel
+from django.template import Context, loader
+import settings
+getcontext().prec=2 #use 2 significant figures for decimal calculations
+
+
+
+def get_related_by_wikilinks(wiki_text):
+    found=re.findall(settings.QM_PATTERN,wiki_text)
+    res=[]
+    for wikilink in found:
+        qmdict={'urlroot':settings.URL_ROOT,'cave':wikilink[2],'year':wikilink[1],'number':wikilink[3]}
+        try:
+            cave_slugs = CaveSlug.objects.filter(cave__kataster_number = qmdict['cave'])
+            qm=QM.objects.get(found_by__cave_slug__in = cave_slugs,
+                              found_by__date__year = qmdict['year'],
+                              number = qmdict['number'])
+            res.append(qm)         
+        except QM.DoesNotExist:
+            print('fail on '+str(wikilink))
+    
+    return res
+
+try:
+      logging.basicConfig(level=logging.DEBUG,
+                           filename=settings.LOGFILE,
+                           filemode='w')
+except:
+      subprocess.call(settings.FIX_PERMISSIONS) 
+      logging.basicConfig(level=logging.DEBUG,
+                           filename=settings.LOGFILE,
+                           filemode='w')
+
+#This class is for adding fields and methods which all of our models will have.
+class TroggleModel(models.Model):
+    new_since_parsing = models.BooleanField(default=False, editable=False)
+    non_public = models.BooleanField(default=False)
+    def object_name(self):
+        return self._meta.object_name
+
+    def get_admin_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, "/admin/core/" + self.object_name().lower() + "/" + str(self.pk))
+
+    class Meta:
+        abstract = True
+
+class TroggleImageModel(ImageModel):
+    new_since_parsing = models.BooleanField(default=False, editable=False)
+    
+    def object_name(self):
+        return self._meta.object_name
+
+    def get_admin_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, "/admin/core/" + self.object_name().lower() + "/" + str(self.pk))
+
+
+    class Meta:
+        abstract = True
+
+# 
+# single Expedition, usually seen by year
+#
+class Expedition(TroggleModel):
+    year        = models.CharField(max_length=20, unique=True)
+    name        = models.CharField(max_length=100)
+        
+    def __unicode__(self):
+        return self.year
+
+    class Meta:
+        ordering = ('-year',)
+        get_latest_by = 'year'
+    
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, reverse('expedition', args=[self.year]))
+    
+    # construction function.  should be moved out
+    def get_expedition_day(self, date):
+        expeditiondays = self.expeditionday_set.filter(date=date)
+        if expeditiondays:
+            assert len(expeditiondays) == 1
+            return expeditiondays[0]
+        res = ExpeditionDay(expedition=self, date=date)
+        res.save()
+        return res
+        
+    def day_min(self):
+        res = self.expeditionday_set.all()
+        return res and res[0] or None
+    
+    def day_max(self):
+        res = self.expeditionday_set.all()
+        return res and res[len(res) - 1] or None
+        
+        
+
+class ExpeditionDay(TroggleModel):
+    expedition  = models.ForeignKey("Expedition")
+    date        = models.DateField()
+
+    class Meta:
+        ordering = ('date',)  
+
+    def GetPersonTrip(self, personexpedition):
+        personexpeditions = self.persontrip_set.filter(expeditionday=self)
+        return personexpeditions and personexpeditions[0] or None
+    
+        
+#
+# single Person, can go on many years
+#
+class Person(TroggleModel):
+    first_name  = models.CharField(max_length=100)
+    last_name   = models.CharField(max_length=100)
+    is_vfho     = models.BooleanField(help_text="VFHO is the Vereines für Höhlenkunde in Obersteier, a nearby Austrian caving club.", default=False)
+    mug_shot    = models.CharField(max_length=100, blank=True,null=True)
+    blurb = models.TextField(blank=True,null=True)
+    
+    #href        = models.CharField(max_length=200)
+    orderref    = models.CharField(max_length=200)  # for alphabetic 
+    
+    #the below have been removed and made methods. I'm not sure what the b in bisnotable stands for. - AC 16 Feb
+    #notability  = models.FloatField()               # for listing the top 20 people
+    #bisnotable  = models.BooleanField(default=False)
+    user	= models.OneToOneField(User, null=True, blank=True)
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT,reverse('person',kwargs={'first_name':self.first_name,'last_name':self.last_name}))
+
+    class Meta:
+        verbose_name_plural = "People"
+        ordering = ('orderref',)  # "Wookey" makes too complex for: ('last_name', 'first_name') 
+    
+    def __unicode__(self):
+        if self.last_name:
+            return "%s %s" % (self.first_name, self.last_name)
+        return self.first_name
+
+    
+    def notability(self):
+        notability = Decimal(0)
+        for personexpedition in self.personexpedition_set.all():
+             if not personexpedition.is_guest:
+                notability += Decimal(1) / (2012 - int(personexpedition.expedition.year))
+        return notability
+
+    def bisnotable(self):
+        return self.notability() > Decimal(1)/Decimal(3)
+    
+    def surveyedleglength(self):
+        return sum([personexpedition.surveyedleglength()  for personexpedition in self.personexpedition_set.all()])
+    
+    def first(self):
+        return self.personexpedition_set.order_by('-expedition')[0]
+    def last(self):
+        return self.personexpedition_set.order_by('expedition')[0]
+    
+    #def Sethref(self):
+        #if self.last_name:
+            #self.href = self.first_name.lower() + "_" + self.last_name.lower()
+            #self.orderref = self.last_name + " " + self.first_name
+        #else:
+          #  self.href = self.first_name.lower()
+            #self.orderref = self.first_name
+        #self.notability = 0.0  # set temporarily
+        
+
+#
+# Person's attenance to one Expo
+#
+class PersonExpedition(TroggleModel):
+    expedition  = models.ForeignKey(Expedition)
+    person      = models.ForeignKey(Person)
+    slugfield   = models.SlugField(max_length=50,blank=True,null=True)
+    
+    is_guest    = models.BooleanField(default=False)  
+    COMMITTEE_CHOICES = (
+        ('leader','Expo leader'),
+        ('medical','Expo medical officer'),
+        ('treasurer','Expo treasurer'),
+        ('sponsorship','Expo sponsorship coordinator'),
+        ('research','Expo research coordinator'),
+        )
+    expo_committee_position = models.CharField(blank=True,null=True,choices=COMMITTEE_CHOICES,max_length=200)
+    nickname    = models.CharField(max_length=100,blank=True,null=True)
+    
+    def GetPersonroles(self):
+        res = [ ]
+        for personrole in self.personrole_set.order_by('survexblock'):
+            if res and res[-1]['survexpath'] == personrole.survexblock.survexpath:
+                res[-1]['roles'] += ", " + str(personrole.role)
+            else:
+                res.append({'date':personrole.survexblock.date, 'survexpath':personrole.survexblock.survexpath, 'roles':str(personrole.role)})
+        return res
+
+    class Meta:
+        ordering = ('-expedition',)
+        #order_with_respect_to = 'expedition'
+
+    def __unicode__(self):
+        return "%s: (%s)" % (self.person, self.expedition)
+    
+    
+    #why is the below a function in personexpedition, rather than in person? - AC 14 Feb 09
+    def name(self):
+        if self.nickname:
+            return "%s (%s) %s" % (self.person.first_name, self.nickname, self.person.last_name)
+        if self.person.last_name:
+            return "%s %s" % (self.person.first_name, self.person.last_name)
+        return self.person.first_name
+
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, reverse('personexpedition',kwargs={'first_name':self.person.first_name,'last_name':self.person.last_name,'year':self.expedition.year}))
+	
+    def surveyedleglength(self):
+        survexblocks = [personrole.survexblock  for personrole in self.personrole_set.all() ]
+        return sum([survexblock.totalleglength  for survexblock in set(survexblocks)])
+    
+    # would prefer to return actual person trips so we could link to first and last ones
+    def day_min(self):
+        res = self.persontrip_set.aggregate(day_min=Min("expeditionday__date"))
+        return res["day_min"]
+
+    def day_max(self):
+        res = self.persontrip_set.all().aggregate(day_max=Max("expeditionday__date"))
+        return res["day_max"]
+
+#
+# Single parsed entry from Logbook
+#    
+class LogbookEntry(TroggleModel):
+    date    = models.DateField()#MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.
+    expeditionday = models.ForeignKey("ExpeditionDay", null=True)#MJG wants to KILL THIS (redundant information)
+    expedition  = models.ForeignKey(Expedition,blank=True,null=True)  # yes this is double-
+    #author  = models.ForeignKey(PersonExpedition,blank=True,null=True)  # the person who writes it up doesn't have to have been on the trip.
+    # Re: the above- so this field should be "typist" or something, not "author". - AC 15 jun 09
+    #MJG wants to KILL THIS, as it is typically redundant with PersonTrip.is_logbook_entry_author, in the rare it was not redundanty and of actually interest it could be added to the text.
+    title     = models.CharField(max_length=settings.MAX_LOGBOOK_ENTRY_TITLE_LENGTH)
+    cave_slug = models.SlugField(max_length=50)
+    place     = models.CharField(max_length=100,blank=True,null=True,help_text="Only use this if you haven't chosen a cave")
+    text      = models.TextField()
+    slug      = models.SlugField(max_length=50)
+    filename  = models.CharField(max_length=200,null=True)
+
+    class Meta:
+        verbose_name_plural = "Logbook Entries"
+        # several PersonTrips point in to this object
+        ordering = ('-date',)
+        
+    def __getattribute__(self, item):
+      if item == "cave":  #Allow a logbookentries cave to be directly accessed despite not having a proper foreignkey
+        return CaveSlug.objects.get(slug = self.cave_slug).cave
+      return super(LogbookEntry, self).__getattribute__(item)
+
+    def __init__(self, *args, **kwargs):
+        if "cave" in kwargs.keys():
+            if kwargs["cave"] is not None:
+                kwargs["cave_slug"] = CaveSlug.objects.get(cave=kwargs["cave"], primary=True).slug
+            kwargs.pop("cave")
+        return super(LogbookEntry, self).__init__(*args, **kwargs)
+
+    def isLogbookEntry(self): # Function used in templates
+        return True
+
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, reverse('logbookentry',kwargs={'date':self.date,'slug':self.slug}))
+
+    def __unicode__(self):
+        return "%s: (%s)" % (self.date, self.title)
+
+    def get_next_by_id(self):
+        LogbookEntry.objects.get(id=self.id+1)
+
+    def get_previous_by_id(self):
+        LogbookEntry.objects.get(id=self.id-1)
+
+    def new_QM_number(self):
+        """Returns  """
+        if self.cave:
+            nextQMnumber=self.cave.new_QM_number(self.date.year)
+        else:
+            return none
+        return nextQMnumber
+
+    def new_QM_found_link(self):
+        """Produces a link to a new QM with the next number filled in and this LogbookEntry set as 'found by' """
+        return settings.URL_ROOT + r'/admin/core/qm/add/?' + r'found_by=' + str(self.pk) +'&number=' + str(self.new_QM_number())
+
+    def DayIndex(self):
+        return list(self.expeditionday.logbookentry_set.all()).index(self)
+
+#
+# Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry)
+#
+class PersonTrip(TroggleModel):
+    personexpedition = models.ForeignKey("PersonExpedition",null=True)
+    
+    #expeditionday    = models.ForeignKey("ExpeditionDay")#MJG wants to KILL THIS (redundant information)
+    #date             = models.DateField() #MJG wants to KILL THIS (redundant information)
+    time_underground = models.FloatField(help_text="In decimal hours")
+    logbook_entry    = models.ForeignKey(LogbookEntry)
+    is_logbook_entry_author = models.BooleanField(default=False)
+    
+    
+    # sequencing by person (difficult to solve locally)
+    #persontrip_next  = models.ForeignKey('PersonTrip', related_name='pnext', blank=True,null=True)#MJG wants to KILL THIS (and use funstion persontrip_next_auto)
+    #persontrip_prev  = models.ForeignKey('PersonTrip', related_name='pprev', blank=True,null=True)#MJG wants to KILL THIS(and use funstion persontrip_prev_auto)
+    
+    def persontrip_next(self):
+        futurePTs = PersonTrip.objects.filter(personexpedition = self.personexpedition, logbook_entry__date__gt = self.logbook_entry.date).order_by('logbook_entry__date').all()
+        if len(futurePTs) > 0:
+            return futurePTs[0]
+        else:
+            return None
+
+    def persontrip_prev(self):
+        pastPTs = PersonTrip.objects.filter(personexpedition = self.personexpedition, logbook_entry__date__lt = self.logbook_entry.date).order_by('-logbook_entry__date').all()
+        if len(pastPTs) > 0:
+            return pastPTs[0]
+        else:
+            return None
+
+    def place(self):
+        return self.logbook_entry.cave and self.logbook_entry.cave or self.logbook_entry.place
+
+    def __unicode__(self):
+        return "%s (%s)" % (self.personexpedition, self.logbook_entry.date)
+    
+
+
+##########################################
+# move following classes into models_cave
+##########################################
+
+class Area(TroggleModel):
+    short_name = models.CharField(max_length=100)
+    name = models.CharField(max_length=200, blank=True, null=True)
+    description = models.TextField(blank=True,null=True)
+    parent = models.ForeignKey('Area', blank=True, null=True)
+    def __unicode__(self):
+        if self.parent:
+            return unicode(self.parent) + u" - " + unicode(self.short_name)
+        else:
+            return unicode(self.short_name)
+    def kat_area(self):
+        if self.short_name in ["1623", "1626"]:
+            return self.short_name
+        elif self.parent:
+            return self.parent.kat_area()
+
+class CaveAndEntrance(models.Model):
+    cave = models.ForeignKey('Cave')
+    entrance = models.ForeignKey('Entrance')
+    entrance_letter = models.CharField(max_length=20,blank=True,null=True)
+    def __unicode__(self):
+        return unicode(self.cave) + unicode(self.entrance_letter)
+        
+class CaveSlug(models.Model):
+    cave = models.ForeignKey('Cave')
+    slug = models.SlugField(max_length=50, unique = True)
+    primary = models.BooleanField(default=False)
+    
+
+class Cave(TroggleModel):
+    # too much here perhaps, 
+    official_name = models.CharField(max_length=160)
+    area = models.ManyToManyField(Area, blank=True, null=True)
+    kataster_code = models.CharField(max_length=20,blank=True,null=True)
+    kataster_number = models.CharField(max_length=10,blank=True, null=True)
+    unofficial_number = models.CharField(max_length=60,blank=True, null=True)
+    entrances = models.ManyToManyField('Entrance', through='CaveAndEntrance')
+    explorers = models.TextField(blank=True,null=True)
+    underground_description = models.TextField(blank=True,null=True)
+    equipment = models.TextField(blank=True,null=True)
+    references = models.TextField(blank=True,null=True)
+    survey = models.TextField(blank=True,null=True)
+    kataster_status = models.TextField(blank=True,null=True)
+    underground_centre_line = models.TextField(blank=True,null=True)
+    notes = models.TextField(blank=True,null=True)
+    length = models.CharField(max_length=100,blank=True,null=True)
+    depth = models.CharField(max_length=100,blank=True,null=True)
+    extent = models.CharField(max_length=100,blank=True,null=True)
+    survex_file = models.CharField(max_length=100,blank=True,null=True)
+    description_file = models.CharField(max_length=200,blank=True,null=True)
+    url = models.CharField(max_length=200,blank=True,null=True)
+    filename = models.CharField(max_length=200)
+
+
+    #class Meta:
+    #    unique_together = (("area", "kataster_number"), ("area", "unofficial_number"))
+    # FIXME Kataster Areas and CUCC defined sub areas need seperating    
+
+
+    #href    = models.CharField(max_length=100)
+    
+    class Meta:
+        ordering = ('kataster_code', 'unofficial_number')  
+
+    def hassurvey(self):
+        if not self.underground_centre_line:
+            return "No"
+        if (self.survey.find("<img") > -1 or self.survey.find("<a") > -1 or self.survey.find("<IMG") > -1 or self.survey.find("<A") > -1):
+            return "Yes"
+        return "Missing"
+
+    def hassurveydata(self):
+        if not self.underground_centre_line:
+            return "No"
+        if self.survex_file:
+            return "Yes"
+        return "Missing"
+         
+    def slug(self):
+        primarySlugs = self.caveslug_set.filter(primary = True)
+        if primarySlugs:
+            return primarySlugs[0].slug
+        else:
+            slugs = self.caveslug_set.filter()
+            if slugs:
+                return slugs[0].slug
+
+    def ours(self):
+        return bool(re.search(r'CUCC', self.explorers))
+
+    def reference(self):
+        if self.kataster_number:
+            return "%s-%s" % (self.kat_area(), self.kataster_number)
+        else:
+            return "%s-%s" % (self.kat_area(), self.unofficial_number)
+    
+    def get_absolute_url(self):
+        if self.kataster_number:
+            href = self.kataster_number
+        elif self.unofficial_number:
+            href = self.unofficial_number
+        else:
+            href = official_name.lower()
+        #return settings.URL_ROOT + '/cave/' + href + '/'
+        return urlparse.urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,}))
+
+    def __unicode__(self, sep = u": "):
+        return unicode(self.slug())
+
+    def get_QMs(self):
+        return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all())	
+
+    def new_QM_number(self, year=datetime.date.today().year):
+            """Given a cave and the current year, returns the next QM number."""
+            try:
+                res=QM.objects.filter(found_by__date__year=year, found_by__cave=self).order_by('-number')[0]
+            except IndexError:
+                return 1
+            return res.number+1
+
+    def kat_area(self):
+        for a in self.area.all():
+            if a.kat_area():
+                return a.kat_area()
+    
+    def entrances(self):
+        return CaveAndEntrance.objects.filter(cave=self)
+
+    def singleentrance(self):
+        return len(CaveAndEntrance.objects.filter(cave=self)) == 1
+    
+    def entrancelist(self):
+        rs = []
+        res = ""
+        for e in CaveAndEntrance.objects.filter(cave=self):
+            rs.append(e.entrance_letter)
+        rs.sort()
+        prevR = None
+        n = 0
+        for r in rs:
+            if prevR:
+                if chr(ord(prevR) + 1 ) == r:
+                    prevR = r
+                    n += 1
+                else:
+                    if n == 0:
+                        res += ", " + prevR
+                    else:
+                        res += "&ndash;" + prevR
+            else:
+                prevR = r
+                n = 0
+                res += r
+        if n == 0:
+            res += ", " + prevR
+        else:
+            res += "&ndash;" + prevR
+        return res
+    
+    def writeDataFile(self):
+        try:
+            f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "w")
+        except:
+            subprocess.call(settings.FIX_PERMISSIONS) 
+            f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "w")
+        t = loader.get_template('dataformat/cave.xml')
+        c = Context({'cave': self})
+        u = t.render(c)
+        u8 = u.encode("utf-8")
+        f.write(u8)
+        f.close()
+        
+    def getArea(self):
+        areas = self.area.all()
+        lowestareas = list(areas)
+        for area in areas:
+            if area.parent in areas:
+                try:
+                    lowestareas.remove(area.parent)
+                except:
+                    pass
+        return lowestareas[0]
+
+def getCaveByReference(reference):
+    areaname, code = reference.split("-", 1)
+    print(areaname, code)
+    area = Area.objects.get(short_name = areaname)
+    print(area)
+    foundCaves = list(Cave.objects.filter(area = area,  kataster_number = code).all()) + list(Cave.objects.filter(area = area,  unofficial_number = code).all()) 
+    print(list(foundCaves))
+    assert len(foundCaves) == 1
+    return foundCaves[0]
+
+class OtherCaveName(TroggleModel):
+    name = models.CharField(max_length=160)
+    cave = models.ForeignKey(Cave)
+    def __unicode__(self):
+        return unicode(self.name)
+            
+class EntranceSlug(models.Model):
+    entrance = models.ForeignKey('Entrance')
+    slug = models.SlugField(max_length=50, unique = True)
+    primary = models.BooleanField(default=False)
+
+class Entrance(TroggleModel):
+    name = models.CharField(max_length=100, blank=True,null=True)
+    entrance_description = models.TextField(blank=True,null=True)
+    explorers = models.TextField(blank=True,null=True)
+    map_description = models.TextField(blank=True,null=True)
+    location_description = models.TextField(blank=True,null=True)
+    approach = models.TextField(blank=True,null=True)
+    underground_description = models.TextField(blank=True,null=True)
+    photo = models.TextField(blank=True,null=True)
+    MARKING_CHOICES = (
+        ('P', 'Paint'),
+        ('P?', 'Paint (?)'),
+        ('T', 'Tag'),
+        ('T?', 'Tag (?)'),
+        ('R', 'Needs Retag'),
+        ('S', 'Spit'),
+        ('S?', 'Spit (?)'),
+        ('U', 'Unmarked'),
+        ('?', 'Unknown'))
+    marking = models.CharField(max_length=2, choices=MARKING_CHOICES)
+    marking_comment = models.TextField(blank=True,null=True)
+    FINDABLE_CHOICES = (
+        ('?', 'To be confirmed ...'),
+        ('S', 'Coordinates'),
+        ('L', 'Lost'),
+        ('R', 'Refindable'))
+    findability = models.CharField(max_length=1, choices=FINDABLE_CHOICES, blank=True, null=True)
+    findability_description = models.TextField(blank=True,null=True)
+    alt = models.TextField(blank=True, null=True)
+    northing = models.TextField(blank=True, null=True)
+    easting = models.TextField(blank=True, null=True)
+    tag_station = models.TextField(blank=True, null=True)
+    exact_station = models.TextField(blank=True, null=True)
+    other_station = models.TextField(blank=True, null=True)
+    other_description = models.TextField(blank=True,null=True)
+    bearings = models.TextField(blank=True,null=True)
+    url = models.CharField(max_length=200,blank=True,null=True)
+    filename = models.CharField(max_length=200)
+    cached_primary_slug = models.CharField(max_length=200,blank=True,null=True)
+
+    def __unicode__(self):
+        return unicode(self.slug())
+
+    def exact_location(self):
+        return SurvexStation.objects.lookup(self.exact_station)
+    def other_location(self):
+        return SurvexStation.objects.lookup(self.other_station)
+
+
+    def find_location(self):
+        r = {'':  'To be entered ',
+			 '?': 'To be confirmed:',
+             'S': '',
+             'L': 'Lost:',
+             'R': 'Refindable:'}[self.findability]
+        if self.tag_station:
+            try:
+                s = SurvexStation.objects.lookup(self.tag_station)
+                return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z)
+            except:
+                return r + "%s Tag Station not in dataset" % self.tag_station
+        if self.exact_station:
+            try:
+                s = SurvexStation.objects.lookup(self.exact_station)
+                return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z)
+            except:
+                return r + "%s Exact Station not in dataset" % self.tag_station
+        if self.other_station:
+            try:
+                s = SurvexStation.objects.lookup(self.other_station)
+                return r + "%0.0fE %0.0fN %0.0fAlt %s" % (s.x, s.y, s.z, self.other_description)
+            except:
+                return r + "%s Other Station not in dataset" % self.tag_station
+        if self.FINDABLE_CHOICES == "S":
+            r += "ERROR, Entrance has been surveyed but has no survex point"
+        if self.bearings:
+            return r + self.bearings
+        return r
+
+    def best_station(self):
+        if self.tag_station:
+            return self.tag_station
+        if self.exact_station:
+            return self.exact_station
+        if self.other_station:
+            return self.other_station
+
+    def has_photo(self):
+        if self.photo:
+            if (self.photo.find("<img") > -1 or self.photo.find("<a") > -1 or self.photo.find("<IMG") > -1 or self.photo.find("<A") > -1):
+                return "Yes"
+            else:
+                return "Missing"
+        else:
+            return "No"
+
+    def marking_val(self):
+        for m in self.MARKING_CHOICES:
+            if m[0] == self.marking:
+                return m[1]
+    def findability_val(self):
+        for f in self.FINDABLE_CHOICES:
+            if f[0] == self.findability:
+                return f[1]
+                
+    def tag(self):
+        return SurvexStation.objects.lookup(self.tag_station)
+        
+    def needs_surface_work(self):
+        return self.findability != "S" or not self.has_photo or self.marking != "T"
+
+    def get_absolute_url(self):
+        
+        ancestor_titles='/'.join([subcave.title for subcave in self.get_ancestors()])
+        if ancestor_titles:
+            res = '/'.join((self.get_root().cave.get_absolute_url(), ancestor_titles, self.title))
+        
+        else:
+            res = '/'.join((self.get_root().cave.get_absolute_url(), self.title))
+            
+        return res
+
+    def slug(self):
+        if not self.cached_primary_slug:
+            primarySlugs = self.entranceslug_set.filter(primary = True)
+            if primarySlugs:    
+                self.cached_primary_slug = primarySlugs[0].slug
+                self.save()
+            else:
+                slugs = self.entranceslug_set.filter()
+                if slugs:
+                    self.cached_primary_slug = slugs[0].slug
+                    self.save()
+        return self.cached_primary_slug
+
+    def writeDataFile(self):
+        try:
+            f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w")
+        except:
+            subprocess.call(settings.FIX_PERMISSIONS) 
+            f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w")
+        t = loader.get_template('dataformat/entrance.xml')
+        c = Context({'entrance': self})
+        u = t.render(c)
+        u8 = u.encode("utf-8")
+        f.write(u8)
+        f.close()
+
+class CaveDescription(TroggleModel):
+    short_name = models.CharField(max_length=50, unique = True)
+    long_name = models.CharField(max_length=200, blank=True, null=True)
+    description = models.TextField(blank=True,null=True)
+    linked_subcaves = models.ManyToManyField("NewSubCave", blank=True,null=True)
+    linked_entrances = models.ManyToManyField("Entrance", blank=True,null=True)
+    linked_qms = models.ManyToManyField("QM", blank=True,null=True)
+
+    def __unicode__(self):
+        if self.long_name:
+            return unicode(self.long_name)
+        else:
+            return unicode(self.short_name)
+    
+    def get_absolute_url(self):
+        return urlparse.urljoin(settings.URL_ROOT, reverse('cavedescription', args=(self.short_name,)))
+    
+    def save(self):
+        """
+        Overridden save method which stores wikilinks in text as links in database.
+        """
+        super(CaveDescription, self).save()
+        qm_list=get_related_by_wikilinks(self.description)
+        for qm in qm_list:
+            self.linked_qms.add(qm)
+        super(CaveDescription, self).save()
+
+class NewSubCave(TroggleModel):
+    name = models.CharField(max_length=200, unique = True)
+    def __unicode__(self):
+        return unicode(self.name)
+
+class QM(TroggleModel):
+    #based on qm.csv in trunk/expoweb/1623/204 which has the fields:
+    #"Number","Grade","Area","Description","Page reference","Nearest station","Completion description","Comment"
+    found_by = models.ForeignKey(LogbookEntry, related_name='QMs_found',blank=True, null=True )
+    ticked_off_by = models.ForeignKey(LogbookEntry, related_name='QMs_ticked_off',null=True,blank=True)
+    #cave = models.ForeignKey(Cave)
+    #expedition = models.ForeignKey(Expedition)
+
+    number = models.IntegerField(help_text="this is the sequential number in the year", )
+    GRADE_CHOICES=(
+	('A', 'A: Large obvious lead'),
+	('B', 'B: Average lead'),
+	('C', 'C: Tight unpromising lead'),
+	('D', 'D: Dig'),
+	('X', 'X: Unclimbable aven')
+    )
+    grade = models.CharField(max_length=1, choices=GRADE_CHOICES)
+    location_description = models.TextField(blank=True)
+    #should be a foreignkey to surveystation
+    nearest_station_description = models.CharField(max_length=400,null=True,blank=True)
+    nearest_station = models.CharField(max_length=200,blank=True,null=True)
+    area = models.CharField(max_length=100,blank=True,null=True)
+    completion_description = models.TextField(blank=True,null=True)
+    comment=models.TextField(blank=True,null=True)
+
+    def __unicode__(self):
+        return u"%s %s" % (self.code(), self.grade)
+
+    def code(self):
+        return u"%s-%s-%s" % (unicode(self.found_by.cave)[6:], self.found_by.date.year, self.number)
+
+    def get_absolute_url(self):
+        #return settings.URL_ROOT + '/cave/' + self.found_by.cave.kataster_number + '/' + str(self.found_by.date.year) + '-' + '%02d' %self.number
+        return urlparse.urljoin(settings.URL_ROOT, reverse('qm',kwargs={'cave_id':self.found_by.cave.kataster_number,'year':self.found_by.date.year,'qm_id':self.number,'grade':self.grade}))
+
+    def get_next_by_id(self):
+        return QM.objects.get(id=self.id+1)
+
+    def get_previous_by_id(self):
+        return QM.objects.get(id=self.id-1)
+
+    def wiki_link(self):
+        return u"%s%s%s" % ('[[QM:',self.code(),']]')
+
+photoFileStorage = FileSystemStorage(location=settings.PHOTOS_ROOT, base_url=settings.PHOTOS_URL)
+class DPhoto(TroggleImageModel): 
+    caption = models.CharField(max_length=1000,blank=True,null=True)
+    contains_logbookentry = models.ForeignKey(LogbookEntry,blank=True,null=True)
+    contains_person = models.ManyToManyField(Person,blank=True,null=True)
+    file = models.ImageField(storage=photoFileStorage, upload_to='.',)
+    is_mugshot = models.BooleanField(default=False)
+    contains_cave = models.ForeignKey(Cave,blank=True,null=True)
+    contains_entrance = models.ForeignKey(Entrance, related_name="photo_file",blank=True,null=True)
+    #nearest_survey_point = models.ForeignKey(SurveyStation,blank=True,null=True)
+    nearest_QM = models.ForeignKey(QM,blank=True,null=True)
+    lon_utm = models.FloatField(blank=True,null=True)
+    lat_utm = models.FloatField(blank=True,null=True)
+    
+    class IKOptions:
+        spec_module = 'core.imagekit_specs'
+        cache_dir = 'thumbs'
+        image_field = 'file'
+        
+    #content_type = models.ForeignKey(ContentType)
+    #object_id = models.PositiveIntegerField()
+    #location = generic.GenericForeignKey('content_type', 'object_id')
+
+    def __unicode__(self):
+        return self.caption
+
+scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL)
+def get_scan_path(instance, filename):
+    year=instance.survey.expedition.year
+    #print("WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number, instance.survey.wallet_letter)
+    number=str(instance.survey.wallet_number)
+    if str(instance.survey.wallet_letter) != "None":
+        number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01
+    return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg')
+
+class ScannedImage(TroggleImageModel): 
+    file = models.ImageField(storage=scansFileStorage, upload_to=get_scan_path)
+    scanned_by = models.ForeignKey(Person,blank=True, null=True)
+    scanned_on = models.DateField(null=True)
+    survey = models.ForeignKey('Survey')
+    contents = models.CharField(max_length=20,choices=(('notes','notes'),('plan','plan_sketch'),('elevation','elevation_sketch')))
+    number_in_wallet = models.IntegerField(null=True)
+    lon_utm = models.FloatField(blank=True,null=True)
+    lat_utm = models.FloatField(blank=True,null=True)
+
+    class IKOptions:
+        spec_module = 'core.imagekit_specs'
+        cache_dir = 'thumbs'
+        image_field = 'file'
+    #content_type = models.ForeignKey(ContentType)
+    #object_id = models.PositiveIntegerField()
+    #location = generic.GenericForeignKey('content_type', 'object_id')
+
+    #This is an ugly hack to deal with the #s in our survey scan paths. The correct thing is to write a custom file storage backend which calls urlencode on the name for making file.url but not file.path.
+    def correctURL(self):
+        return string.replace(self.file.url,r'#',r'%23')
+    
+    def __unicode__(self):
+        return get_scan_path(self,'')
+
+class Survey(TroggleModel):
+    expedition = models.ForeignKey('Expedition') #REDUNDANT (logbook_entry)
+    wallet_number = models.IntegerField(blank=True,null=True)
+    wallet_letter = models.CharField(max_length=1,blank=True,null=True)
+    comments = models.TextField(blank=True,null=True)
+    location = models.CharField(max_length=400,blank=True,null=True) #REDUNDANT
+    subcave = models.ForeignKey('NewSubCave', blank=True, null=True)
+    #notes_scan = models.ForeignKey('ScannedImage',related_name='notes_scan',blank=True, null=True)  	#Replaced by contents field of ScannedImage model
+    survex_block  = models.OneToOneField('SurvexBlock',blank=True, null=True)
+    logbook_entry = models.ForeignKey('LogbookEntry')
+    centreline_printed_on = models.DateField(blank=True, null=True)
+    centreline_printed_by = models.ForeignKey('Person',related_name='centreline_printed_by',blank=True,null=True)
+    #sketch_scan = models.ForeignKey(ScannedImage,blank=True, null=True) 					#Replaced by contents field of ScannedImage model
+    tunnel_file = models.FileField(upload_to='surveyXMLfiles',blank=True, null=True)
+    tunnel_main_sketch = models.ForeignKey('Survey',blank=True,null=True)
+    integrated_into_main_sketch_on = models.DateField(blank=True,null=True)
+    integrated_into_main_sketch_by = models.ForeignKey('Person' ,related_name='integrated_into_main_sketch_by', blank=True,null=True)
+    rendered_image = models.ImageField(upload_to='renderedSurveys',blank=True,null=True)
+    def __unicode__(self):
+        return self.expedition.year+"#"+"%02d" % int(self.wallet_number)
+
+    def notes(self):
+        return self.scannedimage_set.filter(contents='notes')
+
+    def plans(self):
+        return self.scannedimage_set.filter(contents='plan')
+
+    def elevations(self):
+        return self.scannedimage_set.filter(contents='elevation')

core/views_caves.py

@@ -21,6 +21,34 @@ import settings
 from PIL import Image, ImageDraw, ImageFont
 import string, os, sys, subprocess
 
+#
+#   NEW CONTENT
+#
+
+
+from troggle.core.models import CaveM, Cave_descriptionM, ExpeditionM
+
+def millenialcaves(request):
+    #RW messing around area
+    caves = CaveM.objects.all()
+    descr = Cave_descriptionM.objects.all()
+    return render_with_context(request,'millenialcaves.html',{'caves': caves,'descriptions' : descr})    
+
+def millenialdescription(request, slug):
+    desc = Cave_descriptionM.objects.get(slug=slug)
+    return render_with_context(request,'cave_uground_description.html', {'cave': desc})
+
+def millenialpeople(request):
+    expos = ExpeditionM.objects.all()
+    return render_with_context(request,'peoplemillenial.html' , {'expos': expos})
+
+
+#
+#   END NEW CONTENT
+#
+
+
+
 def getCave(cave_id):
     """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm."""
     try:
@@ -54,16 +82,16 @@ def caveindex(request):
     caves = Cave.objects.all()
     notablecavehrefs = settings.NOTABLECAVESHREFS
     notablecaves = [Cave.objects.get(kataster_number=kataster_number)  for kataster_number in notablecavehrefs ]
-    caves1623 = list(Cave.objects.filter(area__short_name = "1623"))
+    #caves1623 = list(Cave.objects.filter(area__short_name = "1623"))
+    caves1623 = list(Cave.objects.all())    
     caves1626 = list(Cave.objects.filter(area__short_name = "1626"))
     caves1623.sort(caveCmp)
     caves1626.sort(caveCmp)
     return render_with_context(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':notablecaves, 'cavepage': True})
 
-def millenialcaves(request):
-    #RW messing around area
-    return HttpResponse("Test text", content_type="text/plain")
-    
+
+
+
 
 
 def cave3d(request, cave_id=''):

core/views_survex.py

@@ -40,6 +40,62 @@ survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau
 
 *end [surveyname]"""        
         
+
+def millenialcaves(request):
+    cavesdir = os.path.join(settings.SURVEX_DATA, "caves-1623")
+    #cavesdircontents = { }
+    
+    onefilecaves = [ ]
+    multifilecaves = [ ]
+    subdircaves = [ ]
+    
+    millenialcaves = [ ]
+
+    
+    # go through the list and identify the contents of each cave directory
+    for cavedir in os.listdir(cavesdir):
+        if cavedir in ["144", "40"]: #????? RW
+            continue
+            
+        gcavedir = os.path.join(cavesdir, cavedir) #directory od 'large' cave        
+
+        if os.path.isdir(gcavedir) and cavedir[0] != ".":            
+            subdirs, subsvx = identifycavedircontents(gcavedir)
+            survdirobj = [ ]
+            
+            for lsubsvx in subsvx:
+                survdirobj.append(("caves-1623/"+cavedir+"/"+lsubsvx, lsubsvx))
+            
+            # caves with subdirectories
+            if subdirs:
+                subsurvdirs = [ ]
+                for subdir in subdirs:
+                    dsubdirs, dsubsvx = identifycavedircontents(os.path.join(gcavedir, subdir))
+                    assert not dsubdirs
+                    lsurvdirobj = [ ]
+                    for lsubsvx in dsubsvx:
+                        lsurvdirobj.append(("caves-1623/"+cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))
+                    subsurvdirs.append((lsurvdirobj[0], lsurvdirobj[1:]))
+                subdircaves.append((cavedir, (survdirobj[0], survdirobj[1:]), subsurvdirs))
+            
+            # multifile caves
+            elif len(survdirobj) > 1:
+                multifilecaves.append((survdirobj[0], survdirobj[1:]))
+            # single file caves
+            else:
+                #print("survdirobj = ")
+                #print(survdirobj)
+                onefilecaves.append(survdirobj[0])
+    
+    caves = Cave.objects.all()
+
+    return render_to_response('millenialcaves.html', {'settings': settings , 'caves':caves , "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves })
+
+
+
+
+
+
         
 def ReplaceTabs(stext):
     res = [ ]

databaseResetM.py

@@ -0,0 +1,127 @@
+import os
+import time
+import settings
+os.environ['PYTHONPATH'] = settings.PYTHON_PATH
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
+from django.core import management
+from django.db import connection
+from django.contrib.auth.models import User
+from django.http import HttpResponse
+from django.core.urlresolvers import reverse
+from troggle.core.models import Cave, Entrance
+from troggle.core.models import PersonM, SurveyM, CaveM, ExpeditionM, Logbook_entryM, Cave_descriptionM
+import troggle.flatpages.models
+
+databasename=settings.DATABASES['default']['NAME']
+expouser=settings.EXPOUSER
+expouserpass=settings.EXPOUSERPASS
+expouseremail=settings.EXPOUSER_EMAIL
+
+def destroy():
+    if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
+        try:
+            os.remove(databasename)
+        except OSError:
+            pass
+    else:
+        cursor = connection.cursor()
+        cursor.execute("DROP DATABASE %s" % databasename)
+        cursor.execute("CREATE DATABASE %s" % databasename)
+        cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % databasename)
+        cursor.execute("USE %s" % databasename)
+    management.call_command('syncdb', interactive=False)
+    user = User.objects.create_user(expouser, expouseremail, expouserpass)
+    user.is_staff = True
+    user.is_superuser = True
+    user.save()
+    print('Nuked the database and rebuilt it. You savage monster')
+
+def gracefull_flush():
+    CaveM.objects.all().delete()
+    PersonM.objects.all().delete()
+    SurveyM.objects.all().delete()
+    ExpeditionM.objects.all().delete()
+    Logbook_entryM.objects.all().delete()
+    Cave_descriptionM.objects.all().delete()
+    print('Deleted contents of the database, ready to load new stuff :)')
+
+def load_redirects():
+    for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
+        f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
+        f.save()
+
+def load_surveys():
+    SurveyM.objects.all().delete()
+    import troggle.parsers.surveysM
+    troggle.parsers.surveysM.load()
+
+def load_caves():
+    import troggle.parsers.cavesM
+    troggle.parsers.cavesM.load()
+
+def load_people():
+    import troggle.parsers.peopleM
+    troggle.parsers.peopleM.load()
+
+def load_all():
+    load_caves()
+    load_surveys()
+    load_people()
+    load_redirects()
+    load_links()
+    print('Loaded everything. Your database is ready to go :)')
+
+
+def help():
+    print("""Usage is 'python databaseResetM.py <command>'
+             where command is:
+             UNLOADERS:
+             gracefull_flush - flushes new (M-style) databases contents but keeps tables existing
+             destroy - destroys entire database and builds empty tables
+
+             LOADERS:
+             load_all - loads all tables and links
+             load_caves - loads all caves
+             load_surveys - loads all surveys (corresponds to .svx files)
+             load_people - loads all people
+             load_redirects - load page redirects
+             load_links - loads links between classes (run last! can't link non-existent things)
+             
+             OTHER:
+             help - displays this page
+             ----------------
+             This is a new version of database management written by RW 2019   
+             ----------------         
+             """)
+
+if __name__ == "__main__":
+    import troggle.core.models
+    import sys
+    import django
+    django.setup()
+    if "destroy" in sys.argv:
+        destroy()
+    elif "gracefull_flush" in sys.argv:
+        gracefull_flush()
+
+    elif "load_all" in sys.argv:
+        load_all()
+    elif "load_caves" in sys.argv:
+        load_caves()
+    elif "load_surveys" in sys.argv:
+        load_surveys()
+    elif "load_people" in sys.argv:
+        load_people()
+    elif "load_redirects" in sys.argv:
+        load_redirects()
+    elif "load_links" in sys.argv:
+        load_links()
+    elif "help" in sys.argv:
+        help()
+    else:
+        print("%s not recognised" % sys.argv)
+        help()
+
+
+    
+    

media/css/cavetables.css

@@ -0,0 +1,52 @@
+body {
+    all: initial;
+    font-size: 100%;
+}
+
+div#inputf {
+    display: inline-block;
+    width: 300px;
+    text-align: justify;
+    margin-top: 0px;
+    margin-bottom: 5px
+}
+
+.menu, ul#links{
+    display: none;
+}
+
+table {
+  border-spacing: 0;
+  width: 100%;
+  border: 1px solid #ddd;
+  font-family: monospace;
+}
+
+th {
+  cursor: pointer;
+  background-color: #bbb
+}
+
+th, td {
+  padding: 16px;
+  max-height: 40px;
+}
+
+tr:nth-child(even) {
+  background-color: #f2f2f2
+}
+
+p {
+
+  margin-right: 80px;
+  margin-left: 80px;
+}
+
+button {
+  width: 300px
+}
+span#mono {
+  font-family: monospace;
+  background-color: #eee;
+  font-size: 120%;
+}

media/scripts/TableSort.js

@@ -0,0 +1,159 @@
+function filterTable(tablename)
+{
+  table = document.getElementById(tablename);
+
+  mindepth = document.getElementById("CaveDepthMin").value;
+  maxdepth = document.getElementById("CaveDepthMax").value;
+  if(mindepth==0)mindepth=-999999;
+  if(maxdepth==0)maxdepth= 999999;
+
+  minlength = document.getElementById("CaveLengthMin").value;
+  maxlength = document.getElementById("CaveLengthMax").value;
+  if(minlength==0)minlength=-999999;
+  if(maxlength==0)maxlength= 999999;
+
+  visitdate = document.getElementById("VisitDate").value;
+
+  visitor = document.getElementById("Visitor").value;
+
+  cavename = document.getElementById("CaveName").value.toLowerCase();
+
+  incomplete = document.getElementById("Incomplete").checked;
+
+  var regexmode = false;
+  if(visitor[0]=='/' && visitor[visitor.length-1]=='/')
+  {
+     regexmode = true;
+     visitor = new RegExp(visitor.substr(1,visitor.length-2));
+  }
+  else
+  {
+     visitor.toLowerCase();
+  }
+
+  rows = table.rows;
+  for(i=1; i< rows.length; i++)
+  {
+    name  = (rows[i].getElementsByTagName("TD")[1]).innerHTML.toLowerCase();
+
+    depth = (rows[i].getElementsByTagName("TD")[2]).innerHTML.toLowerCase();
+    depth = Number(depth.replace(/[^0-9.]/g,''));
+
+    length = (rows[i].getElementsByTagName("TD")[3]).innerHTML.toLowerCase();
+    length = Number(length.replace(/[^0-9.]/g,''));
+
+    date = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase();
+
+    //recentvisitor = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase();
+    recentvisitor = ""
+
+    if(cavename != "" && !name.includes(cavename))
+    {
+      rows[i].style.visibility = "collapse";
+    }
+    if(depth<mindepth || depth>maxdepth)
+    {
+      rows[i].style.visibility = "collapse";
+    }
+    if(length<minlength || length>maxlength)
+    {
+      rows[i].style.visibility = "collapse";
+    }
+    if(date < visitdate)
+    {
+      rows[i].style.visibility = "collapse";
+    }
+    if(visitor != "" && regexmode && !visitor.test(recentvisitor))
+    {
+      rows[i].style.visibility = "collapse";
+    }
+    if(visitor != "" && !regexmode && !recentvisitor.includes(visitor))
+    {
+      rows[i].style.visibility = "collapse";
+    }
+    
+    crow=rows[i].getElementsByTagName("TD");
+    for(var j=0; j<crow.length; j++)
+    {
+      if(crow[j].innerHTML == "" && incomplete)
+      {
+        rows[i].style.visibility = "collapse";
+        break;
+      }
+    }
+
+  }
+}
+
+function filterTableReset(tablename)
+{
+  table = document.getElementById(tablename);
+  rows = table.rows;
+  for(i=1; i< rows.length; i++)
+  {
+    rows[i].style.visibility = "visible";
+  }
+}
+
+
+function isOrdered(kvarray,numeric)
+{
+  for(var i=0;i<kvarray.length-1;i++)
+  {
+    if(numeric==1 && Number(kvarray[i][0])>Number(kvarray[i+1][0]))
+    {
+      return false;
+    }
+    if(numeric!=1 && kvarray[i][0]>kvarray[i+1][0])
+    {
+      return false;
+    }
+  }
+  return true;
+}
+
+function sortTable(n, tablename, numeric) {
+  table = document.getElementById(tablename);
+  rows = table.rows;
+  var ordering = [];
+  var i;
+  
+  //construct key-value pairs for sorting
+  for(i = 1; i < rows.length; i++) //remember header rows
+  {
+    key = rows[i].getElementsByTagName("TD")[n];
+    key = key.innerHTML.toLowerCase();
+    if(numeric==1)
+    {
+      key=key.replace(/[^0-9.]/g,'')
+    }
+    ordering.push([key,i]);
+  }
+
+  var ascending = isOrdered(ordering,numeric);
+  
+  //sort either numerically or alphabetically
+  if(numeric==1)
+  {
+    ordering.sort((x,y) => Number(x[0])-Number(y[0]));
+  }
+  else
+  {
+    ordering.sort(); //sorts alphabetically
+  }
+
+  if(ascending) ordering.reverse();
+
+  for(i = 0; i < ordering.length; i++) //add sorted list at the end of the table
+  {
+    var keyval = ordering[i];
+    id = keyval[1]; //get rownumber of n^th sorted value
+    cln = rows[id].cloneNode(true); //deep clone of current node
+    table.insertBefore(cln,null); //add n^th row at the end
+  }
+  for(i = 1; i < ordering.length+1; i++) //remove unsorted nodes
+  {
+    table.deleteRow(1);// 0 -> header; 1 -> first row
+  }
+
+}

parsers/cavesM.py

@@ -0,0 +1,213 @@
+
+import troggle.core.models as models #import models for various objects
+from django.conf import settings
+import xml.etree.ElementTree as ET #this is used to parse XML's
+import subprocess
+import re
+
+#
+#    This parser has to find several things:
+#    There are files of .html format in expoweb area - they contain some of the important information
+#    There is a similar number of .svx files in loser are - they contain all the measurements
+#
+#    Previous version was incredibly slow due to various shitty ideas about finding things 
+#    and overelayance on python when handling regular expressions, new version delegates heavy lifting to shell
+#    and handles more sophisticated bits only
+#
+
+def load():
+    print('Hi! I\'m caves parser. Ready to work')
+    
+    print('Loading caves of 1623 area')
+    loadarea('1623')
+
+    print('Loading caves of 1626 area')
+    loadarea('1626')
+
+
+def loadarea(areacode):
+
+    if not file_exists(settings.SURVEX_DATA+'1623-and-1626.3d'):
+        print('Computing master .3d file')
+        bash('cavern -o'+settings.SURVEX_DATA+' '+settings.SURVEX_DATA+'1623-and-1626.svx')
+    else:
+        print('Loading from existing master .3d file')
+
+    master3d = bash('dump3d -d '+settings.SURVEX_DATA+'1623-and-1626.3d').splitlines()
+    master3dN = [x for x in master3d if ('NODE' in x)] #list of nodes of master survex file  
+    master3dL = [x for x in master3d if ('LINE' in x)] #list of nodes of master survex file      
+
+    print('Searching all cave dirs files')
+    basedir = settings.SURVEX_DATA+'caves-'+areacode+'/'
+
+    cavedirs = bash("find "+basedir+" -maxdepth 1 -type d").splitlines() #this command finds all directories
+    print('Obtained list of directories! (#dirs='+str(len(cavedirs))+')')
+    ndirs = len(cavedirs) #remember number of dirs for nice debug output
+
+    for cavedir in cavedirs:
+        if cavedir==basedir:
+            continue #skip the basedir - a non-proper subdirectory
+        cavename = bash('echo '+cavedir+' | rev | cut -f1 -d \'/\' | rev').splitlines()[0] #get final bit of the directory
+        
+        test = bash('if [ ! -f '+cavedir+'/'+cavename+'.svx ] ; then echo MISSING; fi')#test for file exisence
+        if not file_exists(cavedir+'/'+cavename+'.svx'):
+            msg = models.Parser_messageM(parsername='caves',content=cavedir+'/'+cavename+' MISSING!',message_type='warn')
+            print('Cave missing'+cavename+' :(')
+            msg.save()
+            continue
+        fullname=cavedir+'/'+cavename+'.svx'        
+        print('Found cave:'+cavename)
+        cavernout = bash('cavern -o '+cavedir+' '+fullname) #make cavern process the thing
+        if 'cavern: error:' in cavernout:
+            msg = models.Parser_messageM(parsername='caves',content=cavedir+'/'+cavename+' Survex file messed up!',message_type='warn')
+            print('Fucked svx'+cavename+' :(')
+            msg.save()
+            continue
+        
+        cavernout = cavernout.splitlines()
+        depth = float(([x for x in cavernout if ('Total vertical length' in x)][0].split()[-1])[:-2])
+        length = float(([x for x in cavernout if ('Total length' in x)][0].split()[6])[:-1])
+        cavefile = open(fullname,'r')
+        cavefilecontents = cavefile.read().splitlines()
+        surveyname = [x for x in cavefilecontents if ('*begin ') in x][0].split()[1].lower()          
+        try:            
+            title = [x for x in cavefilecontents if ('*title ') in x][0].split()[1]              
+        except:
+            syrveyname = "Untitled"
+
+        relevant_nodes = [x for x in master3dN if (('['+areacode+'.'+surveyname+'.' in x) or ('['+areacode+'.'+surveyname+']' in x))] 
+        entrance_nodes = [x for x in relevant_nodes if 'ENTRANCE' in x]
+        surface_nodes = [x for x in relevant_nodes if 'SURFACE' in x]
+        location_nodes = []
+        print('rel_nodes'+str(len(relevant_nodes)))
+        if len(entrance_nodes) > 0:
+            location_nodes = entrance_nodes
+        elif len(surface_nodes) > 0:
+            location_nodes = surface_nodes
+        elif len(relevant_nodes) > 0:
+            location_nodes = relevant_nodes
+
+        try:
+            location = sorted(location_nodes, key = lambda y : float(y.split()[3])).pop()
+        except:
+            print(location_nodes)
+            location = 'Not found'
+        
+        relevant_lines = [x for x in master3dL if (('['+areacode+'.'+surveyname+'.' in x) or ('['+areacode+'.'+surveyname+']' in x))]
+        try:
+            lastleg = sorted(relevant_lines, key = lambda y : y.split().pop()).pop()
+        except:
+            lastleg = ['LINE 1900.01.01']
+        try:        
+            lastdate = lastleg.split().pop()
+            if 'STYLE' in lastdate:
+                lastdate = lastleg.split().pop().pop()
+        except:
+            lastdate = '1900.01.01'
+        
+        entrance = ' '.join(location.split()[1:3])
+        print((('depth','length','surv name','entr','date'),(depth,length,surveyname,entrance,lastdate))) #sanity check print
+                
+        
+        newcave =  models.CaveM(
+            survex_file = fullname,
+            total_length = length,
+            name=areacode+'.'+surveyname,
+            total_depth = depth,
+            date = lastdate,
+            entrance = entrance)
+        newcave.save()
+    #end of reading survex masterfiles
+    
+    print ("Reading cave descriptions")
+    cavefiles = bash('find '+settings.CAVEDESCRIPTIONS+' -name \'*.html\'').splitlines()
+    for fn in cavefiles:
+        f = open(fn, "r")
+        print(fn)
+        contents = f.read()    
+        
+        slug = re.sub(r"\s+", "", extractXML(contents,'caveslug'))
+        desc = extractXML(contents,'underground_description')
+        name = slug[5:] #get survex compatible name
+        area = slug[0:4]
+        
+        print([area,name])
+        
+        if desc==None or name==None:
+            msg = models.Parser_messageM(parsername='caves',content=fn+' Description meesed up!',message_type='warn')
+            print('Fucked description '+fn+' :(')
+            msg.save()
+            continue
+
+            print(area+'/'+name+'/'+name+'.svx')
+        
+        updatecave = models.CaveM.objects.filter(survex_file__icontains=area+'/'+name+'/'+name+'.svx')
+        if len(updatecave)>1:
+            print('Non unique solution - skipping. Name:'+name)
+        elif len(updatecave)==0:
+            print('Cave with no survex data:'+name)
+            continue
+        else: #exaclty one match
+            print('Adding desc:'+name)
+            updatecave = updatecave[0]
+            updatecave.description = '/cave/descriptionM/'+slug #area-name
+            updatecave.title=name
+            updatecave.save()
+
+            slugS = slug
+            explorersS = extractXML(contents,'explorers')
+            underground_descriptionS = extractXML(contents,'underground_description')
+            equipmentS = extractXML(contents,'equipment')
+            referencesS = extractXML(contents,'references')
+            surveyS = extractXML(contents,'survey')
+            kataster_statusS = extractXML(contents,'kataster_status')
+            underground_centre_lineS = extractXML(contents,'underground_centre_line')
+            survex_fileS = extractXML(contents,'survex_file')
+            notesS = extractXML(contents,'notes')
+
+
+            newcavedesc =  models.Cave_descriptionM(
+            slug = slugS,            
+            explorers = explorersS,
+            underground_description = underground_descriptionS, 
+            equipment = equipmentS, 
+            references = referencesS, 
+            survey = surveyS, 
+            kataster_status = kataster_statusS, 
+            underground_centre_line = underground_centre_lineS, 
+            survex_file = survex_fileS, 
+            notes = notesS)
+            newcavedesc.save()
+            
+
+            
+            
+    #end of reading cave descriptions
+    
+def file_exists(filename):
+    test = bash('if [ ! -f '+filename+' ] ; then echo MISSING; fi')#test for file exisence
+    if 'MISSING' in test: #send error message to the database
+        return False
+    return True       
+
+def extractXML(contents,tag):
+    #find correct lines
+    lines = contents.splitlines()
+    beg = [x for x in lines if ('<'+tag+'>' in x)]
+    end = [x for x in lines if ('</'+tag+'>' in x)]
+    if (not beg) or (not end):
+        return None       
+    begi = lines.index(beg[0])
+    endi = lines.index(end[0])
+    if endi!=begi:
+        segment = '\n'.join(lines[begi:endi+1])
+    else:
+        segment = lines[begi:endi+1][0]
+
+    hit = re.findall('<'+tag+'>(.*)</'+tag+'>', segment, re.S)[0]
+    return hit
+
+def bash(cmd): #calls command in bash shell, returns output
+    process = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
+    output, error = process.communicate()
+    return output

parsers/people.py

@@ -62,8 +62,6 @@ def LoadPersonsExpos():
     
     # make persons
     print "Loading personexpeditions"
-    #expoers2008 = """Edvin Deadman,Kathryn Hopkins,Djuke Veldhuis,Becka Lawson,Julian Todd,Natalie Uomini,Aaron Curtis,Tony Rooke,Ollie Stevens,Frank Tully,Martin Jahnke,Mark Shinwell,Jess Stirrups,Nial Peters,Serena Povia,Olly Madge,Steve Jones,Pete Harley,Eeva Makiranta,Keith Curtis""".split(",")
-    #expomissing = set(expoers2008)
 
     for personline in personreader:
         name = personline[header["Name"]]
@@ -85,36 +83,7 @@ def LoadPersonsExpos():
                 nonLookupAttribs = {'nickname':nickname, 'is_guest':(personline[header["Guest"]] == "1")}
                 save_carefully(models.PersonExpedition, lookupAttribs, nonLookupAttribs)
 
-
-    # this fills in those people for whom 2008 was their first expo
-    #print "Loading personexpeditions 2008"
-    #for name in expomissing:
-        # firstname, lastname = name.split()
-        # is_guest = name in ["Eeva Makiranta", "Keith Curtis"]
-        # print "2008:", name
-        # persons = list(models.Person.objects.filter(first_name=firstname, last_name=lastname))
-        # if not persons:
-            # person = models.Person(first_name=firstname, last_name = lastname, is_vfho = False, mug_shot = "")
-            # #person.Sethref()
-            # person.save()
-        # else:
-            # person = persons[0]
-        # expedition = models.Expedition.objects.get(year="2008")
-        # personexpedition = models.PersonExpedition(person=person, expedition=expedition, nickname="", is_guest=is_guest)
-        # personexpedition.save()
-
-    #Notability is now a method of person. Makes no sense to store it in the database; it would need to be recalculated every time something changes. - AC 16 Feb 09
-    # could rank according to surveying as well
-    #print "Setting person notability"
-    #for person in models.Person.objects.all():
-        #person.notability = 0.0
-        #for personexpedition in person.personexpedition_set.all():
-            #if not personexpedition.is_guest:
-                #person.notability += 1.0 / (2012 - int(personexpedition.expedition.year))
-        #person.bisnotable = person.notability > 0.3 # I don't know how to filter by this
-        #person.save()
-        
-        
+                
 # used in other referencing parser functions
 # expedition name lookup cached for speed (it's a very big list)
 Gpersonexpeditionnamelookup = { }

parsers/peopleM.py

@@ -0,0 +1,27 @@
+from django.conf import settings
+import troggle.core.models as models
+
+def load():
+    folkfile = open(settings.EXPOWEB+"noinfo/folk.csv")
+    personlines = folkfile.read().splitlines()
+    persontable = [x.split(',') for x in personlines]
+    years = [persontable[0][i] for i in range(5,len(persontable[0]))]
+    for year in years:
+        newexpedition = models.ExpeditionM( date = year )
+        newexpedition.save()
+    for row in persontable[1:]: #skip header
+        attendedid = [i for i, x in enumerate(row) if '1' in x]
+        attendedyears = [persontable[0][i] for i in attendedid if i >= 5]
+        name = row[0]
+        print(name+' has attended: '+', '.join(attendedyears))
+        newperson = models.PersonM(
+        name = name)
+        newperson.save()
+        for year in attendedyears:
+            target = models.ExpeditionM.objects.get(date=year)
+            newperson.expos_attended.add( target )
+    print('Person -> Expo table created!')
+
+        
+        
+    

parsers/surveysM.py

@@ -0,0 +1,65 @@
+from django.conf import settings
+import subprocess, re
+import troggle.core.models as models
+
+def load():
+    print('Load survex files and relations')
+    load_area('1623')
+
+def load_area(areacode):
+
+    print('Searching all cave dirs files')
+    basedir = settings.SURVEX_DATA+'caves-'+areacode+'/'
+
+    cavedirs = bash("find "+basedir+" -maxdepth 1 -type d").splitlines() #this command finds all directories
+    print('Obtained list of directories! (#dirs='+str(len(cavedirs))+')')
+    
+    for cavedir in cavedirs:
+        if cavedir==basedir:
+            continue #skip the basedir - a non-proper subdirectory
+        parentname = bash('echo '+cavedir+' | rev | cut -f1 -d \'/\' | rev').splitlines()[0] #get final bit of the directory
+        parentcave = models.CaveM.objects.filter(survex_file__icontains=cavedir)
+        if len(parentcave)>1:
+            print('Non unique parent - skipping. Name:'+parentname)
+        elif len(parentcave)==0:
+            print('Error! parent not created:'+parentname)
+            continue
+        else: #exaclty one match
+            print('Adding relations of:'+parentname)
+            parentcave = parentcave[0]
+            
+            surveyfiles = bash('find '+cavedir+' -name \'*.svx\'').splitlines()
+            for fn in surveyfiles:
+                print(fn)
+                svxcontents = open(fn,'r').read().splitlines()
+                try:
+                    dateline = [x for x in svxcontents if ('*date' in x)][0]
+                    date = re.findall('\\d\\d\\d\\d\\.\\d\\d\\.\\d\\d', dateline, re.S)[0]
+                    
+                    
+                except:
+                    if( len( [x for x in svxcontents if ('*date' in x)] ) == 0 ):
+                        continue #skip dateless files                     
+                    print('Date format error in '+fn)
+                    print('Dateline = '+ '"'.join([x for x in svxcontents if ('*date' in x)]))
+                    date = '1900.01.01' 
+                
+            
+                newsurvex = models.SurveyM(survex_file=fn, date=date)
+                newsurvex.save()
+                parentcave.surveys.add(newsurvex)
+                parentcave.save()
+                        
+
+def file_exists(filename):
+    test = bash('if [ ! -f '+filename+' ] ; then echo MISSING; fi')#test for file exisence
+    if 'MISSING' in test: #send error message to the database
+        return False
+    return True       
+
+def bash(cmd): #calls command in bash shell, returns output
+    process = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
+    output, error = process.communicate()
+    return output
+        
+    

settings.py

@@ -71,6 +71,9 @@ if django.VERSION[0] == 1 and django.VERSION[1] < 4:
 else:
     authmodule = 'django.contrib.auth.context_processors.auth'
 
+TOPCAMPX=411571.00
+TOPCAMPY=5282639.00
+
 TEMPLATE_CONTEXT_PROCESSORS = ( authmodule, "core.context.troggle_context", ) 
 
 LOGIN_REDIRECT_URL = '/'

templates/caveindex.html

@@ -20,7 +20,7 @@
 <table class="searchable">
 {% for cave in caves1623 %}
 
-  <tr><td> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> </td></tr>
+  <tr><td> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> {{ cave.slug }}</td></tr>
 
 {% endfor %}
 </table>

templates/millenialcaves.html

@@ -0,0 +1,69 @@
+<html>
+<head>
+
+
+<link rel="stylesheet" type="text/css" href="{{ settings.MEDIA_URL }}/css/cavetables.css">
+
+</head>
+
+<body>
+
+<h2>Caves of loserplateau (locations acording to all.3d)</h2>
+<span style="font-size:70%">
+Name contains:<br>
+<div id="inputf"><input type="text" name="CaveName" id="CaveName" style="width:100%"></div><br>
+Depth between (min, max) in meters (0 disables filter):<br>
+<div id="inputf"><input type="number" name="CaveDepthMin" id="CaveDepthMin" style="width:45%"> - <input type="number" name="CaveDepthMax" id="CaveDepthMax" style="width:45%"></div><br>
+Length between (min, max) in meters (0 disables filter):<br>
+<div id="inputf"><input type="number" name="CaveLengthMin" id="CaveLengthMin" style="width:45%"> - <input type="number" name="CaveLengthMax" id="CaveLengthMax" style="width:45%"></div><br>
+Last visit after (date in YYYY.MM.DD format works best):<br>
+<div id="inputf"><input type="text" name="VisitDate" id="VisitDate" style="width:100%"></div><br>
+Last visited by (single word or regular expression, search is not case sensitive):<br>
+(e.g. <span id="mono">/da.e/</span> matches both Dave and Dane, <span id="mono">/w..k|ol{2}y/</span> matches either Wook and Olly)<br>
+<div id="inputf"><input type="text" name="Visitor" id="Visitor" style="width:100%"></div><br>
+Hide incomplete entries:<br>
+<div id="inputf"><input type="checkbox" name="Incomplete" id="Incomplete" style="width:100%"></div><br><br>
+
+<button onclick="filterTable('caves_table')">Filter</button><br>
+<button onclick="filterTableReset('caves_table')">Reset filters</button><br>
+
+Click on column headers to sort/reverse sort<br><br><br>
+</span>
+
+
+
+
+<table id="caves_table">
+<tr>
+<th onclick="sortTable(0,'caves_table',0)">Cave survex id</th>
+<th onclick="sortTable(1,'caves_table',0)">Cave name</th>
+<th onclick="sortTable(2,'caves_table',1)">Cave depth</th>
+<th onclick="sortTable(3,'caves_table',1)">Cave length</th>
+<th onclick="sortTable(4,'caves_table',0)">Last leg date</th>
+<th onclick="sortTable(5,'caves_table',0)">Cave location (UTM)</th>
+<th onclick="sortTable(6,'caves_table',0)">Cave location (lat/lon)</th>
+<th onclick="sortTable(7,'caves_table',1)">Top camp distance [m]</th>
+</tr>
+
+{% for cave in caves %}
+<tr>
+    <td><a href={{cave.description}}>{{ cave.name         }}</a></td>
+    <td>{{ cave.title            }}</td>
+    <td>{{ cave.total_depth      }}</td>
+    <td>{{ cave.total_length     }}</td>
+    <td>{{ cave.date             }}</td>
+    <td>33U {{ cave.entrance     }}</td>
+    <td>{{ cave.lat_lon_entrance }}</td>
+    <td>{{ cave.top_camp_distance}}</td>
+</tr>
+{% endfor %}
+
+</table>
+
+<script type="text/javascript" src="{{ settings.MEDIA_URL }}/scripts/TableSort.js"></script>
+
+</body>
+
+
+</html>
+

urls.py

@@ -22,9 +22,11 @@ admin.autodiscover()
 
 actualurlpatterns = patterns('',
 	
-    url(r'^testingurl/?$' , views_caves.millenialcaves, name="testing"),
-
+    
     url(r'^millenialcaves/?$',     views_caves.millenialcaves,	name="millenialcaves"),
+    url(r'^millenialpeople/?$',     views_caves.millenialpeople,	name="millenialpeople"),
+    url(r'^cave/descriptionM/([^/]+)/?$', views_caves.millenialdescription),
+    #url(r'^cave/description/([^/]+)/?$', views_caves.caveDescription),
     
     url(r'^troggle$',              views_other.frontpage,      name="frontpage"),
     url(r'^todo/$',              views_other.todo,      name="todo"),