troggle/core/models.py

import string
import os
import datetime
import logging
import re
import subprocess

from urllib.request import *
from urllib.parse import *
from urllib.error import *
from decimal import Decimal, getcontext
getcontext().prec=2 #use 2 significant figures for decimal calculations

import settings

from django.forms import ModelForm
from django.db import models
from django.contrib import admin
from django.core.files.storage import FileSystemStorage
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.db.models import Min, Max
from django.conf import settings
from django.core.urlresolvers import reverse
from django.template import Context, loader

from troggle.core.models_survex import *


def get_related_by_wikilinks(wiki_text):
    found=re.findall(settings.QM_PATTERN,wiki_text)
    res=[]
    for wikilink in found:
        qmdict={'urlroot':settings.URL_ROOT,'cave':wikilink[2],'year':wikilink[1],'number':wikilink[3]}
        try:
            cave_slugs = CaveSlug.objects.filter(cave__kataster_number = qmdict['cave'])
            qm=QM.objects.get(found_by__cave_slug__in = cave_slugs,
                              found_by__date__year = qmdict['year'],
                              number = qmdict['number'])
            res.append(qm)         
        except QM.DoesNotExist:
            print(('fail on '+str(wikilink)))
    
    return res

try:
      logging.basicConfig(level=logging.DEBUG,
                           filename=settings.LOGFILE,
                           filemode='w')
except:
      subprocess.call(settings.FIX_PERMISSIONS) 
      logging.basicConfig(level=logging.DEBUG,
                           filename=settings.LOGFILE,
                           filemode='w')

#This class is for adding fields and methods which all of our models will have.
class TroggleModel(models.Model):
    new_since_parsing = models.BooleanField(default=False, editable=False)
    non_public = models.BooleanField(default=False)
    def object_name(self):
        return self._meta.object_name

    def get_admin_url(self):
        return urllib.parse.urljoin(settings.URL_ROOT, "/admin/core/" + self.object_name().lower() + "/" + str(self.pk))

    class Meta:
        abstract = True

class TroggleImageModel(models.Model):
    new_since_parsing = models.BooleanField(default=False, editable=False)
    
    def object_name(self):
        return self._meta.object_name

    def get_admin_url(self):
        return urllib.parse.urljoin(settings.URL_ROOT, "/admin/core/" + self.object_name().lower() + "/" + str(self.pk))


    class Meta:
        abstract = True

# 
# single Expedition, usually seen by year
#
class Expedition(TroggleModel):
    year        = models.CharField(max_length=20, unique=True)
    name        = models.CharField(max_length=100)
        
    def __str__(self):
        return self.year

    class Meta:
        ordering = ('-year',)
        get_latest_by = 'year'
    
    def get_absolute_url(self):
        return urllib.parse.urljoin(settings.URL_ROOT, reverse('expedition', args=[self.year]))
    
    # construction function.  should be moved out
    def get_expedition_day(self, date):
        expeditiondays = self.expeditionday_set.filter(date=date)
        if expeditiondays:
            assert len(expeditiondays) == 1
            return expeditiondays[0]
        res = ExpeditionDay(expedition=self, date=date)
        res.save()
        return res
        
    def day_min(self):
        res = self.expeditionday_set.all()
        return res and res[0] or None
    
    def day_max(self):
        res = self.expeditionday_set.all()
        return res and res[len(res) - 1] or None


class ExpeditionDay(TroggleModel):
    expedition  = models.ForeignKey("Expedition")
    date        = models.DateField()

    class Meta:
        ordering = ('date',)

    def GetPersonTrip(self, personexpedition):
        personexpeditions = self.persontrip_set.filter(expeditionday=self)
        return personexpeditions and personexpeditions[0] or None

class Person(TroggleModel):
    """single Person, can go on many years
    """
    first_name  = models.CharField(max_length=100)
    last_name   = models.CharField(max_length=100)
    fullname    = models.CharField(max_length=200)
    is_vfho     = models.BooleanField(help_text="VFHO is the Vereines für Höhlenkunde in Obersteier, a nearby Austrian caving club.", default=False)
    mug_shot    = models.CharField(max_length=100, blank=True,null=True)
    blurb = models.TextField(blank=True,null=True)
    
    #href        = models.CharField(max_length=200)
    orderref    = models.CharField(max_length=200)  # for alphabetic 
    user	= models.OneToOneField(User, null=True, blank=True)
    def get_absolute_url(self):
        return urllib.parse.urljoin(settings.URL_ROOT,reverse('person',kwargs={'first_name':self.first_name,'last_name':self.last_name}))

    class Meta:
        verbose_name_plural = "People"
        ordering = ('orderref',)  # "Wookey" makes too complex for: ('last_name', 'first_name') 
    
    def __str__(self):
        if self.last_name:
            return "%s %s" % (self.first_name, self.last_name)
        return self.first_name

    
    def notability(self):
        notability = Decimal(0)
        max_expo_val = 0

        max_expo_year = Expedition.objects.all().aggregate(Max('year'))
        max_expo_val = int(max_expo_year['year__max']) + 1

        for personexpedition in self.personexpedition_set.all():
             if not personexpedition.is_guest:
                print((personexpedition.expedition.year))
                notability += Decimal(1) / (max_expo_val - int(personexpedition.expedition.year))
        return notability

    def bisnotable(self):
        return self.notability() > Decimal(1)/Decimal(3)
    
    def surveyedleglength(self):
        return sum([personexpedition.surveyedleglength()  for personexpedition in self.personexpedition_set.all()])
    
    def first(self):
        return self.personexpedition_set.order_by('-expedition')[0]
    def last(self):
        return self.personexpedition_set.order_by('expedition')[0]
    
    #def Sethref(self):
        #if self.last_name:
            #self.href = self.first_name.lower() + "_" + self.last_name.lower()
            #self.orderref = self.last_name + " " + self.first_name
        #else:
          #  self.href = self.first_name.lower()
            #self.orderref = self.first_name
        #self.notability = 0.0  # set temporarily
        

class PersonExpedition(TroggleModel):
    """Person's attendance to one Expo
    """
    expedition  = models.ForeignKey(Expedition)
    person      = models.ForeignKey(Person)
    slugfield   = models.SlugField(max_length=50,blank=True,null=True)
    
    is_guest    = models.BooleanField(default=False)  
    COMMITTEE_CHOICES = (
        ('leader','Expo leader'),
        ('medical','Expo medical officer'),
        ('treasurer','Expo treasurer'),
        ('sponsorship','Expo sponsorship coordinator'),
        ('research','Expo research coordinator'),
        )
    expo_committee_position = models.CharField(blank=True,null=True,choices=COMMITTEE_CHOICES,max_length=200)
    nickname    = models.CharField(max_length=100,blank=True,null=True)
    
    def GetPersonroles(self):
        res = [ ]
        for personrole in self.personrole_set.order_by('survexblock'):
            if res and res[-1]['survexpath'] == personrole.survexblock.survexpath:
                res[-1]['roles'] += ", " + str(personrole.role)
            else:
                res.append({'date':personrole.survexblock.date, 'survexpath':personrole.survexblock.survexpath, 'roles':str(personrole.role)})
        return res

    class Meta:
        ordering = ('-expedition',)
        #order_with_respect_to = 'expedition'

    def __str__(self):
        return "%s: (%s)" % (self.person, self.expedition)
    
    #why is the below a function in personexpedition, rather than in person? - AC 14 Feb 09
    def name(self):
        if self.nickname:
            return "%s (%s) %s" % (self.person.first_name, self.nickname, self.person.last_name)
        if self.person.last_name:
            return "%s %s" % (self.person.first_name, self.person.last_name)
        return self.person.first_name

    def get_absolute_url(self):
        return urllib.parse.urljoin(settings.URL_ROOT, reverse('personexpedition',kwargs={'first_name':self.person.first_name,'last_name':self.person.last_name,'year':self.expedition.year}))
	
    def surveyedleglength(self):
        survexblocks = [personrole.survexblock  for personrole in self.personrole_set.all() ]
        return sum([survexblock.totalleglength  for survexblock in set(survexblocks)])
    
    # would prefer to return actual person trips so we could link to first and last ones
    def day_min(self):
        res = self.persontrip_set.aggregate(day_min=Min("expeditionday__date"))
        return res["day_min"]

    def day_max(self):
        res = self.persontrip_set.all().aggregate(day_max=Max("expeditionday__date"))
        return res["day_max"]

class LogbookEntry(TroggleModel):
    """Single parsed entry from Logbook
    """
    LOGBOOK_ENTRY_TYPES = (
        ("wiki", "Wiki style logbook"),
        ("html", "Html style logbook")
    )

    date    = models.DateField()#MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.ld()
    expeditionday = models.ForeignKey("ExpeditionDay", null=True)#MJG wants to KILL THIS (redundant information)
    expedition  = models.ForeignKey(Expedition,blank=True,null=True)  # yes this is double-
    title      = models.CharField(max_length=settings.MAX_LOGBOOK_ENTRY_TITLE_LENGTH)
    cave_slug  = models.SlugField(max_length=50)
    place      = models.CharField(max_length=100,blank=True,null=True,help_text="Only use this if you haven't chosen a cave")
    text       = models.TextField()
    slug       = models.SlugField(max_length=50)
    filename   = models.CharField(max_length=200,null=True)
    entry_type = models.CharField(default="wiki",null=True,choices=LOGBOOK_ENTRY_TYPES,max_length=50)

    class Meta:
        verbose_name_plural = "Logbook Entries"
        # several PersonTrips point in to this object
        ordering = ('-date',)
        
    def __getattribute__(self, item):
        if item == "cave":  
            #Allow a logbookentries cave to be directly accessed despite not having a proper foreignkey
            return CaveSlug.objects.get(slug = self.cave_slug).cave
        # parse error in python3.8
        # https://stackoverflow.com/questions/41343263/provide-classcell-example-for-python-3-6-metaclass
        #https://github.com/django/django/pull/7653
        #return TroggleModel.__getattribute__(item) 
        #return super(LogbookEntry, self).__getattribute__(item) # works in py3.5, fails in 3.8
        return TroggleModel.__getattribute__(self,item) # works in py 3.5 AND in 3.8

    def __init__(self, *args, **kwargs):
        if "cave" in list(kwargs.keys()):
            if kwargs["cave"] is not None:
                kwargs["cave_slug"] = CaveSlug.objects.get(cave=kwargs["cave"], primary=True).slug
            kwargs.pop("cave")
        # parse error in python3.8        
        return TroggleModel.__init__(self, *args, **kwargs) # seems OK in 3.5 & 3.8! failure later elsewhere with 3.8
        #return TroggleModel().__init__(self, *args, **kwargs) # parses OK, fails at runtime in 3.8
        #return super().__init__(self, *args, **kwargs) # fails in 3.8
        #return super().__init__(*args, **kwargs) # works in py3.5 fails in 3.8
        #return super(LogbookEntry, self).__init__(*args, **kwargs) # works in py3.5
        #return TroggleModel.__init__(*args, **kwargs) # fails in py3.5, runtime fail in 3.8

    def isLogbookEntry(self): # Function used in templates
        return True

    def get_absolute_url(self):
        return urllib.parse.urljoin(settings.URL_ROOT, reverse('logbookentry',kwargs={'date':self.date,'slug':self.slug}))

    def __str__(self):
        return "%s: (%s)" % (self.date, self.title)

    def get_next_by_id(self):
        LogbookEntry.objects.get(id=self.id+1)

    def get_previous_by_id(self):
        LogbookEntry.objects.get(id=self.id-1)

    def new_QM_number(self):
        """Returns  """
        if self.cave:
            nextQMnumber=self.cave.new_QM_number(self.date.year)
        else:
            return None
        return nextQMnumber

    def new_QM_found_link(self):
        """Produces a link to a new QM with the next number filled in and this LogbookEntry set as 'found by' """
        return settings.URL_ROOT + r'/admin/core/qm/add/?' + r'found_by=' + str(self.pk) +'&number=' + str(self.new_QM_number())

    def DayIndex(self):
        return list(self.expeditionday.logbookentry_set.all()).index(self)


#
# Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry)
#
class PersonTrip(TroggleModel):
    personexpedition = models.ForeignKey("PersonExpedition",null=True)
    
    #expeditionday    = models.ForeignKey("ExpeditionDay")#MJG wants to KILL THIS (redundant information)
    #date             = models.DateField() #MJG wants to KILL THIS (redundant information)
    time_underground = models.FloatField(help_text="In decimal hours")
    logbook_entry    = models.ForeignKey(LogbookEntry)
    is_logbook_entry_author = models.BooleanField(default=False)
    
    
    # sequencing by person (difficult to solve locally)
    #persontrip_next  = models.ForeignKey('PersonTrip', related_name='pnext', blank=True,null=True)#MJG wants to KILL THIS (and use funstion persontrip_next_auto)
    #persontrip_prev  = models.ForeignKey('PersonTrip', related_name='pprev', blank=True,null=True)#MJG wants to KILL THIS(and use funstion persontrip_prev_auto)
    
    def persontrip_next(self):
        futurePTs = PersonTrip.objects.filter(personexpedition = self.personexpedition, logbook_entry__date__gt = self.logbook_entry.date).order_by('logbook_entry__date').all()
        if len(futurePTs) > 0:
            return futurePTs[0]
        else:
            return None

    def persontrip_prev(self):
        pastPTs = PersonTrip.objects.filter(personexpedition = self.personexpedition, logbook_entry__date__lt = self.logbook_entry.date).order_by('-logbook_entry__date').all()
        if len(pastPTs) > 0:
            return pastPTs[0]
        else:
            return None

    def place(self):
        return self.logbook_entry.cave and self.logbook_entry.cave or self.logbook_entry.place

    def __str__(self):
        return "%s (%s)" % (self.personexpedition, self.logbook_entry.date)
    


##########################################
# move following classes into models_cave
##########################################

class Area(TroggleModel):
    short_name = models.CharField(max_length=100)
    name = models.CharField(max_length=200, blank=True, null=True)
    description = models.TextField(blank=True,null=True)
    parent = models.ForeignKey('Area', blank=True, null=True)
    def __str__(self):
        if self.parent:
            return str(self.parent) + " - " + str(self.short_name)
        else:
            return str(self.short_name)
    def kat_area(self):
        if self.short_name in ["1623", "1626"]:
            return self.short_name
        elif self.parent:
            return self.parent.kat_area()

class CaveAndEntrance(models.Model):
    cave = models.ForeignKey('Cave')
    entrance = models.ForeignKey('Entrance')
    entrance_letter = models.CharField(max_length=20,blank=True,null=True)
    def __str__(self):
        return str(self.cave) + str(self.entrance_letter)
        
class CaveSlug(models.Model):
    cave = models.ForeignKey('Cave')
    slug = models.SlugField(max_length=50, unique = True)
    primary = models.BooleanField(default=False)
    

class Cave(TroggleModel):
    # too much here perhaps, 
    official_name = models.CharField(max_length=160)
    area = models.ManyToManyField(Area, blank=True, null=True)
    kataster_code = models.CharField(max_length=20,blank=True,null=True)
    kataster_number = models.CharField(max_length=10,blank=True, null=True)
    unofficial_number = models.CharField(max_length=60,blank=True, null=True)
    entrances = models.ManyToManyField('Entrance', through='CaveAndEntrance')
    explorers = models.TextField(blank=True,null=True)
    underground_description = models.TextField(blank=True,null=True)
    equipment = models.TextField(blank=True,null=True)
    references = models.TextField(blank=True,null=True)
    survey = models.TextField(blank=True,null=True)
    kataster_status = models.TextField(blank=True,null=True)
    underground_centre_line = models.TextField(blank=True,null=True)
    notes = models.TextField(blank=True,null=True)
    length = models.CharField(max_length=100,blank=True,null=True)
    depth = models.CharField(max_length=100,blank=True,null=True)
    extent = models.CharField(max_length=100,blank=True,null=True)
    survex_file = models.CharField(max_length=100,blank=True,null=True)
    description_file = models.CharField(max_length=200,blank=True,null=True)
    url = models.CharField(max_length=200,blank=True,null=True)
    filename = models.CharField(max_length=200)


    #class Meta:
    #    unique_together = (("area", "kataster_number"), ("area", "unofficial_number"))
    # FIXME Kataster Areas and CUCC defined sub areas need seperating    


    #href    = models.CharField(max_length=100)
    
    class Meta:
        ordering = ('kataster_code', 'unofficial_number')  

    def hassurvey(self):
        if not self.underground_centre_line:
            return "No"
        if (self.survey.find("<img") > -1 or self.survey.find("<a") > -1 or self.survey.find("<IMG") > -1 or self.survey.find("<A") > -1):
            return "Yes"
        return "Missing"

    def hassurveydata(self):
        if not self.underground_centre_line:
            return "No"
        if self.survex_file:
            return "Yes"
        return "Missing"
         
    def slug(self):
        primarySlugs = self.caveslug_set.filter(primary = True)
        if primarySlugs:
            return primarySlugs[0].slug
        else:
            slugs = self.caveslug_set.filter()
            if slugs:
                return slugs[0].slug

    def ours(self):
        return bool(re.search(r'CUCC', self.explorers))

    def reference(self):
        if self.kataster_number:
            return "%s-%s" % (self.kat_area(), self.kataster_number)
        else:
            return "%s-%s" % (self.kat_area(), self.unofficial_number)
    
    def get_absolute_url(self):
        if self.kataster_number:
            href = self.kataster_number
        elif self.unofficial_number:
            href = self.unofficial_number
        else:
            href = self.official_name.lower()
        #return settings.URL_ROOT + '/cave/' + href + '/'
        return urllib.parse.urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,}))

    def __str__(self, sep = ": "):
        return str("slug:"+self.slug())

    def get_QMs(self):
        return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all())	

    def new_QM_number(self, year=datetime.date.today().year):
            """Given a cave and the current year, returns the next QM number."""
            try:
                res=QM.objects.filter(found_by__date__year=year, found_by__cave=self).order_by('-number')[0]
            except IndexError:
                return 1
            return res.number+1

    def kat_area(self):
        for a in self.area.all():
            if a.kat_area():
                return a.kat_area()
    
    def entrances(self):
        return CaveAndEntrance.objects.filter(cave=self)

    def singleentrance(self):
        return len(CaveAndEntrance.objects.filter(cave=self)) == 1
    
    def entrancelist(self):
        rs = []
        res = ""
        for e in CaveAndEntrance.objects.filter(cave=self):
            rs.append(e.entrance_letter)
        rs.sort()
        prevR = None
        n = 0
        for r in rs:
            if prevR:
                if chr(ord(prevR) + 1 ) == r:
                    prevR = r
                    n += 1
                else:
                    if n == 0:
                        res += ", " + prevR
                    else:
                        res += "&ndash;" + prevR
            else:
                prevR = r
                n = 0
                res += r
        if n == 0:
            res += ", " + prevR
        else:
            res += "&ndash;" + prevR
        return res
    
    def writeDataFile(self):
        try:
            f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "wb")
        except:
            subprocess.call(settings.FIX_PERMISSIONS) 
            f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "wb")
        t = loader.get_template('dataformat/cave.xml')
        c = Context({'cave': self})
        u = t.render(c)
        u8 = u.encode("utf-8")
        f.write(u8)
        f.close()
        
    def getArea(self):
        areas = self.area.all()
        lowestareas = list(areas)
        for area in areas:
            if area.parent in areas:
                try:
                    lowestareas.remove(area.parent)
                except:
                    pass
        return lowestareas[0]

def getCaveByReference(reference):
    areaname, code = reference.split("-", 1)
    #print(areaname, code)
    area = Area.objects.get(short_name = areaname)
    #print(area)
    foundCaves = list(Cave.objects.filter(area = area,  kataster_number = code).all()) + list(Cave.objects.filter(area = area,  unofficial_number = code).all())
    print((list(foundCaves)))
    if len(foundCaves) == 1:
        return foundCaves[0]
    else:
        return False

class OtherCaveName(TroggleModel):
    name = models.CharField(max_length=160)
    cave = models.ForeignKey(Cave)
    def __str__(self):
        return str(self.name)
            
class EntranceSlug(models.Model):
    entrance = models.ForeignKey('Entrance')
    slug = models.SlugField(max_length=50, unique = True)
    primary = models.BooleanField(default=False)

class Entrance(TroggleModel):
    name = models.CharField(max_length=100, blank=True,null=True)
    entrance_description = models.TextField(blank=True,null=True)
    explorers = models.TextField(blank=True,null=True)
    map_description = models.TextField(blank=True,null=True)
    location_description = models.TextField(blank=True,null=True)
    approach = models.TextField(blank=True,null=True)
    underground_description = models.TextField(blank=True,null=True)
    photo = models.TextField(blank=True,null=True)
    MARKING_CHOICES = (
        ('P', 'Paint'),
        ('P?', 'Paint (?)'),
        ('T', 'Tag'),
        ('T?', 'Tag (?)'),
        ('R', 'Needs Retag'),
        ('S', 'Spit'),
        ('S?', 'Spit (?)'),
        ('U', 'Unmarked'),
        ('?', 'Unknown'))
    marking = models.CharField(max_length=2, choices=MARKING_CHOICES)
    marking_comment = models.TextField(blank=True,null=True)
    FINDABLE_CHOICES = (
        ('?', 'To be confirmed ...'),
        ('S', 'Coordinates'),
        ('L', 'Lost'),
        ('R', 'Refindable'))
    findability = models.CharField(max_length=1, choices=FINDABLE_CHOICES, blank=True, null=True)
    findability_description = models.TextField(blank=True,null=True)
    alt = models.TextField(blank=True, null=True)
    northing = models.TextField(blank=True, null=True)
    easting = models.TextField(blank=True, null=True)
    tag_station = models.TextField(blank=True, null=True)
    exact_station = models.TextField(blank=True, null=True)
    other_station = models.TextField(blank=True, null=True)
    other_description = models.TextField(blank=True,null=True)
    bearings = models.TextField(blank=True,null=True)
    url = models.CharField(max_length=200,blank=True,null=True)
    filename = models.CharField(max_length=200)
    cached_primary_slug = models.CharField(max_length=200,blank=True,null=True)

    def __str__(self):
        return str(self.slug())

    def exact_location(self):
        return SurvexStation.objects.lookup(self.exact_station)
    def other_location(self):
        return SurvexStation.objects.lookup(self.other_station)


    def find_location(self):
        r = {'':  'To be entered ',
			 '?': 'To be confirmed:',
             'S': '',
             'L': 'Lost:',
             'R': 'Refindable:'}[self.findability]
        if self.tag_station:
            try:
                s = SurvexStation.objects.lookup(self.tag_station)
                return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z)
            except:
                return r + "%s Tag Station not in dataset" % self.tag_station
        if self.exact_station:
            try:
                s = SurvexStation.objects.lookup(self.exact_station)
                return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z)
            except:
                return r + "%s Exact Station not in dataset" % self.tag_station
        if self.other_station:
            try:
                s = SurvexStation.objects.lookup(self.other_station)
                return r + "%0.0fE %0.0fN %0.0fAlt %s" % (s.x, s.y, s.z, self.other_description)
            except:
                return r + "%s Other Station not in dataset" % self.tag_station
        if self.FINDABLE_CHOICES == "S":
            r += "ERROR, Entrance has been surveyed but has no survex point"
        if self.bearings:
            return r + self.bearings
        return r

    def best_station(self):
        if self.tag_station:
            return self.tag_station
        if self.exact_station:
            return self.exact_station
        if self.other_station:
            return self.other_station

    def has_photo(self):
        if self.photo:
            if (self.photo.find("<img") > -1 or self.photo.find("<a") > -1 or self.photo.find("<IMG") > -1 or self.photo.find("<A") > -1):
                return "Yes"
            else:
                return "Missing"
        else:
            return "No"

    def marking_val(self):
        for m in self.MARKING_CHOICES:
            if m[0] == self.marking:
                return m[1]
    def findability_val(self):
        for f in self.FINDABLE_CHOICES:
            if f[0] == self.findability:
                return f[1]
                
    def tag(self):
        return SurvexStation.objects.lookup(self.tag_station)
        
    def needs_surface_work(self):
        return self.findability != "S" or not self.has_photo or self.marking != "T"

    def get_absolute_url(self):
        
        ancestor_titles='/'.join([subcave.title for subcave in self.get_ancestors()])
        if ancestor_titles:
            res = '/'.join((self.get_root().cave.get_absolute_url(), ancestor_titles, self.title))
        
        else:
            res = '/'.join((self.get_root().cave.get_absolute_url(), self.title))
            
        return res

    def slug(self):
        if not self.cached_primary_slug:
            primarySlugs = self.entranceslug_set.filter(primary = True)
            if primarySlugs:    
                self.cached_primary_slug = primarySlugs[0].slug
                self.save()
            else:
                slugs = self.entranceslug_set.filter()
                if slugs:
                    self.cached_primary_slug = slugs[0].slug
                    self.save()
        return self.cached_primary_slug

    def writeDataFile(self):
        try:
            f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w")
        except:
            subprocess.call(settings.FIX_PERMISSIONS) 
            f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w")
        t = loader.get_template('dataformat/entrance.xml')
        c = Context({'entrance': self})
        u = t.render(c)
        u8 = u.encode("utf-8")
        f.write(u8)
        f.close()

class CaveDescription(TroggleModel):
    short_name = models.CharField(max_length=50, unique = True)
    long_name = models.CharField(max_length=200, blank=True, null=True)
    description = models.TextField(blank=True,null=True)
    linked_subcaves = models.ManyToManyField("NewSubCave", blank=True,null=True)
    linked_entrances = models.ManyToManyField("Entrance", blank=True,null=True)
    linked_qms = models.ManyToManyField("QM", blank=True,null=True)

    def __str__(self):
        if self.long_name:
            return str(self.long_name)
        else:
            return str(self.short_name)
    
    def get_absolute_url(self):
        return urllib.parse.urljoin(settings.URL_ROOT, reverse('cavedescription', args=(self.short_name,)))
    
    def save(self):
        """
        Overridden save method which stores wikilinks in text as links in database.
        """      
        TroggleModel.save()
        #super(CaveDescription, self).save() # fails in python 3.8, OK in python 3.5
        qm_list=get_related_by_wikilinks(self.description)
        for qm in qm_list:
            self.linked_qms.add(qm)
        TroggleModel.save() 
        #super(CaveDescription, self).save() # fails in python 3.8, OK in python 3.5

class NewSubCave(TroggleModel):
    name = models.CharField(max_length=200, unique = True)
    def __str__(self):
        return str(self.name)

class QM(TroggleModel):
    #based on qm.csv in trunk/expoweb/1623/204 which has the fields:
    #"Number","Grade","Area","Description","Page reference","Nearest station","Completion description","Comment"
    found_by = models.ForeignKey(LogbookEntry, related_name='QMs_found',blank=True, null=True )
    ticked_off_by = models.ForeignKey(LogbookEntry, related_name='QMs_ticked_off',null=True,blank=True)
    #cave = models.ForeignKey(Cave)
    #expedition = models.ForeignKey(Expedition)

    number = models.IntegerField(help_text="this is the sequential number in the year", )
    GRADE_CHOICES=(
    ('A', 'A: Large obvious lead'),
    ('B', 'B: Average lead'),
    ('C', 'C: Tight unpromising lead'),
    ('D', 'D: Dig'),
    ('X', 'X: Unclimbable aven')
    )
    grade = models.CharField(max_length=1, choices=GRADE_CHOICES)
    location_description = models.TextField(blank=True)
    nearest_station_description = models.CharField(max_length=400,null=True,blank=True)
    nearest_station_name = models.CharField(max_length=200,blank=True,null=True)
    nearest_station = models.ForeignKey(SurvexStation,null=True,blank=True)
    area = models.CharField(max_length=100,blank=True,null=True)
    completion_description = models.TextField(blank=True,null=True)
    comment=models.TextField(blank=True,null=True)

    def __str__(self):
        return "%s %s" % (self.code(), self.grade)

    def code(self):
        return "%s-%s-%s" % (str(self.found_by.cave)[6:], self.found_by.date.year, self.number)

    def get_absolute_url(self):
        #return settings.URL_ROOT + '/cave/' + self.found_by.cave.kataster_number + '/' + str(self.found_by.date.year) + '-' + '%02d' %self.number
        return urllib.parse.urljoin(settings.URL_ROOT, reverse('qm',kwargs={'cave_id':self.found_by.cave.kataster_number,'year':self.found_by.date.year,'qm_id':self.number,'grade':self.grade}))

    def get_next_by_id(self):
        return QM.objects.get(id=self.id+1)

    def get_previous_by_id(self):
        return QM.objects.get(id=self.id-1)

    def wiki_link(self):
        return "%s%s%s" % ('[[QM:',self.code(),']]')

#photoFileStorage = FileSystemStorage(location=settings.PHOTOS_ROOT, base_url=settings.PHOTOS_URL)
#class DPhoto(TroggleImageModel): 
    #caption = models.CharField(max_length=1000,blank=True,null=True)
    #contains_logbookentry = models.ForeignKey(LogbookEntry,blank=True,null=True)
    #contains_person = models.ManyToManyField(Person,blank=True,null=True)
    # replace link to copied file with link to original file location
    #file = models.ImageField(storage=photoFileStorage, upload_to='.',)
    #is_mugshot = models.BooleanField(default=False)
    #contains_cave = models.ForeignKey(Cave,blank=True,null=True)
    #contains_entrance = models.ForeignKey(Entrance, related_name="photo_file",blank=True,null=True)
    #nearest_survey_point = models.ForeignKey(SurveyStation,blank=True,null=True)
    #nearest_QM = models.ForeignKey(QM,blank=True,null=True)
    #lon_utm = models.FloatField(blank=True,null=True)
    #lat_utm = models.FloatField(blank=True,null=True)
    
    # class IKOptions:
        # spec_module = 'core.imagekit_specs'
        # cache_dir = 'thumbs'
        # image_field = 'file'
        
    #content_type = models.ForeignKey(ContentType)
    #object_id = models.PositiveIntegerField()
    #location = generic.GenericForeignKey('content_type', 'object_id')

    # def __str__(self):
        # return self.caption

scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL)
def get_scan_path(instance, filename):
    year=instance.survey.expedition.year
    #print("WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number, instance.survey.wallet_letter)
    number=str(instance.survey.wallet_number)
    if str(instance.survey.wallet_letter) != "None":
        number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01
    return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg')

class ScannedImage(TroggleImageModel): 
    file = models.ImageField(storage=scansFileStorage, upload_to=get_scan_path)
    scanned_by = models.ForeignKey(Person,blank=True, null=True)
    scanned_on = models.DateField(null=True)
    survey = models.ForeignKey('Survey')
    contents = models.CharField(max_length=20,choices=(('notes','notes'),('plan','plan_sketch'),('elevation','elevation_sketch')))
    number_in_wallet = models.IntegerField(null=True)
    lon_utm = models.FloatField(blank=True,null=True)
    lat_utm = models.FloatField(blank=True,null=True)

    class IKOptions:
        spec_module = 'core.imagekit_specs'
        cache_dir = 'thumbs'
        image_field = 'file'
    #content_type = models.ForeignKey(ContentType)
    #object_id = models.PositiveIntegerField()
    #location = generic.GenericForeignKey('content_type', 'object_id')

    #This is an ugly hack to deal with the #s in our survey scan paths. The correct thing is to write a custom file storage backend which calls urlencode on the name for making file.url but not file.path.
    def correctURL(self):
        return string.replace(self.file.url,r'#',r'%23')

    def __str__(self):
        return get_scan_path(self,'')

class Survey(TroggleModel):
    expedition = models.ForeignKey('Expedition') #REDUNDANT (logbook_entry)
    wallet_number = models.IntegerField(blank=True,null=True)
    wallet_letter = models.CharField(max_length=1,blank=True,null=True)
    comments = models.TextField(blank=True,null=True)
    location = models.CharField(max_length=400,blank=True,null=True) #REDUNDANT
    subcave = models.ForeignKey('NewSubCave', blank=True, null=True)
    #notes_scan = models.ForeignKey('ScannedImage',related_name='notes_scan',blank=True, null=True)  	#Replaced by contents field of ScannedImage model
    survex_block  = models.OneToOneField('SurvexBlock',blank=True, null=True)
    logbook_entry = models.ForeignKey('LogbookEntry')
    centreline_printed_on = models.DateField(blank=True, null=True)
    centreline_printed_by = models.ForeignKey('Person',related_name='centreline_printed_by',blank=True,null=True)
    #sketch_scan = models.ForeignKey(ScannedImage,blank=True, null=True) 					#Replaced by contents field of ScannedImage model
    tunnel_file = models.FileField(upload_to='surveyXMLfiles',blank=True, null=True)
    tunnel_main_sketch = models.ForeignKey('Survey',blank=True,null=True)
    integrated_into_main_sketch_on = models.DateField(blank=True,null=True)
    integrated_into_main_sketch_by = models.ForeignKey('Person' ,related_name='integrated_into_main_sketch_by', blank=True,null=True)
    rendered_image = models.ImageField(upload_to='renderedSurveys',blank=True,null=True)
    def __str__(self):
        return self.expedition.year+"#"+"%02d" % int(self.wallet_number)

    def notes(self):
        return self.scannedimage_set.filter(contents='notes')

    def plans(self):
        return self.scannedimage_set.filter(contents='plan')

    def elevations(self):
        return self.scannedimage_set.filter(contents='elevation')

class DataIssue(TroggleModel):
    date = models.DateTimeField(auto_now_add=True, blank=True)
    parser = models.CharField(max_length=50, blank=True, null=True)
    message = models.CharField(max_length=400, blank=True, null=True)

    class Meta:
        ordering = ['date']

    def __str__(self):
        return "%s - %s" % (self.parser, self.message)