troggle/core/models/caves.py

import os
import os
import re
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path

from django.db import models, DataError
from django.template import loader

import settings
from troggle.core.models.logbooks import QM
from troggle.core.models.survex import SurvexStation, utmToLatLng
from troggle.core.models.troggle import DataIssue, TroggleModel
from troggle.core.utils import TROG, writetrogglefile, parse_aliases

# Use the TROG global object to cache the cave lookup list. No good for multi-user.., or even multi-page. Pointless in fact.
Gcavelookup = TROG["caves"]["gcavelookup"]
Gcave_count = TROG["caves"]["gcavecount"]

Gcavelookup = None
Gcave_count = None

"""The model declarations for Areas, Caves and Entrances
"""

todo = """
- Why do we have CaveAndEntrance objects ? These do not need to be explcit for a many:many relationship these days
  
- Restore constraint:   unique_together = (("area", "kataster_number"), ("area", "unofficial_number"))
  or replace by a unique 'slug' field, better.
"""


class CaveAndEntrance(models.Model):
    """This class is ONLY used to create a FormSet for editing the cave and all its
    entrances in one form.
    CASCADE means that if the cave or the entrance is deleted, then this CaveAndEntrance 
    is deleted too
    NOT NEEDED anymore if we insist that cave:entrances have 1:n multiplicity.
    """
    cave = models.ForeignKey("Cave", on_delete=models.CASCADE)
    entrance = models.ForeignKey("Entrance", on_delete=models.CASCADE)
    entranceletter = models.CharField(max_length=20, blank=True, null=True)

    class Meta:
        unique_together = [["cave", "entrance"], ["cave", "entranceletter"]]
        ordering = ["entranceletter"]

    def __str__(self):
        return str(self.cave) + str(self.entranceletter)
        

def get_cave_leniently(caveid):
    try:
        c = getCave(caveid)
        if c:
            return c
    except:
        # print(f"get_cave_leniently FAIL {caveid}")
        try:
            c = getCave("1623-"+caveid)
            if c:
                return c 
        except:
            return None

class Cave(TroggleModel):
    # (far) too much here perhaps,
    areacode = models.CharField(max_length=4, blank=True, null=True) # could use models.IntegerChoices
    subarea = models.CharField(max_length=25, blank=True, null=True) # 9, 8c etc.
    depth = models.CharField(max_length=100, blank=True, null=True)
    description_file = models.CharField(max_length=200, blank=True, null=True)
    entrances = models.ManyToManyField("Entrance", through="CaveAndEntrance")
    equipment = models.TextField(blank=True, null=True)
    explorers = models.TextField(blank=True, null=True)
    extent = models.CharField(max_length=100, blank=True, null=True)
    filename = models.CharField(max_length=200) # if a cave is 'pending' this is not set. Otherwise it is.
    fully_explored = models.BooleanField(default=False)
    kataster_code = models.CharField(max_length=20, blank=True, null=True)
    kataster_number = models.CharField(max_length=10, blank=True, null=True)
    kataster_status = models.TextField(blank=True, null=True)
    length = models.CharField(max_length=100, blank=True, null=True)
    notes = models.TextField(blank=True, null=True)
    official_name = models.CharField(max_length=160)
    references = models.TextField(blank=True, null=True)
    survex_file = models.CharField(max_length=100, blank=True, null=True) # should be a foreign key?
    survey = models.TextField(blank=True, null=True)
    # underground_centre_line = models.TextField(blank=True, null=True)
    underground_description = models.TextField(blank=True, null=True)
    unofficial_number = models.CharField(max_length=60, blank=True, null=True)
    url = models.CharField(max_length=300, blank=True, null=True, unique = True)

    class Meta:
        # we do not enforce uniqueness at the db level as that causes confusing errors for newbie maintainers
        #    unique_together = (("area", "kataster_number"), ("area", "unofficial_number")) 
        ordering = ("kataster_code", "unofficial_number")

    def slug(self):
        return self.newslug()
        primarySlugs = self.caveslug_set.filter(primary=True)
        if primarySlugs:
            return primarySlugs[0].slug
        else:
            slugs = self.caveslug_set.filter()
            if slugs:
                return slugs[0].slug
            else:
                return str(self.id)
                
    def newslug(self):
        return f"{self.areacode}-{self.number()}"

    def ours(self):
        return bool(re.search(r"CUCC", self.explorers))
        
    def number(self):
        if self.kataster_number:
            return self.kataster_number
        else:
            return self.unofficial_number

    def get_absolute_url(self):
        # we do not use URL_ROOT any more.
        # if self.kataster_number:
            # pass
        # elif self.unofficial_number:
            # pass
        # else:
            # self.official_name.lower()
        return "/"+ self.url  # not good Django style? NEEDS actual URL

    def url_parent(self):
        if self.url:
            return self.url.rsplit("/", 1)[0]
        else:
            return "NO cave.url"

    def __str__(self, sep=": "):
        return str(self.slug())

    def get_open_QMs(self):
        """Searches for all QMs that reference this cave."""
        # qms = self.qm_set.all().order_by('expoyear', 'block__date')
        qms = QM.objects.filter(cave=self).order_by(
            "expoyear", "block__date"
        )  # a QuerySet, see https://docs.djangoproject.com/en/dev/ref/models/querysets/#order-by
        qmsopen = qms.filter(ticked=False)
        return qmsopen  # a QuerySet

    def get_ticked_QMs(self):
        """Searches for all QMs that reference this cave."""
        qms = QM.objects.filter(cave=self).order_by(
            "expoyear", "block__date"
        )  
        qmticked = qms.filter(ticked=True)
        return qmticked  # a QuerySet

    def get_QMs(self):
        qms = self.get_open_QMs() | self.get_ticked_QMs() # set union operation
        return qms  # a QuerySet
        
    def entrances(self):
        return CaveAndEntrance.objects.filter(cave=self)
        
    def no_location(self):
        no_data = True
        for e in CaveAndEntrance.objects.filter(cave=self):
            if e.entrance.best_station() and e.entrance.best_station() != "":
                #print(self, e, e.entrance.best_station())
                try:
                    x = e.entrance.best_station_object().x
                    no_data = False
                except:
                    pass
        return no_data

    def singleentrance(self):
        return len(CaveAndEntrance.objects.filter(cave=self)) == 1

    def entrancelist(self):
        rs = []
        res = ""
        for e in CaveAndEntrance.objects.filter(cave=self):
            if e.entranceletter:
                rs.append(e.entranceletter)
        rs.sort()
        prevR = ""
        n = 0
        for r in rs:
            if prevR:
                if chr(ord(prevR) + 1) == r:
                    prevR = r
                    n += 1
                else:
                    if n == 0:
                        res += ", " + prevR
                    else:
                        res += "–" + prevR
            else:
                prevR = r
                n = 0
                res += r
        if n == 0:
            if res:
                res += ", " + prevR
        else:
            res += "–" + prevR
        return res

    def file_output(self):
        """This produces the content which wll be re-saved as the cave_data html file.
        """
        if not self.filename:
            self.filename = self.slug() + ".html"
            self.save()
        
        filepath = Path(settings.CAVEDESCRIPTIONS, self.filename)

        t = loader.get_template("dataformat/cave.xml")
        now = datetime.now(timezone.utc)
        c = dict({"cave": self, "date": now})
        content = t.render(c)
        return (filepath, content, "utf8")
        
    def writeDataFile(self):
        filepath, content, coding = self.file_output()
        writetrogglefile(filepath, content)
        return

class Entrance(TroggleModel):
    MARKING_CHOICES = (
        ("P", "Paint"),
        ("P?", "Paint (?)"),
        ("T", "Tag"),
        ("T?", "Tag (?)"),
        ("R", "Needs Retag"),
        ("S", "Spit"),
        ("S?", "Spit (?)"),
        ("U", "Unmarked"),
        ("?", "Unknown"),
    )    
    FINDABLE_CHOICES = (("?", "To be confirmed ..."), ("S", "Coordinates"), ("L", "Lost"), ("R", "Refindable"))
    alt = models.TextField(blank=True, null=True)
    approach = models.TextField(blank=True, null=True)
    bearings = models.TextField(blank=True, null=True)
    entrance_description = models.TextField(blank=True, null=True)
    explorers = models.TextField(blank=True, null=True)
    filename = models.CharField(max_length=200)
    findability = models.CharField(max_length=1, choices=FINDABLE_CHOICES, blank=True, null=True, default="?")
    findability_description = models.TextField(blank=True, null=True)
    lastvisit = models.TextField(blank=True, null=True)
    lat_wgs84 = models.TextField(blank=True, null=True) # manually entered not calculated
    location_description = models.TextField(blank=True, null=True)
    long_wgs84 = models.TextField(blank=True, null=True)  # manually entered not calculated
    # map_description = models.TextField(blank=True, null=True)
    marking = models.CharField(max_length=2, choices=MARKING_CHOICES, default="?")
    marking_comment = models.TextField(blank=True, null=True)
    name = models.CharField(max_length=100, blank=True, null=True)
    other_description = models.TextField(blank=True, null=True)
    photo = models.TextField(blank=True, null=True)
    slug = models.SlugField(max_length=50, unique=True, default="default_slug_id")
    underground_description = models.TextField(blank=True, null=True)

    tag_station = models.TextField(blank=True, null=True)
    other_station = models.TextField(blank=True, null=True)

    class Meta:
        ordering = ["caveandentrance__entranceletter"]

    def __str__(self):
        return str(self.slug)

    def single(self, station):
        if not station:
            return None
        try:
            single = SurvexStation.objects.get(name = station)
            return single
        except:
            stations = SurvexStation.objects.filter(name = station)
            print(f" # EXCEPTION looking for '{station}' in all stations. (Entrance {self})")
            if len(stations) > 1:
                print(f" # MULTIPLE stations found with same name '{station}' in Entrance {self}:")
                for s in stations:
                    print(f" # {s.id=} - {s.name}  {s.latlong()}") # .id is Django internal field, not one of ours
                return stations[0]
            else:
                return None

    def singleletter(self):
        """Used in template/dataformat/cave.xml to write out a replacement cave_data file
        why is this not working?
        """
        cavelist = self.cavelist
        try:
            first = cavelist[0]
            ce = CaveAndEntrance.objects.get(entrance=self, cave=first)
        except:
            # will fail if no caves in cavelist or if the cave isnt in the db
            return "Z"
        print(f"singleletter() access for first cave in {cavelist=}")
        if ce.entranceletter == "":
            print(f"### BLANK LETTER")
            return "Y"
        else:
            letter = ce.entranceletter
            print(f"### LETTER {letter}")
        return letter

    def other_location(self):
        return self.single(self.other_station)
 
    def find_location(self):
        r = {"": "To be entered ", "?": "To be confirmed:", "S": "", "L": "Lost:", "R": "Refindable:"}[self.findability]
        if self.tag_station:
            try:
                s = SurvexStation.objects.lookup(self.tag_station)
                return r + f"{s.x:0.0f}E {s.y:0.0f}N {s.z:0.0f}Alt"
            except:
                return r + f"{self.tag_station} Tag Station not in dataset"
        if self.other_station:
            try:
                s = SurvexStation.objects.lookup(self.other_station)
                return r + f"{s.x:0.0f}E {s.y:0.0f}N {s.z:0.0f}Alt {self.other_description}"
            except:
                return r + f"{self.tag_station} Other Station not in dataset"
        if self.FINDABLE_CHOICES == "S":
            r += "ERROR, Entrance has been surveyed but has no survex point"
        if self.bearings:
            return r + self.bearings
        return r

    def best_station(self):
        if self.tag_station:
            return self.tag_station
        if self.other_station:
            return self.other_station
            
    def best_station_object(self):
        bs = self.best_station()
        return SurvexStation.objects.get(name=bs)

    def has_photo(self):
        if self.photo:
            if (
                self.photo.find("<img") > -1
                or self.photo.find("<a") > -1
                or self.photo.find("<IMG") > -1
                or self.photo.find("<A") > -1
            ):
                return "Yes"
            else:
                return "Missing"
        else:
            return "No"

    def marking_val(self):
        for m in self.MARKING_CHOICES:
            if m[0] == self.marking:
                return m[1]

    def findability_val(self):
        for f in self.FINDABLE_CHOICES:
            if f[0] == self.findability:
                return f[1]

    def tag(self):
        return self.single(self.tag_station)
    def other(self):
        return self.single(self.other_station)

    def needs_surface_work(self):
        return self.findability != "S" or not self.has_photo or self.marking != "T"

    def get_absolute_url(self):
        # This can't be right..
        res = "/".join((self.get_root().cave.get_absolute_url(), self.title))
        return self.url_parent()

    def cavelist(self):
        rs = []
        for e in CaveAndEntrance.objects.filter(entrance=self):
            if e.cave:
                rs.append(e.cave)
        return rs
        
    def firstcave(self):
        for e in CaveAndEntrance.objects.filter(entrance=self):
            if e.cave:
                return(e.cave)
        
    def get_file_path(self):
        return Path(settings.ENTRANCEDESCRIPTIONS, self.filename)

    def file_output(self):
        if not self.filename:
            self.filename = self.slug + ".html"
            self.save()
        filepath = self.get_file_path()

        t = loader.get_template("dataformat/entrance.xml")
        now = datetime.now(timezone.utc)
        c = dict({"entrance": self, "date": now})
        content = t.render(c)
        return (filepath, content, "utf8")

    def writeDataFile(self):
        filepath, content, coding = self.file_output()
        writetrogglefile(filepath, content)
        return

    def url_parent(self):
        if self.url:
            return self.url.rsplit("/", 1)[0]
        else:
            cavelist = self.cavelist()
            if len(self.cavelist()) == 1:
                return cavelist[0].url_parent()
            else:
                return ""
             
    def latlong(self):
        """Gets lat long assuming that it has to get it from the associated stations
        """           
        station = None
        if self.other_station:
            try:
                station = SurvexStation.objects.get(name = self.other_station)
            except:
                pass
        if self.tag_station:
            try:
                station = SurvexStation.objects.get(name = self.tag_station)
            except:
                pass
        if station:
            return station.latlong()

    def lat(self):
        if self.latlong():
            return self.latlong()[0]
        else:
            return None
        
    def long(self):
        if self.latlong():
            return self.latlong()[1]
        else:
            return None
    
    def best_alt(self):
        return  self.best_station_object().z
    def best_srtm_alt(self):
        return  self.best_station_object().srtm_alt

def GetCaveLookup():
    """A very relaxed way of finding probably the right cave given almost any string which might serve to identify it

    lookup function modelled on GetPersonExpeditionNameLookup
    repeated assignment each call, needs refactoring

    Used when parsing wallets contents.json file too in views/uploads.py

    Needs to be a proper function that raises an exception if there is a duplicate.
    OR we could set it to return None if there are duplicates, and require the caller to
    fall back on doing the actual database query it wants rather than using  this cache shortcut
    """
    def bad_alias(a,k):
        # this is an error
        if a.lower() in Gcavelookup:
            Gcavelookup[key] = Gcavelookup[a.lower()]
            message = f" - Warning, capitalisation error in alias list. cave for id '{a}' does not exist but {a.lower()} does."
            print(message)
            DataIssue.objects.update_or_create(parser="aliases", message=message)
        else:
            message = f" *  Coding or cave existence mistake, cave for id '{a}' does not exist. Expecting to set key alias '{k}' to it"
            DataIssue.objects.update_or_create(parser="aliases", message=message)            
        
    
    duplicates = {}

    def checkcaveid(cave, id):
        global Gcavelookup
        if id not in Gcavelookup:
            Gcavelookup[id] = cave
            Gcave_count[id] += 1
        else:
            if cave == Gcavelookup[id]:
                pass  # same id, same cave
            else:  # same id but different cave, e.g. 122 => 1623-122 and 1626-122
                # We want to keep the 1623- and get rid of the other one
                if cave.areacode == "1623":
                    Gcavelookup[id] = cave
                duplicates[id] = 1
   
    global Gcavelookup
    if Gcavelookup:
        return Gcavelookup
    Gcavelookup = {"NONEPLACEHOLDER": None}
    global Gcave_count
    Gcave_count = defaultdict(int)  # sets default value to int(0)

    for cave in Cave.objects.all(): # Note that this collects recently created Caves too
        key = cave.official_name.lower()
        if key != "" and key != "unamed" and key != "unnamed":
            if Gcave_count[key] > 0:
                # message = f" -  Warning: ignoring alias id '{id:3}'. Caves '{Gcavelookup[id]}' and '{cave}'. "
                # print(message)
                # DataIssue.objects.create(parser="aliases", message=message)
                duplicates[key] = 1
            else:
                Gcavelookup[key] = cave
                Gcave_count[key] += 1
        if cave.kataster_number:
            # NOTE this will set an alias for "145" not "1623-145"
            checkcaveid(cave, cave.kataster_number)  # we do expect 1623/55 and 1626/55 to cause clash, removed below

        # the rest of these are 'nice to have' but may validly already be set
        if cave.unofficial_number:
            unoffn = cave.unofficial_number.lower()
            checkcaveid(cave, unoffn)

        if cave.filename:
            # this is the slug - or should be 
            fn = cave.filename.replace(".html", "").lower()
            checkcaveid(cave, fn)

        if cave.slug():
            # also possibly done already. checking for weird slug values..
            try:
                slug = cave.slug().lower()
                checkcaveid(cave, slug)
            except:
                print(cave, cave.slug())

    # These might alse create more duplicate entries
    aliases = []
    # read the two files in /cave_data/
    for ca in ["cavealiasesold.txt", "cavealiases.txt"]:
        pairs, report = parse_aliases(ca)
        aliases += pairs
        
    # print(f"Loaded aliases, {len(aliases)} found\n{report}\n {aliases}")
        
    # On reset, these aliases only work if the cave already properly exists with an entry in :expoweb:/cave_data/
    # but as the aliases are recomputed repeatedly, eventually they work on PENDING caves too

    for key, alias in aliases:
        if not alias in Gcavelookup:
            bad_alias(alias, key)
        else:
            if key in Gcavelookup:
                # already set by a different method, but is it the same cave?
                if Gcavelookup[key] == Gcavelookup[alias]:
                    pass
                else:
                    # aliases wrong - these are different caves
                    message = f" - Alias list is mis-identifying different caves {key}:{Gcavelookup[key]} != {alias}:{Gcavelookup[alias]} "
                    print(message)
                    DataIssue.objects.create(parser="alias", message=message)
                    # Gcave_count[key] += 1
            Gcavelookup[key] = Gcavelookup[alias]


    addmore = {}
    for id in Gcavelookup:
        addmore[id.replace("-", "_")] = Gcavelookup[id]
        addmore[id.replace("-", "_")] = Gcavelookup[id]

        addmore[id.replace("-", "_").upper()] = Gcavelookup[id]
        addmore[id.replace("-", "_").lower()] = Gcavelookup[id]
        addmore[id.replace("_", "-").upper()] = Gcavelookup[id]
        addmore[id.replace("_", "-").lower()] = Gcavelookup[id]
    Gcavelookup = {**addmore, **Gcavelookup}

    addmore = {}

    ldup = []
    for d in duplicates:
        # if an alias resolves to 2 or more caves, remove it as an alias
        # NOTE such an  alisas is restored, assuming a 1623 area, when parsing Wallets - but only wallets.
        #print(f"{Gcavelookup[d]=} {Gcave_count[d]=}")
        if Gcavelookup[d].areacode == "1623":
            # then leave it, treat as OK
            pass
        else:
            Gcavelookup.pop(d)
            Gcave_count.pop(d) # so should not get a duplicate msg below..
            ldup.append(d)
    if ldup:
            message = f" - Ambiguous aliases being removed: {ldup}"
            print(message)
            update_dataissue("aliases ok", message)
                
    for c in Gcave_count:
        if Gcave_count[c] > 1:
            message = f" ** Duplicate cave id count={Gcave_count[c]} id:'{Gcavelookup[c]}' cave __str__:'{c}'"
            print(message)
            update_dataissue("aliases", message)
 
    return Gcavelookup

# @transaction.atomic
def update_dataissue(parsercode, message):
    try:
        DataIssue.objects.update_or_create(parser=parsercode, message=message)
    except DataError as e:
        # bollocks, swallow this.DANGEROUS. Assuming this is the 
        # (1406, "Data too long for column 'message' at row1")
        # fault in the mariaDb/Django setup.
        exept_msg = f"Is this the  (1406, Data too long for column 'message' at row1) problem?\nexception:{e}"
        raise
    except:
        # never mind, make a duplicate
        DataIssue.objects.create(parser=parsercode, message=message)