troggle-unchained/core/models/survex.py

import datetime
import json
import operator
import os
import re
from functools import reduce
from pathlib import Path
from urllib.parse import urljoin

from django.conf import settings
from django.db import models
from django.urls import reverse

# from troggle.core.models.troggle import DataIssue # circular import. Hmm

class SurvexDirectory(models.Model):
    path = models.CharField(max_length=200)
    cave = models.ForeignKey('Cave', blank=True, null=True,on_delete=models.SET_NULL)
    primarysurvexfile = models.ForeignKey('SurvexFile', related_name='primarysurvexfile', blank=True, null=True,on_delete=models.SET_NULL)
    # could also include files in directory but not referenced
    
    class Meta:
        ordering = ('id',)
        verbose_name_plural = "Survex directories"

    def __str__(self):
        return "[SurvexDirectory:"+str(self.path)  + " |  Primary svx:" + str(self.primarysurvexfile.path) +".svx ]" 


class SurvexFile(models.Model):
    path = models.CharField(max_length=200)
    survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True,on_delete=models.SET_NULL)
    cave = models.ForeignKey('Cave', blank=True, null=True,on_delete=models.SET_NULL)
    
    class Meta:
        ordering = ('id',)

    # Don't change from the default as that breaks troggle webpages and internal referencing!
    # def __str__(self):
        # return "[SurvexFile:"+str(self.path) + "-" + str(self.survexdirectory) + "-" + str(self.cave)+"]" 

    def exists(self):
        fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx")
        return os.path.isfile(fname)
    
    def OpenFile(self):
        fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx")
        return open(fname)
    
    def SetDirectory(self):
        dirpath = os.path.split(self.path)[0]
        # pointless search every time we import a survex file if we know there are no duplicates..
        # don't use this for initial import.
        survexdirectorylist = SurvexDirectory.objects.filter(cave=self.cave, path=dirpath)
        if survexdirectorylist:
            self.survexdirectory = survexdirectorylist[0]
        else:
            survexdirectory = SurvexDirectory(path=dirpath, cave=self.cave, primarysurvexfile=self)
            survexdirectory.save()
            self.survexdirectory = survexdirectory
        self.save()
        
    def __str__(self):
        return self.path

class SurvexStationLookUpManager(models.Manager):
    def lookup(self, name):
        blocknames, sep, stationname = name.rpartition(".")
        return self.get(block = SurvexBlock.objects.lookup(blocknames),
                        name__iexact = stationname)

class SurvexStation(models.Model):
    name        = models.CharField(max_length=100)   
    block       = models.ForeignKey('SurvexBlock', null=True,on_delete=models.SET_NULL)
#   equate      = models.ForeignKey('SurvexEquate', blank=True, null=True,on_delete=models.SET_NULL)
    objects = SurvexStationLookUpManager()
    x = models.FloatField(blank=True, null=True)
    y = models.FloatField(blank=True, null=True)
    z = models.FloatField(blank=True, null=True)
    
    def path(self):
        r = self.name
        b = self.block
        while True:
            if b.name:
                r = b.name + "." + r
            if b.parent:
                b = b.parent
            else:
                return r

    class Meta:
        ordering = ('id',)
    def __str__(self):
        return self.name and str(self.name) or 'no name'

#
# Single SurvexBlock 
# 
class SurvexBlockLookUpManager(models.Manager):
    def lookup(self, name):
        if name == "":
            blocknames = []
        else:
            blocknames = name.split(".")
        block = SurvexBlock.objects.get(parent=None, survexfile__path=settings.SURVEX_TOPNAME)
        for blockname in blocknames:
            block = SurvexBlock.objects.get(parent=block, name__iexact=blockname)
        return block

class SurvexBlock(models.Model):
    """One begin..end block within a survex file. The basic elemt of a survey trip.
    """
    objects = SurvexBlockLookUpManager()
    name       = models.CharField(max_length=100)
    title      = models.CharField(max_length=200)
    parent     = models.ForeignKey('SurvexBlock', blank=True, null=True,on_delete=models.SET_NULL)
    cave       = models.ForeignKey('Cave', blank=True, null=True,on_delete=models.SET_NULL)
    
    date       = models.DateField(blank=True, null=True)
    expeditionday = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.SET_NULL)
    expedition = models.ForeignKey('Expedition', blank=True, null=True,on_delete=models.SET_NULL)
        
    survexfile = models.ForeignKey("SurvexFile", blank=True, null=True,on_delete=models.SET_NULL)
    survexpath = models.CharField(max_length=200)   # the path for the survex stations
    
    scanswallet = models.ForeignKey("Wallet", null=True,on_delete=models.SET_NULL)  # only ONE wallet per block. Th emost recent seen overwites.. ugh.
    
    legsall   = models.IntegerField(null=True)  # summary data for this block
    legslength = models.FloatField(null=True)
    
    class Meta:
        ordering = ('id',)

    def __str__(self):
        return "[SurvexBlock:"+ str(self.name) + "-path:" + \
            str(self.survexpath) + "-cave:" + \
            str(self.cave) + "]" 
            
    def __str__(self):
        return self.name and str(self.name) or 'no name'

    def isSurvexBlock(self): # Function used in templates
        return True

    def DayIndex(self):
        return list(self.expeditionday.survexblock_set.all()).index(self)

class SurvexPersonRole(models.Model):
    survexblock         = models.ForeignKey('SurvexBlock',on_delete=models.CASCADE)
        # increasing levels of precision, Surely we only need survexblock and person now that we have no link to a logbook entry?
    personname          = models.CharField(max_length=100)
    person              = models.ForeignKey('Person', blank=True, null=True,on_delete=models.SET_NULL)
    personexpedition    = models.ForeignKey('PersonExpedition', blank=True, null=True,on_delete=models.SET_NULL)
    persontrip          = models.ForeignKey('PersonTrip', blank=True, null=True,on_delete=models.SET_NULL)   # logbook
    expeditionday       = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.SET_NULL)
    
    def __str__(self):
        return str(self.personname) + " - " + str(self.survexblock) 

class Wallet(models.Model):
    '''We do not keep the JSON values in the database, we query them afresh each time,
    but we will change this when we need to do a Django query on e.g. personame
    '''
    fpath               = models.CharField(max_length=200)
    walletname          = models.CharField(max_length=200)
    walletdate          = models.DateField(blank=True, null=True)
    walletyear          = models.DateField(blank=True, null=True)
   
    class Meta:
        ordering = ('walletname',)
    
    def get_absolute_url(self):
        return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)}))

    def get_json(self):
        """Read the JSON file for the wallet and do stuff
        """
        #jsonfile = Path(self.fpath, 'contents.json')
        
        # Get from git repo instead
        # :drawings: walletjson/2022/2022#01/contents.json
        # fpath = /mnt/d/EXPO/expofiles/surveyscans/1999/1999#02
        fp = Path(self.fpath)
        wname = fp.name
        wyear = fp.parent.name
        wurl = f"/scanupload/{self.walletname}" # .replace('#', ':')     
        
        jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
        if not Path(jsonfile).is_file():
            #print(f'{jsonfile} is not a file')
            return None
        else:
            with open(jsonfile) as json_f:
                try:
                    waldata = json.load(json_f)
                except:
                    message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
                    #print(message)
                    raise
                if waldata["date"]:
                    datestr = waldata["date"].replace('.','-')
                    try:
                        thisdate = datetime.date.fromisoformat(datestr)
                    except ValueError:
                        # probably a single digit day number. HACKUS MAXIMUS.
                        # clearly we need to fix this when we first import date strings..
                        datestr = datestr[:-1] + '0' + datestr[-1]
                        print(f' - {datestr=} ')
                        try:
                            thisdate = datetime.date.fromisoformat(datestr)
                            self.walletdate =  thisdate
                            self.save()
                            try:
                                waldata["date"] = thisdate.isoformat()
                            except:
                                message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file"
                                from troggle.core.models.troggle import \
                                    DataIssue
                                DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) 
                        except:
                            message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
                            from troggle.core.models.troggle import DataIssue
                            DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) 
        return waldata
        
    def year(self): 
        '''This gets the year syntactically without opening and reading the JSON
        '''
        if len(self.walletname) < 5:
             return None       
        if self.walletname[4] != "#":
             return None
        year = int(self.walletname[0:4])
        if year < 1975 or year > 2050:
            return None  
        else:
            self.walletyear =  datetime.date(year, 1, 1)
            self.save()
            return str(year)
        

    # Yes this is horribly, horribly inefficient, esp. for a page that have date, people and cave in it
    def date(self):
        """Reads all the JSON data just to get the JSNON date.
        """
        if self.walletdate:
            return self.walletdate
        if not self.get_json():
            return None
        jsondata = self.get_json() # use walrus operator?

        datestr = jsondata["date"]
        if not datestr:
            return None
        else:
            datestr = datestr.replace('.','-')    
            try:
                samedate = datetime.date.fromisoformat(datestr)
                self.walletdate = samedate.isoformat()
            except:
                try:
                    samedate = datetime.date.fromisoformat(datestr[:10])
                    self.walletdate = samedate.isoformat()
                except:
                    samedate = None
            self.save()
            return self.walletdate
        
    def people(self):
        if not self.get_json():
            return None
        jsondata = self.get_json()
        return jsondata["people"]
        
    def cave(self):
        if not self.get_json():
            return None
        jsondata = self.get_json()
        return jsondata["cave"]

    def name(self):
        if not self.get_json():
            return None
        jsondata = self.get_json()
        return jsondata["name"]

    def get_fnames(self):
        '''Filenames without the suffix, i.e. without the ".jpg"
        '''
        dirpath = Path(settings.SCANS_ROOT, self.fpath) # does nowt as fpath is a rooted path already
        files = []
        if not self.fpath:
            files.append(f"Incorrect path to wallet contents: '{self.fpath}'")
            return files
        if not dirpath.is_dir():
            files.append(f"Incorrect path to wallet contents: '{self.fpath}'")
            return files
        else:
            try:
                for f in dirpath.iterdir():
                    if f.is_file():
                        files.append(Path(f.name).stem)
                    else:
                        files.append(f"-{Path(f.name).stem}-")
            except FileNotFoundError:
                files.append("FileNotFoundError")
                pass
        return files
        
    def fixsurvextick(self, tick):
        blocks = SurvexBlock.objects.filter(scanswallet = self)
        result = tick
        for b in blocks:    
            if b.survexfile: # if any exist in db, no check for validity or a real file. Refactor.
                result = "seagreen" #  slightly different shade of green
        return result

    def get_ticks(self):
        """Reads all the JSON data and sets the colour of the completion tick for each condition
        """
        ticks = {}
        waldata = self.get_json()
        if not waldata:
            ticks["S"] = "black"
            ticks["C"] = "black"
            ticks["Q"] = "black"
            ticks["N"] = "black"
            ticks["P"] = "black"
            ticks["E"] = "black"
            ticks["T"] = "black"
            ticks["W"] = "black"
            return ticks
        ticks = {}
        
        # Initially, are there any required survex files present ?
        # Note that we can't set the survexblock here on the wallet as that info is only available while parsing the survex file
        survexok = "red"
        ticks["S"] = "red"
        if waldata["survex not required"]:
            survexok = "green"
            ticks["S"] = "green"
        else:
            if waldata["survex file"]:
                if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
                    waldata["survex file"] = [waldata["survex file"]]
                ngood = 0
                nbad = 0
                ticks["S"] = "purple"
                for sx in waldata["survex file"]:
                #this logic appears in several places, inc uploads.py). Refactor.
                    if sx !="":
                        if Path(sx).suffix.lower() != ".svx":
                            sx = sx + ".svx"
                        if (Path(settings.SURVEX_DATA) / sx).is_file():
                            ngood += 1
                        else:
                            nbad += 1
                if nbad == 0 and ngood >= 1:
                    ticks["S"] = "green"
                elif nbad >= 1 and ngood >= 1:
                    ticks["S"] = "orange"
                elif nbad >= 1 and ngood == 0:
                    ticks["S"] = "red"
                else:
                    ticks["S"] = "black"
        
       # Cave Description
        if waldata["description written"]: 
            ticks["C"] = "green"
        else:
            ticks["C"] = survexok
        # QMs
        if waldata["qms written"]:
            ticks["Q"] = "green"
        else:
            ticks["Q"] = survexok
        if not self.year():
            ticks["Q"] = "darkgrey"
        else:
           if int(self.year()) < 2015:
                ticks["Q"] = "lightgrey"
       
                
        # Notes, Plan, Elevation; Tunnel
        if waldata["electronic survey"]:
            ticks["N"] = "green"
            ticks["P"] = "green"
            ticks["E"] = "green"
            ticks["T"] = "green"
        else:
        
            files = self.get_fnames()
            
            # Notes required
            notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False)
            notes_scanned = reduce(operator.or_, [f.endswith("notes") for f in files], notes_scanned)
            if notes_scanned:
                ticks["N"] = "green"
            else:
                ticks["N"] = "red"

            # Plan drawing required
            plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False)
            plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned)
            plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"])
            if plan_drawing_required:
                ticks["P"] = "red"
            else:
                ticks["P"] = "green"

            # Elev drawing required
            elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False)
            elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned)
            elev_scanned = reduce(operator.or_, [f.endswith("elevation") for f in files], elev_scanned)
            elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"])
            if elev_drawing_required:
                ticks["E"] = "red"
            else:
                ticks["E"] = "green"

            # Tunnel / Therion
            if elev_drawing_required or plan_drawing_required:
                ticks["T"] = "red"
            else:
                ticks["T"] = "green"
      

        # Website
        if waldata["website updated"]:
            ticks["W"] = "green"
        else:
            ticks["W"] = "red"
  
        return ticks
   
    def __str__(self):
        return "[" + str(self.walletname) + " (Wallet)]"

class SingleScan(models.Model):
    """A single file holding an image. Could be raw notes, an elevation plot or whatever
    """
    ffile    = models.CharField(max_length=200)
    name     = models.CharField(max_length=200)
    wallet   = models.ForeignKey("Wallet", null=True,on_delete=models.SET_NULL)
    
    class Meta:
        ordering = ('name',)
    
    def get_absolute_url(self):
        return urljoin(settings.URL_ROOT, reverse('scansingle', kwargs={"path":re.sub("#", "%23", self.wallet.walletname), "file":self.name}))
 
    def __str__(self):
        return "Scan Image: " + str(self.name) + " in " + str(self.wallet)

class DrawingFile(models.Model):
    """A file holding a Therion (several types) or a Tunnel drawing
    """
    dwgpath          = models.CharField(max_length=200)
    dwgname          = models.CharField(max_length=200)
    dwgwallets       = models.ManyToManyField("Wallet") # implicitly links via folders to scans to SVX files
    scans            = models.ManyToManyField("SingleScan")  # implicitly links via scans to SVX files
    dwgcontains      = models.ManyToManyField("DrawingFile")  # case when its a frame type
    filesize         = models.IntegerField(default=0)
    npaths           = models.IntegerField(default=0)
    survexfiles      = models.ManyToManyField("SurvexFile")  # direct link to SVX files - not populated yet

    class Meta:
        ordering = ('dwgpath',)

    def __str__(self):
        return "Drawing File: " + str(self.dwgname) + " (" + str(self.filesize) + " bytes)"