troggle/core/views/editor_helpers.py

import io
import re
import shutil
import piexif

from pathlib import Path

import django.forms as forms
from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
from django.http import JsonResponse
from django.shortcuts import render
from django.template import loader
from django.urls import reverse
from django.conf import settings as django_settings

from PIL import Image

import troggle.settings as settings
from troggle.core.utils import ( COOKIE_MAX_AGE, 
    WriteAndCommitError, get_editor, 
    git_string, 
    write_binary_file, write_and_commit, write_files,
    current_expo, random_slug, ensure_dir_exists,
    is_identified_user
    )

from .auth import login_required_if_public

MAX_IMAGE_WIDTH = 1000
MAX_IMAGE_HEIGHT = 800

THUMBNAIL_WIDTH = 200
THUMBNAIL_HEIGHT = 200

"""This is all written by Martin Green, and completely undocumented.

It uses a lot of opinionated Django default magic, none of which I am familiar with.
Philip

The function image_selector() is only called from a template file doing a popup input form,
but it stores files directly in expoweb/i /l /t instead of in
/expoweb/1623/2018-MS-01/i , /l, /t
so ALL new caves have photos in the smae place, which makes the default upload EXTREMELY CONFUSING
"""

def get_dir(path):
    "From a path sent from urls.py, determine the directory."
    # todo re-write this to use modern pathlib functions
    if "/" in path:
        return path.rsplit("/", 1)[0]
    else:
        return ""

def insert_gps_data(image_path, gps_data):
    """Function to insert GPS data into the resized image
    """
    exif_dict = piexif.load(image_path)
    exif_dict['GPS'] = gps_data
    exif_bytes = piexif.dump(exif_dict)
    piexif.insert(exif_bytes, image_path)
    
def image_selector(request, path):
    """Returns available images
    called from 
    templates/html_editor_scripts_css.html:          $('#image_popup_content').load("{% url 'image_selector' path %}" + path, function() {
    """
    directory = get_dir(path)
    print(f" - image selector {directory=} {path=}")
    thumbnailspath = Path(settings.EXPOWEB) / directory / "t"
    thumbnails = []
    if thumbnailspath.is_dir():
        for f in thumbnailspath.iterdir():
            if f.is_file():
                if directory:
                    base = f"{directory}/"
                else:
                    base = ""
                thumbnail_url = reverse("expopage", args=[f"{base}t/{f.name}"])
                name_base = f.name.rsplit(".", 1)[0]
                page_path_base = Path(settings.EXPOWEB) / directory / "l"
                if (page_path_base / (f"{name_base}.htm")).is_file():
                    page_url = reverse("expopage", args=[f"{base}l/{name_base}.htm"])
                else:
                    page_url = reverse("expopage", args=[f"{base}l/{name_base}.html"])

                thumbnails.append({"thumbnail_url": thumbnail_url, "page_url": page_url})

    return render(request, "image_selector.html", {"thumbnails": thumbnails})
    
def reorient_image(img, exif_dict):
     if piexif.ImageIFD.Orientation in exif_dict["0th"]:
            # print(f"reorient_image():  found ImageIFD.Orientation in 0th ")
            # for ifd in exif_dict:
                # print(f"reorient_image(): \"{ifd}\"\n    {exif_dict[ifd]} ")
            orientation = exif_dict["0th"].pop(piexif.ImageIFD.Orientation)
            
            if orientation == 2:
                img = img.transpose(Image.FLIP_LEFT_RIGHT)
            elif orientation == 3:
                img = img.rotate(180)
            elif orientation == 4:
                img = img.rotate(180).transpose(Image.FLIP_LEFT_RIGHT)
            elif orientation == 5:
                img = img.rotate(-90, expand=True).transpose(Image.FLIP_LEFT_RIGHT)
            elif orientation == 6:
                img = img.rotate(-90, expand=True)
            elif orientation == 7:
                img = img.rotate(90, expand=True).transpose(Image.FLIP_LEFT_RIGHT)
            elif orientation == 8:
                img = img.rotate(90, expand=True)
     return img
     
def dms2dd(degrees, minutes, seconds, direction):
    dd = float(degrees) + float(minutes)/60 + float(seconds)/(60*60);
    if direction == 'S' or direction == 'W':
        dd *= -1
    return dd;
    
def extract_gps(dict):
    """Produce a set of annotations to add to an image description
    
    The problem is that at any time one or more of the exif data points might 
    be missing from a particular photo, even GPSVersionID, so we need a lot
    of data existence checking or it will crash.
    """
    def is_present(gpsifd):
        item = getattr(piexif.GPSIFD, gpsifd)
        if item in dict:
            print(f"   {gpsifd} = id '{item}'")
            return dict[item]
        return None
        
    def extract(gpsifd):
        if item:=is_present(gpsifd): # walrus
            n, d =  item
            return n/d
        return None
        
    def rational(tup):
        nom, denom = tup
        return nom/denom
    
    def rationalise(item):
        df, mf, sf = item
        d = rational(df)
        m = rational(mf)
        s = rational(sf)
        return (d, m, s)

    compass_points = ["N", "NE", "E", "SE", "S", "SW", "W", "NW", "N"]
    if bearing := extract("GPSImgDirection"):
        compass_lookup = round(bearing / 45)
        nsew = compass_points[compass_lookup] 
        if item := is_present("GPSImgDirectionRef"): 
            if item == b"M":
                ref = "Magnetic"
            elif item == b"T":
                ref = "True"  
        else:
            ref =""         
        direction = f"Direction of view: {nsew:2} ({bearing:.0f}°{ref})"
    else:
        direction = ""
    
    if version := is_present("GPSVersionID"):
        print(f"GPS exif {version=}")
    
    if alt :=  extract("GPSAltitude"):
        altitude = f"{alt:.0f}m above sea-level"
    else:
        altitude = ""
        
    
    if ds := is_present("GPSDateStamp"):
        ds = ds.decode()
    else:
        ds = ""
    
    if  item := is_present("GPSTimeStamp"):
        h, m, s = rationalise(item)
        timestamp_utc = f"{ds} {h:02.0f}:{m:02.0f}:{s:02.0f} +00:00 UTC"
    else:
        timestamp_utc = f"{ds}"
    
    print(f"attempting latitude")
    if  ref := is_present("GPSLatitudeRef"): 
        latref = ref.decode()
    else:
        latref = ""

    if  item := is_present("GPSLatitude"): 
        d, m, s = rationalise(item)
        latitude = f"{d:02.0f}:{m:02.0f}:{s:02.0f} {latref}"
        print(f"{latitude=}")
        latitude = dms2dd(d, m, s, latref) 
        print(f"{latitude=}")
    else:
        print("failed to find latitude")

    print(f"attempting logitude")
    if  ref := is_present("GPSLongitudeRef"): 
        lonref = ref.decode()
    else:
        lonref = ""

    if  item := is_present("GPSLongitude"): 
        d, m, s = rationalise(item)
        longitude = f"{d:02.0f}:{m:02.0f}:{s:02.0f} {lonref}"
        print(f"{longitude=}")
        longitude = dms2dd(d, m, s, lonref) 
        print(f"{longitude=}")
    else:
        print("failed to find latitude")

    location = f'<a href="https://www.openstreetmap.org/?mlat={latitude}&mlon={longitude}">{latitude:09.6f} {latref}, {longitude:010.6f} {lonref}</a>'
    # 3 digits for longitude (0-359) or +/-(0-180), 2 for latitude +/-(0-90)
    # we might want to rectify longitude to be always +(0-359)?
    print(direction)
    print(altitude) 
    print(timestamp_utc)
    # location = dms2dd() # to do...
    print(location)

    return f"{direction}<br />{location}<br />{altitude}</br />{timestamp_utc}<br />"

def fix_dump_bugs(exif_dict):
    """piexif has several bugs, this gets around it.
    The Exif standard leaves some instance types "undefined". Great :-(
    
    see https://github.com/hMatoba/Piexif/issues/83
    see EXIF standard https://www.cipa.jp/std/documents/e/DC-008-2012_E.pdf
    """
    if 41729 in exif_dict['Exif'] and isinstance(exif_dict['Exif'][41729], int):
        # SceneType = 1 for a directly photogrpahed image
        cc = exif_dict['Exif'][41729]
        print(f"PIEXIF BUG workaround: 41729 {cc} is {type(cc)}") 
        exif_dict['Exif'][41729] = str(exif_dict['Exif'][41729]).encode('utf-8')    

    if 37121 in exif_dict['Exif']:
        cc = exif_dict['Exif'][37121]
        if isinstance(cc, tuple):
            print(f"PIEXIF BUG workaround: 37121 {cc} is {type(cc)}") 
            exif_dict['Exif'][37121] = ",".join([str(v) for v in cc]).encode("ASCII")    
    # if 37380 in exif_dict['Exif']:
        # # exposure bias
        # cc = exif_dict['Exif'][37380]
        # if isinstance(cc, tuple):
            # print(f"PIEXIF BUG workaround: 37380 exposure bias: {cc} is {type(cc)}") 
            # exif_dict['Exif'][37380] = (0, 1)   

    if 50728 in exif_dict['Exif']:
        cc = exif_dict['Exif'][50728]
        if isinstance(cc, tuple):
            if cc <= 1:
                rational = f"({cc * 1000:.0f}, 1000)"
            else:
                rational = f"(1000, {cc * 1000:.0f})"
            print(f"PIEXIF BUG workaround: 50728 {cc} is {type(cc)} - using {rational}") 
            exif_dict['Exif'][50728] = rational  
       
    return exif_dict
    
@login_required_if_public
def new_image_form(request, path):
    """Manages a form to upload new images
    
    This returns JSON, not a normal response, because it is called
    by JavaScript ajax.
    
    exif_dict = piexif.load(im.info["exif"])
    exif_dict = {"0th":zeroth_ifd, "Exif":exif_ifd, "GPS":gps_ifd, ...more}   
    """
    THUMB_QUALITY = 70
    IMAGE_QUALITY = 85
    directory = get_dir(path)
    # print(f"new_image_form():  {directory=}  {path=}")
    
    identified_login  = is_identified_user(request.user)
    editor = get_editor(request)   
    # print(f"{django_settings.FILE_UPLOAD_MAX_MEMORY_SIZE=}")
    # FILE_UPLOAD_MAX_MEMORY_SIZE = 0 # force uploaded files to be temporary in /tmp, not in-memory
    if request.method == "POST":
        # print(f"new_image_form():  POST ")
        form = NewWebImageForm(request.POST, request.FILES, directory=directory)    
        if identified_login:
            # disable editing the git id string as we get it from the logged-on user data
            form.fields["who_are_you"].widget.attrs["readonly"]="readonly"
            print(form.fields["who_are_you"].widget.attrs)

        if form.is_valid():
            # print(f"new_image_form():  form is valid ")
            year = form.cleaned_data["year"]
            descrip = form.cleaned_data["description"]
            editor = form.cleaned_data["who_are_you"]
            editor = git_string(editor)            
            title = form.cleaned_data["header"]
            referer = request.headers["Referer"].replace("_edit","") # original page being edited
            host = request.headers["Host"] 
            f = request.FILES["file_"]
            if not title:
                title = f.name
            binary_data = io.BytesIO()
            for chunk in f.chunks():
                binary_data.write(chunk)
            i = Image.open(binary_data)
            i_original = i
            if "exif" in i.info:
                exif_dict = piexif.load(i.info["exif"])
                if "GPS" in exif_dict:
                    gps_data = exif_dict['GPS']
                    print(f"new_image_form() EXIF loaded from {f.name}\n    {gps_data=}")
                    gps_annotations = extract_gps(gps_data) 
                    descrip += gps_annotations
                i = reorient_image(i, exif_dict)
                try:
                    exif = piexif.dump(fix_dump_bugs(exif_dict))
                except:
                    exif = None
                # date and time from exif data
                if 36867 in exif_dict['Exif'] and 36880 in exif_dict['Exif']:
                    descrip += f"\n\n{exif_dict['Exif'][36867].decode()} {exif_dict['Exif'][36880].decode()}"
                    if not year:
                        year = exif_dict['Exif'][36867].decode()[:4]
            else:
                year = current_expo() # replace with year from photo exif if possible
                exif = None          
            
            width, height = i.size
            # print(f"new_image_form():  {i.size=}")
            if width > MAX_IMAGE_WIDTH or height > MAX_IMAGE_HEIGHT:
                scale = max(width / MAX_IMAGE_WIDTH, height / MAX_IMAGE_HEIGHT)
                # print(f"new_image_form():  rescaling {scale=}")
                try:
                    i = i.resize((int(width / scale), int(height / scale)), Image.LANCZOS)
                except Exception as e:
                    print(f"new_image_form():  rescaling exception: {e} ")
                    
                # print(f"new_image_form():  rescaled ")
            tscale = max(width / THUMBNAIL_WIDTH, height / THUMBNAIL_HEIGHT)
            t = i.resize((int(width / tscale), int(height / tscale)), Image.LANCZOS)
            t = t.convert('RGB')
            i = i.convert('RGB')
            ib = io.BytesIO()        
            tb = io.BytesIO()
            if "exif" in i.info:
                exif_dict = piexif.load(i.info["exif"])
                exif_dict['GPS'] = gps_data # saved from before
                try:
                    exif_bytes = piexif.dump(fix_dump_bugs(exif_dict))
                except Exception as e:
                    print(f"EXCEPTION {e}\n     {exif_dict=}\n     {gps_data=}")
                    raise
                i.save(ib, format='JPEG', quality = IMAGE_QUALITY, exif=exif_bytes) 
                
                exif_dict = piexif.load(t.info["exif"])
                exif_dict['GPS'] = gps_data # saved from before
                exif_bytes = piexif.dump(fix_dump_bugs(exif_dict))          
                t.save(tb, format='JPEG', quality = THUMB_QUALITY, exif=exif_bytes) 
                   
            i.save(ib, format='JPEG', quality = IMAGE_QUALITY) 
            t.save(tb, format='JPEG', quality = THUMB_QUALITY) 
                        
            image_rel_path, thumb_rel_path, desc_rel_path = form.get_rel_paths()
            # print(f"new_image_form(): \n {image_rel_path=}\n {thumb_rel_path=}\n {desc_rel_path=}")
            image_page_template = loader.get_template("image_page_template.html") 
            image_page = image_page_template.render(
                {
                    "header": title,
                    "description": descrip,
                    "photographer": form.cleaned_data["photographer"],
                    "year": year,
                    "filepath": f"/{image_rel_path}",
                }
            )
            image_path, thumb_path, desc_path = form.get_full_paths()
            # Create directories if required
            for full_path in image_path, thumb_path, desc_path:
                # print(full_path, full_path.parent)
                full_path.parent.mkdir(parents=True, exist_ok=True)
            try:
                # change_message = form.cleaned_data["change_message"]
                write_and_commit(
                    [
                        (desc_path, image_page, "utf-8"),
                        (image_path, ib.getbuffer(), False),
                        (thumb_path, tb.getbuffer(), False),
                    ],
                    # f"{change_message} - online adding of an image",
                    f"Online adding of an image to {path}",
                    editor # this works, a new who_are_you typed on the Image form is used as the git comment 
                )
            except WriteAndCommitError as e:
                print(f"new_image_form(): WriteAndCommitError: {e}")
                return JsonResponse({"error": e})
            except Exception as e:
                print(f"new_image_form(): EXCEPTION: {e}")
                return JsonResponse({"error": e})
                
            linked_image_template = loader.get_template("linked_image_template.html")
            html_snippet = linked_image_template.render(
                {"thumbnail_url": f"/{thumb_rel_path}", "page_url": f"/{desc_rel_path}"}, request
            )
            save_original_in_expofiles(f, year, form.cleaned_data["photographer"], host, image_rel_path, referer)
            j_response = JsonResponse({"html": html_snippet})
            j_response.set_cookie('editor_id', editor, max_age=COOKIE_MAX_AGE) # does NOT seem to work updating who_are_you cookie
            return j_response
    else:
        # print(f"new_image_form():  not POST ")
        initial={"who_are_you":editor, 
            "year": "", "photographer": extract_git_name(editor), 
            "change_message": "Uploading photo"}
        form = NewWebImageForm(directory=directory, initial=initial )
        if identified_login:
            # disable editing the git id string as we get it from the logged-on user data
            form.fields["who_are_you"].widget.attrs["readonly"]="readonly"
            print(form.fields["who_are_you"].widget.attrs)
            
    # print(f"new_image_form():  POST and not POST ")
    template = loader.get_template("new_image_form.html")
    htmlform = template.render({"form": form, "path": path}, request)
    return JsonResponse({"form": htmlform})

def extract_git_name(git_str):
    pattern = r"^([^<]+)"
    match = re.match(pattern, git_str)
    if match:
        return match.group(1).strip()
    return "Anon."    

def save_original_in_expofiles(f, year, photographer, host, handbook_directory, page):
    """Moves the uploaded file from /tmp to EXPOFILES
    
    This may be redundant, if the original was already in EXPOFILES, but this
    will catch photos uploaded directly from phones which otherwise never
    get recorded properly in original format.
    
    Django does small files <2.5 MB in memory, which is a pain.
    
    to do: also store a *.url file with the image file saying where it is used in the handbook.
    """
    if photographer:
        photographer = photographer.strip().replace(" ","")
    else:
        photographer = "Anonymous"
    directory = settings.EXPOFILES / "photos" / year / photographer / "expoweb_originals" 
    filepath = (directory / f.name)
    ensure_dir_exists(filepath)
    
    if isinstance(f, InMemoryUploadedFile):
        # print("In-memory file content:", f,f.content_type, f.size, f.charset, f.content_type_extra, f.read())
        f.open() # rewind to beginning
        content = f.read()
        write_binary_file(filepath, content)
        write_url_file(filepath, f.name, handbook_directory, page)
    elif isinstance(f, TemporaryUploadedFile):
        if filepath.is_file:
            print(f"+++++ Out of cheese error\n     Destination EXISTS {filepath}")
            tail = random_slug(str(filepath), 2)
            newname = f"{filepath.stem}_{tail}{filepath.suffix}"
            filepath = filepath.parent / newname
            print(f"+++++ The turtle moves\n     Attempting to use  {filepath}")
        if Path(f.temporary_file_path()).is_file:
            # print(f"+++++ Found {f.temporary_file_path()}")
            try:
                dest = shutil.move(f.temporary_file_path(), filepath)     
                write_url_file(filepath, host, handbook_directory, page)
            except Exception as e:
                print("+++++ ",e)
                raise
        else:
            print(f"+++++ Out of cheese error\n     Can't find {f.temporary_file_path()}")
    else:
        msg = f"Unknown uploaded file type: {f}, should be a temporary file or in-memory."
        print(msg)
        raise TypeError(msg)
    return

def write_url_file(targetpath, host, handbook_rel_path, page):      
    # the  ".url" is there, just never visible in Windows Explorer.
    
    
    # FIND AND FIX the correct host for this !
    content = f"[InternetShortcut]\nURL={page}\n\n[TroggleImage]\nURL=http://{host}/{handbook_rel_path}"
    print(content)
    filepath = targetpath.with_suffix(".url")
    write_files([(filepath, content, "utf8")]) 
    
class NewWebImageForm(forms.Form):
    """The form used by the editexpopage function"""

    header = forms.CharField(
        widget=forms.TextInput(
            attrs={"size": "60", "placeholder": "Enter title (displayed as a header and in the tab)"}
        ), required=False
    )
    file_ = forms.FileField()
    description = forms.CharField(
        widget=forms.Textarea(attrs={"cols": 80, "rows": 20, "placeholder": "Describe the photo (using HTML)"}), required=False
    )
    photographer = forms.CharField(
        widget=forms.TextInput(attrs={"size": "60", "placeholder": "Photographers name"}), required=False
    )
    year = forms.CharField(
        widget=forms.TextInput(attrs={"size": "60", "placeholder": "Year photo was taken"}), required=False
    )
    # change_message = forms.CharField(
        # widget=forms.Textarea(attrs={"cols": 80, "rows": 3, "placeholder": "Describe the change made (for git)"}), required=False
    # )
    who_are_you = forms.CharField(
        widget=forms.TextInput(attrs={"style": "font-size: 90%", "size": "75",
            "placeholder": "Anathema Device <anathema@potatohut.expo>",
            "title":"Type in your real name, and your email between angle-brackets."
            }), 
        # label = "Editor",
        )

 
    
    def __init__(self, *args, **kwargs):
        self.directory = Path(kwargs.pop("directory"))
        super(forms.Form, self).__init__(*args, **kwargs)

    def get_rel_paths(self):
        f = self.cleaned_data["file_"]
        return [
            self.directory / "i" / (f.name.rsplit(".", 1)[0] + ".jpg"),
            self.directory / "t" / (f.name.rsplit(".", 1)[0] + ".jpg"),
            self.directory / "l" / (f.name.rsplit(".", 1)[0] + ".html"),
        ]

    def get_full_paths(self):
        return [Path(settings.EXPOWEB) / x for x in self.get_rel_paths()] # this is where we would want to insert the /caveid/ ?!

    def clean_file_(self):
        for rel_path, full_path in zip(self.get_rel_paths(), self.get_full_paths()):
            if full_path.exists():
                raise forms.ValidationError(f"File already exists in {rel_path}")
        return self.cleaned_data["file_"]


class HTMLarea(forms.Textarea):
    """This is called from CaveForm in core/forms.py which is called from core/views/caves.py when editing a cave description
    (and similarly for Entrance Descriptions). It is also called from core/views/expo.py when editing expo (i.e. handbook) pages.
    """
    template_name = "widgets/HTMLarea.html"

    def __init__(self, *args, **kwargs):
        self.preview = kwargs.pop("preview", False)
        super(forms.Textarea, self).__init__(*args, **kwargs)

    def get_context(self, name, value, attrs):
        c = super(forms.Textarea, self).get_context(name, value, attrs)
        c["preview"] = self.preview
        return c