import io
import re
import shutil
import piexif
from pathlib import Path
import django.forms as forms
from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
from django.http import JsonResponse
from django.shortcuts import render
from django.template import loader
from django.urls import reverse
from django.conf import settings as django_settings
from PIL import Image
import troggle.settings as settings
from troggle.core.utils import ( COOKIE_MAX_AGE,
WriteAndCommitError, get_editor,
git_string,
write_binary_file, write_and_commit, write_files,
current_expo, random_slug, ensure_dir_exists,
is_identified_user
)
from .auth import login_required_if_public
MAX_IMAGE_WIDTH = 1000
MAX_IMAGE_HEIGHT = 800
THUMBNAIL_WIDTH = 200
THUMBNAIL_HEIGHT = 200
"""This is all written by Martin Green, and completely undocumented.
It uses a lot of opinionated Django default magic, none of which I am familiar with.
Philip
The function image_selector() is only called from a template file doing a popup input form,
but it stores files directly in expoweb/i /l /t instead of in
/expoweb/1623/2018-MS-01/i , /l, /t
so ALL new caves have photos in the smae place, which makes the default upload EXTREMELY CONFUSING
"""
def get_dir(path):
"From a path sent from urls.py, determine the directory."
# todo re-write this to use modern pathlib functions
if "/" in path:
return path.rsplit("/", 1)[0]
else:
return ""
def insert_gps_data(image_path, gps_data):
"""Function to insert GPS data into the resized image
"""
exif_dict = piexif.load(image_path)
exif_dict['GPS'] = gps_data
exif_bytes = piexif.dump(exif_dict)
piexif.insert(exif_bytes, image_path)
def image_selector(request, path):
"""Returns available images
called from
templates/html_editor_scripts_css.html: $('#image_popup_content').load("{% url 'image_selector' path %}" + path, function() {
"""
directory = get_dir(path)
print(f" - image selector {directory=} {path=}")
thumbnailspath = Path(settings.EXPOWEB) / directory / "t"
thumbnails = []
if thumbnailspath.is_dir():
for f in thumbnailspath.iterdir():
if f.is_file():
if directory:
base = f"{directory}/"
else:
base = ""
thumbnail_url = reverse("expopage", args=[f"{base}t/{f.name}"])
name_base = f.name.rsplit(".", 1)[0]
page_path_base = Path(settings.EXPOWEB) / directory / "l"
if (page_path_base / (f"{name_base}.htm")).is_file():
page_url = reverse("expopage", args=[f"{base}l/{name_base}.htm"])
else:
page_url = reverse("expopage", args=[f"{base}l/{name_base}.html"])
thumbnails.append({"thumbnail_url": thumbnail_url, "page_url": page_url})
return render(request, "image_selector.html", {"thumbnails": thumbnails})
def reorient_image(img, exif_dict):
if piexif.ImageIFD.Orientation in exif_dict["0th"]:
# print(f"reorient_image(): found ImageIFD.Orientation in 0th ")
# for ifd in exif_dict:
# print(f"reorient_image(): \"{ifd}\"\n {exif_dict[ifd]} ")
orientation = exif_dict["0th"].pop(piexif.ImageIFD.Orientation)
if orientation == 2:
img = img.transpose(Image.FLIP_LEFT_RIGHT)
elif orientation == 3:
img = img.rotate(180)
elif orientation == 4:
img = img.rotate(180).transpose(Image.FLIP_LEFT_RIGHT)
elif orientation == 5:
img = img.rotate(-90, expand=True).transpose(Image.FLIP_LEFT_RIGHT)
elif orientation == 6:
img = img.rotate(-90, expand=True)
elif orientation == 7:
img = img.rotate(90, expand=True).transpose(Image.FLIP_LEFT_RIGHT)
elif orientation == 8:
img = img.rotate(90, expand=True)
return img
def dms2dd(degrees, minutes, seconds, direction):
dd = float(degrees) + float(minutes)/60 + float(seconds)/(60*60);
if direction == 'S' or direction == 'W':
dd *= -1
return dd;
def extract_gps(dict):
"""Produce a set of annotations to add to an image description
The problem is that at any time one or more of the exif data points might
be missing from a particular photo, even GPSVersionID, so we need a lot
of data existence checking or it will crash.
"""
def is_present(gpsifd):
item = getattr(piexif.GPSIFD, gpsifd)
if item in dict:
print(f" {gpsifd} = id '{item}'")
return dict[item]
return None
def extract(gpsifd):
if item:=is_present(gpsifd): # walrus
n, d = item
return n/d
return None
def rational(tup):
nom, denom = tup
return nom/denom
def rationalise(item):
df, mf, sf = item
d = rational(df)
m = rational(mf)
s = rational(sf)
return (d, m, s)
latitude = ""
longitude = ""
compass_points = ["N", "NE", "E", "SE", "S", "SW", "W", "NW", "N"]
if bearing := extract("GPSImgDirection"):
compass_lookup = round(bearing / 45)
nsew = compass_points[compass_lookup]
if item := is_present("GPSImgDirectionRef"):
if item == b"M":
ref = "Magnetic"
elif item == b"T":
ref = "True"
else:
ref =""
direction = f"Direction of view: {nsew:2} ({bearing:.0f}°{ref})"
else:
direction = ""
if version := is_present("GPSVersionID"):
print(f"GPS exif {version=}")
if alt := extract("GPSAltitude"):
altitude = f"{alt:.0f}m above sea-level"
else:
altitude = ""
if ds := is_present("GPSDateStamp"):
ds = ds.decode()
else:
ds = ""
if item := is_present("GPSTimeStamp"):
h, m, s = rationalise(item)
timestamp_utc = f"{ds} {h:02.0f}:{m:02.0f}:{s:02.0f} +00:00 UTC"
else:
timestamp_utc = f"{ds}"
print(f"attempting latitude")
if ref := is_present("GPSLatitudeRef"):
latref = ref.decode()
else:
latref = ""
if item := is_present("GPSLatitude"):
d, m, s = rationalise(item)
latitude = f"{d:02.0f}:{m:02.0f}:{s:02.0f} {latref}"
print(f"{latitude=}")
latitude = dms2dd(d, m, s, latref)
print(f"{latitude=}")
else:
print("failed to find latitude")
print(f"attempting logitude")
if ref := is_present("GPSLongitudeRef"):
lonref = ref.decode()
else:
lonref = ""
if item := is_present("GPSLongitude"):
d, m, s = rationalise(item)
longitude = f"{d:02.0f}:{m:02.0f}:{s:02.0f} {lonref}"
print(f"{longitude=}")
longitude = dms2dd(d, m, s, lonref)
print(f"{longitude=}")
else:
print("failed to find longitude")
if latitude and longitude:
# 3 digits for longitude (0-359) or +/-(0-180), 2 for latitude +/-(0-90)
# we might want to rectify longitude to be always +(0-359) not +/- 180 ?
location = f'<a href="https://www.openstreetmap.org/?mlat={latitude}&mlon={longitude}">{latitude:09.6f} {latref}, {longitude:010.6f} {lonref}</a>'
print(location)
else:
location = ""
print(direction)
print(altitude)
print(timestamp_utc)
# location = dms2dd() # to do...
return f"{direction}<br />{location}<br />{altitude}</br />{timestamp_utc}<br />"
def fix_dump_bugs(exif_dict):
"""piexif has several bugs, this gets around it.
The Exif standard leaves some instance types "undefined". Great :-(
see https://github.com/hMatoba/Piexif/issues/83
see EXIF standard https://www.cipa.jp/std/documents/e/DC-008-2012_E.pdf
"""
if 41729 in exif_dict['Exif'] and isinstance(exif_dict['Exif'][41729], int):
# SceneType = 1 for a directly photogrpahed image
cc = exif_dict['Exif'][41729]
print(f"PIEXIF BUG workaround: 41729 {cc} is {type(cc)}")
exif_dict['Exif'][41729] = str(exif_dict['Exif'][41729]).encode('utf-8')
if 37121 in exif_dict['Exif']:
cc = exif_dict['Exif'][37121]
if isinstance(cc, tuple):
print(f"PIEXIF BUG workaround: 37121 {cc} is {type(cc)}")
exif_dict['Exif'][37121] = ",".join([str(v) for v in cc]).encode("ASCII")
# if 37380 in exif_dict['Exif']:
# # exposure bias
# cc = exif_dict['Exif'][37380]
# if isinstance(cc, tuple):
# print(f"PIEXIF BUG workaround: 37380 exposure bias: {cc} is {type(cc)}")
# exif_dict['Exif'][37380] = (0, 1)
if 50728 in exif_dict['Exif']:
cc = exif_dict['Exif'][50728]
if isinstance(cc, tuple):
if cc <= 1:
rational = f"({cc * 1000:.0f}, 1000)"
else:
rational = f"(1000, {cc * 1000:.0f})"
print(f"PIEXIF BUG workaround: 50728 {cc} is {type(cc)} - using {rational}")
exif_dict['Exif'][50728] = rational
return exif_dict
@login_required_if_public
def new_image_form(request, path):
"""Manages a form to upload new images
This returns JSON, not a normal response, because it is called
by JavaScript ajax.
exif_dict = piexif.load(im.info["exif"])
exif_dict = {"0th":zeroth_ifd, "Exif":exif_ifd, "GPS":gps_ifd, ...more}
"""
THUMB_QUALITY = 70
IMAGE_QUALITY = 85
directory = get_dir(path)
# print(f"new_image_form(): {directory=} {path=}")
identified_login = is_identified_user(request.user)
editor = get_editor(request)
# print(f"{django_settings.FILE_UPLOAD_MAX_MEMORY_SIZE=}")
# FILE_UPLOAD_MAX_MEMORY_SIZE = 0 # force uploaded files to be temporary in /tmp, not in-memory
if request.method == "POST":
# print(f"new_image_form(): POST ")
form = NewWebImageForm(request.POST, request.FILES, directory=directory)
if identified_login:
# disable editing the git id string as we get it from the logged-on user data
form.fields["who_are_you"].widget.attrs["readonly"]="readonly"
print(form.fields["who_are_you"].widget.attrs)
if form.is_valid():
# print(f"new_image_form(): form is valid ")
year = form.cleaned_data["year"]
descrip = form.cleaned_data["description"]
editor = form.cleaned_data["who_are_you"]
editor = git_string(editor)
title = form.cleaned_data["header"]
referer = request.headers["Referer"] # original
host = request.headers["Host"]
print(f"{host=}")
f = request.FILES["file_"]
if not title:
title = f.name
binary_data = io.BytesIO()
for chunk in f.chunks():
binary_data.write(chunk)
i = Image.open(binary_data)
i_original = i
if "exif" in i.info:
exif_dict = piexif.load(i.info["exif"])
if "GPS" in exif_dict:
gps_data = exif_dict['GPS']
print(f"new_image_form() EXIF loaded from {f.name}\n {gps_data=}")
gps_annotations = extract_gps(gps_data)
descrip += gps_annotations
i = reorient_image(i, exif_dict)
try:
exif = piexif.dump(fix_dump_bugs(exif_dict))
except:
exif = None
# date and time from exif data
if 36867 in exif_dict['Exif'] and 36880 in exif_dict['Exif']:
descrip += f"\n\n{exif_dict['Exif'][36867].decode()} {exif_dict['Exif'][36880].decode()}"
year = exif_dict['Exif'][36867].decode()[:4]
else:
year = year # replace with year from photo exif if possible
else:
exif = None
width, height = i.size
# print(f"new_image_form(): {i.size=}")
if width > MAX_IMAGE_WIDTH or height > MAX_IMAGE_HEIGHT:
scale = max(width / MAX_IMAGE_WIDTH, height / MAX_IMAGE_HEIGHT)
# print(f"new_image_form(): rescaling {scale=}")
try:
i = i.resize((int(width / scale), int(height / scale)), Image.LANCZOS)
except Exception as e:
print(f"new_image_form(): rescaling exception: {e} ")
# print(f"new_image_form(): rescaled ")
tscale = max(width / THUMBNAIL_WIDTH, height / THUMBNAIL_HEIGHT)
t = i.resize((int(width / tscale), int(height / tscale)), Image.LANCZOS)
t = t.convert('RGB')
i = i.convert('RGB')
ib = io.BytesIO()
tb = io.BytesIO()
if "exif" in i.info:
exif_dict = piexif.load(i.info["exif"])
exif_dict['GPS'] = gps_data # saved from before
try:
exif_bytes = piexif.dump(fix_dump_bugs(exif_dict))
except Exception as e:
print(f"EXCEPTION {e}\n {exif_dict=}\n {gps_data=}")
raise
i.save(ib, format='JPEG', quality = IMAGE_QUALITY, exif=exif_bytes)
exif_dict = piexif.load(t.info["exif"])
exif_dict['GPS'] = gps_data # saved from before
exif_bytes = piexif.dump(fix_dump_bugs(exif_dict))
t.save(tb, format='JPEG', quality = THUMB_QUALITY, exif=exif_bytes)
i.save(ib, format='JPEG', quality = IMAGE_QUALITY)
t.save(tb, format='JPEG', quality = THUMB_QUALITY)
image_rel_path, thumb_rel_path, desc_rel_path = form.get_rel_paths()
# print(f"new_image_form(): \n {image_rel_path=}\n {thumb_rel_path=}\n {desc_rel_path=}")
image_page_template = loader.get_template("image_page_template.html")
image_page = image_page_template.render(
{
"header": title,
"description": descrip,
"photographer": form.cleaned_data["photographer"],
"year": year,
"filepath": f"/{image_rel_path}",
}
)
image_path, thumb_path, desc_path = form.get_full_paths()
# Create directories if required
for full_path in image_path, thumb_path, desc_path:
# print(full_path, full_path.parent)
full_path.parent.mkdir(parents=True, exist_ok=True)
try:
# change_message = form.cleaned_data["change_message"]
write_and_commit(
[
(desc_path, image_page, "utf-8"),
(image_path, ib.getbuffer(), False),
(thumb_path, tb.getbuffer(), False),
],
# f"{change_message} - online adding of an image",
f"Online adding of an image to {path}",
editor # this works, a new who_are_you typed on the Image form is used as the git comment
)
except WriteAndCommitError as e:
print(f"new_image_form(): WriteAndCommitError: {e}")
return JsonResponse({"error": e})
except Exception as e:
print(f"new_image_form(): EXCEPTION: {e}")
return JsonResponse({"error": e})
linked_image_template = loader.get_template("linked_image_template.html")
html_snippet = linked_image_template.render(
{"thumbnail_url": f"/{thumb_rel_path}", "page_url": f"/{desc_rel_path}"}, request
)
save_original_in_expofiles(f, year, form.cleaned_data["photographer"], host, image_rel_path, referer)
j_response = JsonResponse({"html": html_snippet})
j_response.set_cookie('editor_id', editor, max_age=COOKIE_MAX_AGE) # does NOT seem to work updating who_are_you cookie
return j_response
else:
# print(f"new_image_form(): not POST ")
initial={"who_are_you":editor,
"year": "", "photographer": extract_git_name(editor),
"change_message": "Uploading photo"}
form = NewWebImageForm(directory=directory, initial=initial )
if identified_login:
# disable editing the git id string as we get it from the logged-on user data
form.fields["who_are_you"].widget.attrs["readonly"]="readonly"
print(form.fields["who_are_you"].widget.attrs)
# print(f"new_image_form(): POST and not POST ")
template = loader.get_template("new_image_form.html")
htmlform = template.render({"form": form, "path": path}, request)
return JsonResponse({"form": htmlform})
def extract_git_name(git_str):
pattern = r"^([^<]+)"
match = re.match(pattern, git_str)
if match:
return match.group(1).strip()
return "Anon."
def save_original_in_expofiles(f, year, photographer, host, handbook_directory, page):
"""Moves the uploaded file from /tmp to EXPOFILES
This may be redundant, if the original was already in EXPOFILES, but this
will catch photos uploaded directly from phones which otherwise never
get recorded properly in original format.
Django does small files <2.5 MB in memory, which is a pain, as we can't simply move them.
"""
if photographer:
photographer = photographer.strip().replace(" ","")
else:
photographer = "Anonymous"
directory = settings.EXPOFILES / "photos" / year / photographer / "expoweb_originals"
filepath = (directory / f.name)
ensure_dir_exists(filepath)
if isinstance(f, InMemoryUploadedFile):
# print("In-memory file content:", f,f.content_type, f.size, f.charset, f.content_type_extra, f.read())
f.open() # rewind to beginning
content = f.read()
write_binary_file(filepath, content)
write_url_file(filepath, host, handbook_directory, page)
elif isinstance(f, TemporaryUploadedFile):
if filepath.is_file:
print(f"+++++ Out of cheese error\n Destination EXISTS {filepath}")
tail = random_slug(str(filepath), 2)
newname = f"{filepath.stem}_{tail}{filepath.suffix}"
filepath = filepath.parent / newname
print(f"+++++ The turtle moves\n Attempting to use {filepath}")
if Path(f.temporary_file_path()).is_file:
# print(f"+++++ Found {f.temporary_file_path()}")
try:
dest = shutil.move(f.temporary_file_path(), filepath)
write_url_file(filepath, host, handbook_directory, page)
except Exception as e:
print("+++++ ",e)
raise
else:
print(f"+++++ Out of cheese error\n Can't find {f.temporary_file_path()}")
else:
msg = f"Unknown uploaded file type: {f}, should be a temporary file or in-memory."
print(msg)
raise TypeError(msg)
return
def write_url_file(targetpath, host, handbook_rel_path, page):
"""
page is the referrer page, which is sort-of what we want,
but troggle URLs are such that this is not the same page we want to be visiting, e.g.
if he referrer is a "_edit" page, then we want to go to the view page, not the edit page.
ALSO ths will be different if the page is a Cave Description Page, and Entrance page pr a Handbook page.
"""
# the ".url" is there, just never visible in Windows Explorer.
# FIND AND FIX the correct host for this !
content = f"[InternetShortcut]\nURL={page}\n\n[TroggleImage]\nURL=http://{host}/{handbook_rel_path}"
content = content.replace("localhost:8000", "expo.survex.com")
print(content)
filepath = targetpath.with_suffix(".url")
write_files([(filepath, content, "utf8")])
class NewWebImageForm(forms.Form):
"""The form used by the editexpopage function"""
header = forms.CharField(
widget=forms.TextInput(
attrs={"size": "60", "placeholder": "Enter title (displayed as a header and in the tab)"}
), required=False
)
file_ = forms.FileField()
description = forms.CharField(
widget=forms.Textarea(attrs={"cols": 80, "rows": 20, "placeholder": "Describe the photo (using HTML)"}), required=False
)
photographer = forms.CharField(
widget=forms.TextInput(attrs={"size": "60", "placeholder": "Photographers name"}), required=False
)
year = forms.CharField(
widget=forms.TextInput(attrs={"size": "60", "placeholder": "Year photo was taken"}), required=False
)
# change_message = forms.CharField(
# widget=forms.Textarea(attrs={"cols": 80, "rows": 3, "placeholder": "Describe the change made (for git)"}), required=False
# )
who_are_you = forms.CharField(
widget=forms.TextInput(attrs={"style": "font-size: 90%", "size": "75",
"placeholder": "Anathema Device <anathema@potatohut.expo>",
"title":"Type in your real name, and your email between angle-brackets."
}),
# label = "Editor",
)
def __init__(self, *args, **kwargs):
self.directory = Path(kwargs.pop("directory"))
super(forms.Form, self).__init__(*args, **kwargs)
def get_rel_paths(self):
f = self.cleaned_data["file_"]
return [
self.directory / "i" / (f.name.rsplit(".", 1)[0] + ".jpg"),
self.directory / "t" / (f.name.rsplit(".", 1)[0] + ".jpg"),
self.directory / "l" / (f.name.rsplit(".", 1)[0] + ".html"),
]
def get_full_paths(self):
return [Path(settings.EXPOWEB) / x for x in self.get_rel_paths()] # this is where we would want to insert the /caveid/ ?!
def clean_file_(self):
for rel_path, full_path in zip(self.get_rel_paths(), self.get_full_paths()):
if full_path.exists():
raise forms.ValidationError(f"File already exists in {rel_path}")
return self.cleaned_data["file_"]
class HTMLarea(forms.Textarea):
"""This is called from CaveForm in core/forms.py which is called from core/views/caves.py when editing a cave description
(and similarly for Entrance Descriptions). It is also called from core/views/expo.py when editing expo (i.e. handbook) pages.
"""
template_name = "widgets/HTMLarea.html"
def __init__(self, *args, **kwargs):
self.preview = kwargs.pop("preview", False)
super(forms.Textarea, self).__init__(*args, **kwargs)
def get_context(self, name, value, attrs):
c = super(forms.Textarea, self).get_context(name, value, attrs)
c["preview"] = self.preview
return c