From 52c1dabd0ea242b59047a596ebf7362fffac0a7f Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Fri, 2 Apr 2021 23:21:23 +0100 Subject: [PATCH] survex_file field inconsistency detection & edit --- core/models_caves.py | 3 +- core/views/caves.py | 88 +++++++++++++++++++++++-------- localsettingsWSL.py | 40 ++++++++------ templates/dataformat/cave.xml | 4 +- templates/dataformat/entrance.xml | 4 +- templates/svxcaveseveral.html | 15 ++++-- templates/svxcavesingle.html | 13 +++-- 7 files changed, 120 insertions(+), 47 deletions(-) diff --git a/core/models_caves.py b/core/models_caves.py index c5d3945..ddfb2c9 100644 --- a/core/models_caves.py +++ b/core/models_caves.py @@ -189,7 +189,8 @@ class Cave(TroggleModel): subprocess.call(settings.FIX_PERMISSIONS) f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "wb") t = loader.get_template('dataformat/cave.xml') - c = Context({'cave': self}) + #c = Context({'cave': self}) + c = dict({'cave': self}) u = t.render(c) u8 = u.encode("utf-8") f.write(u8) diff --git a/core/views/caves.py b/core/views/caves.py index 1dd6bf5..b3c3c57 100644 --- a/core/views/caves.py +++ b/core/views/caves.py @@ -1,11 +1,12 @@ import os import string -import subprocess import re import settings import urllib.parse - +import subprocess +from pathlib import Path from PIL import Image, ImageDraw, ImageFont + from django import forms from django.conf import settings from django.urls import reverse @@ -59,6 +60,23 @@ class MapLocations(object): def __str__(self): return "{} map locations".format(len(self.p)) +def getCaves(cave_id): + '''Only gets called if a call to getCave() raises a MultipleObjects exception + + TO DO: search GCavelookup first, which should raise a MultpleObjectsReturned exception if there + are duplicates''' + try: + caves = Cave.objects.filter(kataster_number=cave_id) + caveset = set(caves) + + Gcavelookup = GetCaveLookup() # dictionary makes strings to Cave objects + if cave_id in Gcavelookup: + caveset.add(Gcavelookup[cave_id]) + return list(caveset) + except: + return [] + + def getCave(cave_id): '''Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm. @@ -110,7 +128,6 @@ def getnotablecaves(): notablecaves.append(c) return notablecaves - def caveindex(request): caves = Cave.objects.all() caves1623 = list(Cave.objects.filter(area__short_name = "1623")) @@ -120,23 +137,51 @@ def caveindex(request): return render(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':getnotablecaves(), 'cavepage': True}) def cave3d(request, cave_id=''): + '''This is used to create a download url in templates/cave.html if anyone wants to download the .3d file + The caller template tries kataster first, then unofficial_number if that kataster number does not exist + but only if Cave.survex_file is non-empty + ''' try: - cave = getCave(cave_id) - except Cave.MultipleObjectsReturned: # entirely the wrong action, REPLACE with the right display - caves = Cave.objects.filter(kataster_number=cave_id) - return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves }) + cave = getCave(cave_id) + except ObjectDoesNotExist: + return None + except Cave.MultipleObjectsReturned: + # But only one might have survex data? So scan and return the first that works. + caves = getCaves(cave_id) + for c in caves: + if c.survex_file: + # exists, but may not be a valid file path to a valid .svx file in the Loser repo + return file3d(request, c, c.slug) + else: + return file3d(request, cave, cave_id) + +def file3d(request, cave, cave_id): + '''Produces a .3d file directly for download. + survex_file should be in format 'caves-1623/264/264.svx' but it might be mis-entered as simply '2012-ns-10.svx' + Here we only use the stem of the last part anyway. - survexfilename = settings.SURVEX_DATA + cave.survex_file - threedfilename = settings.THREEDCACHEDIR + '%s.3d' % cave_id - if True or os.path.getmtime(survexfilename) > os.path.getmtime(threedfilename): - subprocess.call(["cavern", "--output=%s" % threedfilename, survexfilename]) - test_file = open(threedfilename, 'rb') - response = HttpResponse(content=test_file, content_type='application/3d')#mimetype is replaced by content_type for django 1.7 - response['Content-Disposition'] = 'attachment; filename=%s.3d' % cave_id - # response['X-Sendfile'] = "%s.3d" % cave_id - # It's usually a good idea to set the 'Content-Length' header too. - # You can also set any other required headers: Cache-Control, etc. - return response + TO DO properly decide whether we want to use the stem of the .svx file or the cave slug . This assumes they are the same... + ''' + survexfilename = Path(settings.SURVEX_DATA, cave.survex_file) + threedfilename = Path(settings.THREEDCACHEDIR, cave_id +'.3d') # assumes cave_id is stem of survex_file. oops. + threedcachedir = Path(settings.THREEDCACHEDIR) + + if not threedfilename.is_file() or os.path.getmtime(survexfilename) > os.path.getmtime(threedfilename): + try: + op = subprocess.check_output([settings.CAVERN, "--log", "--output={}".format(threedcachedir), "{}".format(survexfilename)]) + except OSError as ex: + # propagate this to caller + raise OSError(op) from ex + + if threedfilename.is_file(): + response = HttpResponse(content=open(threedfilename, 'rb'), content_type='application/3d') + response['Content-Disposition'] = 'attachment; filename={}.3d'.format(cave_id) + # response['X-Sendfile'] = "%s.3d" % cave_id + # It's usually a good idea to set the 'Content-Length' header too. + # You can also set any other required headers: Cache-Control, etc. + return response + else: + return None def cavepage(request, karea, subpath): '''Displays a cave description page @@ -217,6 +262,9 @@ def caveLogbook(request, slug): @login_required_if_public def edit_cave(request, slug=None): + '''This is the form that edits all the cave data and writes out an XML file in the :expoweb: repo folder + The format for the file being saved is in templates/dataformat/cave.xml + ''' if slug is not None: cave = Cave.objects.get(caveslug__slug = slug) else: @@ -339,10 +387,6 @@ def surveyindex(request): expeditions=Expedition.objects.order_by("-year") return render(request,'survey.html',locals()) -# def cave_description(request, cavedescription_name): - # cave_description = get_object_or_404(CaveDescription, short_name = cavedescription_name) - # return render(request,'cave_description.html', locals()) - def get_entrances(request, caveslug): cave = Cave.objects.get(caveslug__slug = caveslug) return render(request,'options.html', {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]}) diff --git a/localsettingsWSL.py b/localsettingsWSL.py index 3f44d36..670533e 100644 --- a/localsettingsWSL.py +++ b/localsettingsWSL.py @@ -9,7 +9,8 @@ System for Linux (WSL), on the main server or in the potato hut, using SQLite or mariaDB. It sets the directory locations for the major parts of the system so -that e.g. expofiles can be on a different filesystem. +that e.g. expofiles can be on a different filesystem, or /javascript/ can be in +a system-wide location rather than just a local directory. This file is included at the end of the main troggle/settings.py file so that it overwrites defaults in that file. @@ -22,24 +23,26 @@ print(" * importing troggle/localsettings.py") # - have you checked that credentials.py is in .gitignore ? # - we don't want to have to change the expo system password ! #----------------------------------------------------------------- -# default values, then get overwritten by real secrets +# default values, then get overwritten by real secrets imported from credentials.py EXPOUSERPASS = "nnn:gggggg" EMAIL_HOST_PASSWORD = "insert-real-email-password-here" from credentials import EXPOUSERPASS from credentials import EMAIL_HOST_PASSWORD -SERVERPORT = '8000' EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate. #SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely +SERVERPORT = '8000' # not needed + # --------------------- MEDIA redirections BEGIN --------------------- #REPOS_ROOT_PATH = '/mnt/d/CUCC-Expo/t37/' REPOS_ROOT_PATH = Path(__file__).parent.parent -LIBDIR = REPOS_ROOT_PATH / 'lib' / 'python3.7' +LIBDIR = REPOS_ROOT_PATH / 'lib' / 'python3.7' # should be finding this automatically: python --version etc. TROGGLE_PATH = Path(__file__).parent -TEMPLATE_PATH = os.fspath(TROGGLE_PATH / 'templates') -MEDIA_ROOT = os.fspath(TROGGLE_PATH / 'media') +TEMPLATE_PATH = TROGGLE_PATH / 'templates' +MEDIA_ROOT = TROGGLE_PATH / 'media' +JSLIB_ROOT = TROGGLE_PATH / 'media' / 'jslib' # used for CaveViewer JS utility FILES = Path('/mnt/f/expofiles/') EXPOFILES = Path('/mnt/f/expofiles/') @@ -52,7 +55,7 @@ MEDIA_URL = '/site-media/' DIR_ROOT = ''#this should end in / if a value is given URL_ROOT = '/' -URL_ROOT = 'http://localhost:'+ SERVERPORT +'/' +# URL_ROOT = 'http://localhost:'+ SERVERPORT +'/' MEDIA_URL = urllib.parse.urljoin(URL_ROOT , '/site_media/') SURVEYS_URL = urllib.parse.urljoin(URL_ROOT , '/survey_scans/') @@ -61,7 +64,7 @@ SVX_URL = urllib.parse.urljoin(URL_ROOT , '/survex/') STATIC_URL = urllib.parse.urljoin(URL_ROOT , '/static/') # used by Django admin pages. Do not delete. -JSLIB_URL = urllib.parse.urljoin(URL_ROOT , '/javascript/') # always fails, try to revive it ? +JSLIB_URL = urllib.parse.urljoin(URL_ROOT , '/javascript/') # used for CaveViewer JS utility #STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py # --------------------- MEDIA redirections END --------------------- @@ -115,21 +118,22 @@ TEMPLATES = [ 'django.contrib.messages.context_processors.messages', ], 'loaders': [ - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', #For each app, inc admin, in INSTALLED_APPS, loader looks for /templates - # insert your own TEMPLATE_LOADERS here + 'django.template.loaders.filesystem.Loader', # default lcation is troggle/templates/ + 'django.template.loaders.app_directories.Loader', # needed for admin 'app' ] }, }, ] +# Passwords are loaded from credentials.py by settings.py +EXPOUSERPASS = "nnn:gggggg" # overwritten by loading from credentials.py +EMAIL_HOST_PASSWORD = "insert-real-email-password-here" # overwritten by loading from credentials.py + EXPOUSER = 'expo' -# EXPOUSERPASS = "nnn:gggggg" # loaded from credentials.py EXPOUSER_EMAIL = 'philip.sargent@gmail.com' EMAIL_HOST = "smtp-auth.mythic-beasts.com" EMAIL_HOST_USER = "django-test@klebos.net" # Philip Sargent really -# EMAIL_HOST_PASSWORD = "insert-real-email-password-here" # loaded from credentials.py EMAIL_PORT=587 EMAIL_USE_TLS = True DEFAULT_FROM_EMAIL = 'django-test@klebos.net' @@ -152,8 +156,12 @@ ENTRANCEDESCRIPTIONS = os.fspath(ENTRANCEDESCRIPTIONS) LOGFILE = os.fspath(LOGFILE) SURVEYS = os.fspath(SURVEYS) EXPOWEB = os.fspath(EXPOWEB) -THREEDCACHEDIR = os.fspath(THREEDCACHEDIR) -TUNNEL_DATA = os.fspath(TUNNEL_DATA) -SURVEX_DATA = os.fspath(SURVEX_DATA) +THREEDCACHEDIR = os.fspath(THREEDCACHEDIR) +TUNNEL_DATA = os.fspath(TUNNEL_DATA) +SURVEX_DATA = os.fspath(SURVEX_DATA) REPOS_ROOT_PATH = os.fspath(REPOS_ROOT_PATH) +TEMPLATE_PATH = os.fspath(TROGGLE_PATH) +MEDIA_ROOT = os.fspath(MEDIA_ROOT) +JSLIB_ROOT = os.fspath(JSLIB_ROOT) + print(" + finished importing troggle/localsettings.py") diff --git a/templates/dataformat/cave.xml b/templates/dataformat/cave.xml index abc28e9..a00b9ba 100644 --- a/templates/dataformat/cave.xml +++ b/templates/dataformat/cave.xml @@ -1,5 +1,7 @@ + + - + diff --git a/templates/dataformat/entrance.xml b/templates/dataformat/entrance.xml index 4e7b315..2ff9f6a 100644 --- a/templates/dataformat/entrance.xml +++ b/templates/dataformat/entrance.xml @@ -1,5 +1,7 @@ + - + + diff --git a/templates/svxcaveseveral.html b/templates/svxcaveseveral.html index e855600..2f53818 100644 --- a/templates/svxcaveseveral.html +++ b/templates/svxcaveseveral.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {% block title %}List of survex files{% endblock %} - +{% load wiki_markup %} {% block content %} @@ -14,7 +14,7 @@

{% for survexdirectory in cave.survexdirectory_set.all %} - {{survexdirectory.path}} + This cave has a linked survexdirectory: {{survexdirectory.path}} in the database. See the table below. {% empty %}

If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps because you followed a link from the master caves' survex list page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because the survex (.svx) files have been stored on the server in the @@ -28,6 +28,15 @@ this part of the survey handbook. {% endfor %}

+ +

If you can see a filename here: [ {{cave.survex_file}}  ] + which does not match any in the list below including the directories beginning with caves-162X/ + not just the filename, then (if logged on) you can + click here /cave{{cave.slug}}/edit + to go to a form to correct the online data. + Instructions for filling in this form are in this part + of the survey handbook. +

{% for survexdirectory in cave.survexdirectory_set.all %}

{{survexdirectory.path}}

@@ -86,7 +95,7 @@ {% endfor %} {% endfor %} -
+

{% endfor %} {% endfor %} {% endblock %} diff --git a/templates/svxcavesingle.html b/templates/svxcavesingle.html index 0e64bcf..deca57f 100644 --- a/templates/svxcavesingle.html +++ b/templates/svxcavesingle.html @@ -1,10 +1,10 @@ {% extends "base.html" %} {% block title %}List of survex files{% endblock %} - +{% load wiki_markup %} {% block content %} {% autoescape off %} -

Surveys for {{cave.official_name}} - kataster:{{cave}}

+

Surveys for {{cave.official_name}} - identifier:{{cave}}

{% endautoescape %} @@ -13,7 +13,7 @@ All the processing to extract the survex subdriectories and survex files is done

{% for survexdirectory in cave.survexdirectory_set.all %} - {{survexdirectory.path}} + {{survexdirectory.path}} {% empty %}

If you were expecting to see a list of survex files here and a summary table of who did what and when, perhaps because you followed a link from the master caves' survex list page which showed that such survex files clearly existed, and yet there is nothing here but a blank; then this will be because the survex (.svx) files have been stored on the server in the @@ -27,6 +27,13 @@ All the processing to extract the survex subdriectories and survex files is done this part of the survey handbook. {% endfor %}

+

If you can see a filename here: [ {{cave.survex_file}}  ] which does not match any in the list below including the directories beginning with caves-162X/ + not just the filename, then (if logged on) you can +click here /cave{{cave.slug}}/edit +to go to a form to correct the online data. + Instructions for filling in this form are in this part + of the survey handbook. +

{% for survexdirectory in cave.survexdirectory_set.all %}

{{survexdirectory.path}}