From b6500955880c983235a98a2b96a75fe710fba0ab Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Fri, 14 Jul 2023 11:13:06 +0200 Subject: [PATCH 01/17] calendar S T colour >10 per date --- core/models/logbooks.py | 6 +++--- core/models/survex.py | 14 ++++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/core/models/logbooks.py b/core/models/logbooks.py index b0bde61..34032fd 100644 --- a/core/models/logbooks.py +++ b/core/models/logbooks.py @@ -85,12 +85,12 @@ class LogbookEntry(TroggleModel): if self in todays: index = todays.index(self) else: - print(f"DayIndex: Synchronization error. Restart server. {self}") + print(f"DayIndex: Synchronization error in logbook entries. Restart server or do full reset. {self}") index = 0 if index not in range(0, mx): - print(f"DayIndex: More than {mx-1} LogbookEntry items on one day '{index}' {self}") - index = 0 + print(f"DayIndex: More than {mx-1} LogbookEntry items on one day '{index}' {self}, restarting colour sequence.") + index = index % mx return index diff --git a/core/models/survex.py b/core/models/survex.py index 7b69b64..37e8eb5 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -228,15 +228,17 @@ class SurvexBlock(models.Model): def DayIndex(self): """This is used to set different colours for the different trips on the calendar view of the expedition""" + # print(f"SurvexBlock DayIndex {self.name} '{self.date}' {len(list(SurvexBlock.objects.filter(date=self.date)))} on this date") mx = 10 - try: - index = list(SurvexBlock.objects.filter(date=self.date)).index(self) - except: - print(f"DayIndex: BAD BAD BAD SurvexBlock items on one day '{index}' {self}") + todays = list(SurvexBlock.objects.filter(date=self.date)) + if self in todays: + index = todays.index(self) + else: + print(f"DayIndex: Synchronization error in survex blocks. Restart server or do full reset. {self}") index = 0 if index not in range(0, mx): - print(f"DayIndex: More than {mx-1} SurvexBlock items on one day '{index}' {self}") - index = 0 + print(f"DayIndex: More than {mx-1} SurvexBlock items on one day '{index}' {self}, restarting colour sequence.") + index = index % mx # return list(self.survexblock_set.all()).index(self) return index From 387bdf4f91c5ee0cd3d0c4b49a4addda4f7517f5 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Fri, 14 Jul 2023 11:20:57 +0200 Subject: [PATCH 02/17] shorter msg --- parsers/caves.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsers/caves.py b/parsers/caves.py index b376801..668ac40 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -651,7 +651,7 @@ def read_cave(filename, cave=None): print(message) if description_file[0]: # if not an empty string - message = f' - {slug:12} Note (not an error): complex description filename "{description_file[0]}" inside "{CAVEDESCRIPTIONS}/{filename}"' + message = f' - {slug:12} Note (not an error): complex description filename "{description_file[0]}" inside "cave_data/{filename}"' DataIssue.objects.create(parser="caves ok", message=message, url=f"/{slug}_cave_edit/") print(message) From 4e4824546ac32f47670bf8050481706c486b65e9 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Fri, 14 Jul 2023 11:21:49 +0200 Subject: [PATCH 03/17] stop crash on live system --- core/models/wallets.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/models/wallets.py b/core/models/wallets.py index 6d217c6..9da5695 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -53,6 +53,7 @@ class Wallet(models.Model): jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json" if not Path(jsonfile).is_file(): print(f'{jsonfile} is not a file {wyear=} {wname=} ') + # Should this be a DataIssue? return None else: with open(jsonfile) as json_f: @@ -60,8 +61,9 @@ class Wallet(models.Model): waldata = json.load(json_f) except: message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file" - # print(message) - raise + print(message) + # Should this be a DataIssue? + return None if waldata["date"]: datestr = waldata["date"].replace(".", "-") try: From 26e96839e4ad707904b25093e97f8085d3cb7de9 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Fri, 14 Jul 2023 11:22:48 +0200 Subject: [PATCH 04/17] stop crash onloading wallet with blank caveid --- core/views/wallets_edit.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py index 5bf0032..2387123 100644 --- a/core/views/wallets_edit.py +++ b/core/views/wallets_edit.py @@ -237,8 +237,11 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): i = i.replace("/", "-") caveobject = getCave(i) # only the last one gets recorded.. ouch. else: - caveid = caveid - caveobject = getCave(caveid) + caveid = caveid # ?urk? why? + try: + caveobject = getCave(caveid) # may fail if garbage value ,e.g. space, in wallet data + except MultipleObjectsReturned: + caveobject = None print(f'getCave for id "{waldata["cave"]}" {caveobject}') # if not caveobject.url == waldata["description url"]: # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.') From dc067444447c5382c0ff6b925af9843761d2cff2 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Fri, 14 Jul 2023 11:26:47 +0200 Subject: [PATCH 05/17] fix exception? --- core/views/wallets_edit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py index 2387123..7cf2251 100644 --- a/core/views/wallets_edit.py +++ b/core/views/wallets_edit.py @@ -240,7 +240,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): caveid = caveid # ?urk? why? try: caveobject = getCave(caveid) # may fail if garbage value ,e.g. space, in wallet data - except MultipleObjectsReturned: + except: caveobject = None print(f'getCave for id "{waldata["cave"]}" {caveobject}') # if not caveobject.url == waldata["description url"]: From 6e452b2ee9d87faeba5241f6b62c2460f3b24629 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Thu, 25 May 2023 12:53:41 +0300 Subject: [PATCH 06/17] current settings on server - sanitized --- .gitignore | 2 + .../localsettings2023-04-05-cleansed.py | 160 ++++++++++++++++++ ...=> localsettingsserver2023-01-cleansed.py} | 2 +- .../{settings.py => settings2023-02-10.py} | 0 _deploy/debian/settings2023-04-23.py | 147 ++++++++++++++++ 5 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 _deploy/debian/localsettings2023-04-05-cleansed.py rename _deploy/debian/{localsettingsserver.py => localsettingsserver2023-01-cleansed.py} (98%) rename _deploy/debian/{settings.py => settings2023-02-10.py} (100%) create mode 100644 _deploy/debian/settings2023-04-23.py diff --git a/.gitignore b/.gitignore index 51eb283..e629ee9 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,8 @@ troggle.sqlite - Shortcut.lnk _deploy/debian/localsettings-jan.py _deploy/debian/localsettings-nw.py py310d32 +_deploy/debian/localsettingsserver2023-01-secret.py +_deploy/debian/localsettings2023-04-05-secret.py pydebianbullseye javascript diff --git a/_deploy/debian/localsettings2023-04-05-cleansed.py b/_deploy/debian/localsettings2023-04-05-cleansed.py new file mode 100644 index 0000000..2544dff --- /dev/null +++ b/_deploy/debian/localsettings2023-04-05-cleansed.py @@ -0,0 +1,160 @@ +import os +import sys +import urllib.parse +from pathlib import Path + +"""Settings for a troggle installation which may vary among different +installations: for development or deployment, in a docker image or +python virtual environment (venv), on ubuntu, debian or in Windows +System for Linux (WSL), on the main server or in the potato hut, +using SQLite or mariaDB. + +It sets the directory locations for the major parts of the system so +that e.g. expofiles can be on a different filesystem. + +This file is included at the end of the main troggle/settings.py file so that +it overwrites defaults in that file. + +NOTE this file is vastly out of sync with troggle/_deploy/wsl/localsettings.py +which is the most recent version used in active maintenance. There should be +essential differences, but there and many, many non-essential differences which +should be eliminated for clarity and to use modern idioms. 8 March 2023. +""" + +print(" * importing troggle/localsettings.py") + +# DO NOT check this file into the git repo - it contains real passwords. + +EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever +#SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + 'NAME' : 'troggle', # Or path to database file if using sqlite3. + 'USER' : 'expo', # Not used with sqlite3. + 'PASSWORD' : '123456789012345', # Not used with sqlite3. Not a real password. + 'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. + 'PORT' : '', # Set to empty string for default. Not used with sqlite3. + } +} + + +EXPOUSER = 'expo' +EXPOUSERPASS = 'Not a real password' +EXPOADMINUSER = 'expoadmin' +EXPOADMINUSERPASS = 'Not a real password' +EXPOUSER_EMAIL = 'wookey@wookware.org' +EXPOADMINUSER_EMAIL = 'wookey@wookware.org' + +REPOS_ROOT_PATH = '/home/expo/' +sys.path.append(REPOS_ROOT_PATH) +sys.path.append(REPOS_ROOT_PATH + 'troggle') +# Define the path to the django app (troggle in this case) +PYTHON_PATH = REPOS_ROOT_PATH + 'troggle/' + + +PHOTOS_YEAR = "2023" +# add in 358 when they don't make it crash horribly +NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"] + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + PYTHON_PATH + "templates" + ], + 'OPTIONS': { + 'debug': 'DEBUG', + 'context_processors': [ + # django.template.context_processors.csrf, # is always enabled and cannot be removed, sets csrf_token + 'django.contrib.auth.context_processors.auth', # knowledge of logged-on user & permissions + 'core.context.troggle_context', # in core/troggle.py + 'django.template.context_processors.debug', + #'django.template.context_processors.request', # copy of current request, added in trying to make csrf work + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', # includes a variable MEDIA_URL + 'django.template.context_processors.static', # includes a variable STATIC_URL + 'django.template.context_processors.tz', + 'django.contrib.messages.context_processors.messages', + ], + 'loaders': [ + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', #For each app, inc admin, in INSTALLED_APPS, loader looks for /templates + # insert your own TEMPLATE_LOADERS here + ] + }, + }, +] + +PUBLIC_SITE = True + +# This should be False for normal running +DEBUG = False +CACHEDPAGES = True # experimental page cache for a handful of page types + + +# executables: +CAVERN = 'cavern' # for parsing .svx files and producing .3d files +SURVEXPORT = 'survexport' # for parsing .3d files and producing .pos files + +PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) +LIBDIR = Path(REPOS_ROOT_PATH) / 'lib' / PV + +EXPOWEB = Path(REPOS_ROOT_PATH + 'expoweb/') +SURVEYS = REPOS_ROOT_PATH +SURVEY_SCANS = REPOS_ROOT_PATH + 'expofiles/surveyscans/' +FILES = REPOS_ROOT_PATH + 'expofiles' +PHOTOS_ROOT = REPOS_ROOT_PATH + 'expofiles/photos/' + +TROGGLE_PATH = Path(__file__).parent +TEMPLATE_PATH = TROGGLE_PATH / 'templates' +MEDIA_ROOT = TROGGLE_PATH / 'media' +JSLIB_ROOT = TROGGLE_PATH / 'media' / 'jslib' # used for CaveViewer JS utility + + +CAVEDESCRIPTIONS = EXPOWEB / "cave_data" +ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data" + + +PYTHON_PATH = REPOS_ROOT_PATH + 'troggle/' + + +#URL_ROOT = 'http://expo.survex.com/' +URL_ROOT = '/' +DIR_ROOT = Path("") #this should end in / if a value is given +EXPOWEB_URL = '/' +SURVEYS_URL = '/survey_scans/' + +REPOS_ROOT_PATH = Path(REPOS_ROOT_PATH) + +SURVEX_DATA = REPOS_ROOT_PATH / "loser" +DRAWINGS_DATA = REPOS_ROOT_PATH / "drawings" + + +EXPOFILES = REPOS_ROOT_PATH / "expofiles" +SCANS_ROOT = EXPOFILES / "surveyscans" +PHOTOS_ROOT = EXPOFILES / "photos" + +#EXPOFILES = urllib.parse.urljoin(REPOS_ROOT_PATH, 'expofiles/') +PHOTOS_URL = urllib.parse.urljoin(URL_ROOT, '/photos/') + +# MEDIA_URL is used by urls.py in a regex. See urls.py & core/views_surveys.py +MEDIA_URL = '/site_media/' + + +STATIC_URL = urllib.parse.urljoin(URL_ROOT , '/static/') # used by Django admin pages. Do not delete. +JSLIB_URL = urllib.parse.urljoin(URL_ROOT , '/javascript/') # always fails, try to revive it ? +# STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py + +#TINY_MCE_MEDIA_ROOT = STATIC_ROOT + '/tiny_mce/' # not needed while TinyMCE not installed +#TINY_MCE_MEDIA_URL = STATIC_URL + '/tiny_mce/' # not needed while TinyMCE not installed + +LOGFILE = '/var/log/troggle/troggle.log' +IMPORTLOGFILE = '/var/log/troggle/import.log' + +# Sanitise these to be strings as Django seems to be particularly sensitive to crashing if they aren't +STATIC_URL = str(STATIC_URL) + "/" +MEDIA_URL = str(MEDIA_URL) + "/" + +print(" + finished importing troggle/localsettings.py") diff --git a/_deploy/debian/localsettingsserver.py b/_deploy/debian/localsettingsserver2023-01-cleansed.py similarity index 98% rename from _deploy/debian/localsettingsserver.py rename to _deploy/debian/localsettingsserver2023-01-cleansed.py index d99e61d..a1e22ca 100644 --- a/_deploy/debian/localsettingsserver.py +++ b/_deploy/debian/localsettingsserver2023-01-cleansed.py @@ -32,7 +32,7 @@ DATABASES = { 'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. 'NAME' : 'troggle', # Or path to database file if using sqlite3. 'USER' : 'expo', # Not used with sqlite3. - 'PASSWORD' : 'uFqP56B4XleeyIW', # Not used with sqlite3. + 'PASSWORD' : '123456789012345', # Not used with sqlite3.Not the real password 'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. 'PORT' : '', # Set to empty string for default. Not used with sqlite3. } diff --git a/_deploy/debian/settings.py b/_deploy/debian/settings2023-02-10.py similarity index 100% rename from _deploy/debian/settings.py rename to _deploy/debian/settings2023-02-10.py diff --git a/_deploy/debian/settings2023-04-23.py b/_deploy/debian/settings2023-04-23.py new file mode 100644 index 0000000..938c024 --- /dev/null +++ b/_deploy/debian/settings2023-04-23.py @@ -0,0 +1,147 @@ +""" +Django settings for troggle project. + +For more information on this file, see +https://docs.djangoproject.com/en/dev/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/dev/ref/settings/ +""" +# Imports should be grouped in the following order: + +# 1.Standard library imports. +# 2.Related third party imports. +# 3.Local application/library specific imports. +# 4.You should put a blank line between each group of imports. + + + +print("* importing troggle/settings.py") + +# default value, then gets overwritten by real secrets +SECRET_KEY = "not-the-real-secret-key-a#vaeozn0---^fj!355qki*vj2" + +GIT = "git" # command for running git + +# Note that this builds upon the django system installed +# global settings in +# django/conf/global_settings.py which is automatically loaded first. +# read https://docs.djangoproject.com/en/dev/topics/settings/ + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +# BASE_DIR = os.path.dirname(os.path.dirname(__file__)) + +# Django settings for troggle project. + +ALLOWED_HOSTS = ["*", "expo.survex.com", ".survex.com", "localhost", "127.0.0.1", "192.168.0.5"] + +ADMINS = ( + # ('Your Name', 'your_email@domain.com'), +) +MANAGERS = ADMINS + +# LOGIN_URL = '/accounts/login/' # this is the default value so does not need to be set + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# If running in a Windows environment this must be set to the same as your +# system time zone. +USE_TZ = True +TIME_ZONE = "Europe/London" + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = "en-uk" + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True +USE_L10N = True + +FIX_PERMISSIONS = [] + +# top-level survex file basename (without .svx) +SURVEX_TOPNAME = "1623-and-1626-no-schoenberg-hs" + + +# Caves for which survex files exist, but are not otherwise registered +# replaced (?) by expoweb/cave_data/pendingcaves.txt +# PENDING = ["1626-361", "2007-06", "2009-02", +# "2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06", +# "2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888", +# "2018-pf-01", "2018-pf-02"] + +APPEND_SLASH = ( + False # never relevant because we have urls that match unknown files and produce an 'edit this page' response +) +SMART_APPEND_SLASH = True # not eorking as middleware different after Dj2.0 + + +LOGIN_REDIRECT_URL = "/" # does not seem to have any effect + +SECURE_CONTENT_TYPE_NOSNIFF = True +SECURE_BROWSER_XSS_FILTER = True +# SESSION_COOKIE_SECURE = True # if enabled, cannot login to Django control panel, bug elsewhere? +# CSRF_COOKIE_SECURE = True # if enabled only sends cookies over SSL +X_FRAME_OPTIONS = "DENY" # changed to "DENY" after I eliminated all the iframes e.g. /xmlvalid.html + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" # from Django 3.2 + +INSTALLED_APPS = ( + "django.contrib.admin", + "django.contrib.auth", # includes the url redirections for login, logout + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.admindocs", + "django.forms", # Required to customise widget templates + # 'django.contrib.staticfiles', # We put our CSS etc explicitly in the right place so do not need this + "troggle.core", +) + +FORM_RENDERER = "django.forms.renderers.TemplatesSetting" # Required to customise widget templates + +# See the recommended order of these in https://docs.djangoproject.com/en/dev/ref/middleware/ +# Note that this is a radically different onion architecture from earlier versions though it looks the same, +# see https://docs.djangoproject.com/en/dev/topics/http/middleware/#upgrading-pre-django-1-10-style-middleware +# Seriously, read this: https://www.webforefront.com/django/middlewaredjango.html which is MUCH BETTER than the docs +MIDDLEWARE = [ + #'django.middleware.security.SecurityMiddleware', # SECURE_SSL_REDIRECT and SECURE_SSL_HOST # we don't use this + "django.middleware.gzip.GZipMiddleware", # not needed when expofiles and photos served by apache + "django.contrib.sessions.middleware.SessionMiddleware", # Manages sessions, if CSRF_USE_SESSIONS then it needs to be early + "django.middleware.common.CommonMiddleware", # DISALLOWED_USER_AGENTS, APPEND_SLASH and PREPEND_WWW + "django.middleware.csrf.CsrfViewMiddleware", # Cross Site Request Forgeries by adding hidden form fields to POST + "django.contrib.auth.middleware.AuthenticationMiddleware", # Adds the user attribute, representing the currently-logged-in user + "django.contrib.admindocs.middleware.XViewMiddleware", # this and docutils needed by admindocs + "django.contrib.messages.middleware.MessageMiddleware", # Cookie-based and session-based message support. Needed by admin system + "django.middleware.clickjacking.XFrameOptionsMiddleware", # clickjacking protection via the X-Frame-Options header + #'django.middleware.security.SecurityMiddleware', # SECURE_HSTS_SECONDS, SECURE_CONTENT_TYPE_NOSNIFF, SECURE_BROWSER_XSS_FILTER, SECURE_REFERRER_POLICY, and SECURE_SSL_REDIRECT + #'troggle.core.middleware.SmartAppendSlashMiddleware' # needs adapting after Dj2.0 +] + +ROOT_URLCONF = "troggle.urls" + +WSGI_APPLICATION = "troggle.wsgi.application" # change to asgi as soon as we upgrade to Django 3.0 + +ACCOUNT_ACTIVATION_DAYS = 3 + +# AUTH_PROFILE_MODULE = 'core.person' # used by removed profiles app ? + +QM_PATTERN = "\[\[\s*[Qq][Mm]:([ABC]?)(\d{4})-(\d*)-(\d*)\]\]" + +# Re-enable TinyMCE when Dj upgraded to v3. Also templates/editexpopage.html +# TINYMCE_DEFAULT_CONFIG = { +# 'plugins': "table,spellchecker,paste,searchreplace", +# 'theme': "advanced", +# } +# TINYMCE_SPELLCHECKER = False +# TINYMCE_COMPRESSOR = True + +TEST_RUNNER = "django.test.runner.DiscoverRunner" + +from localsettings import * + +# localsettings needs to take precedence. Call it to override any existing vars. From f8faf6c2ba327e0ede1a7d2ca21827f9ad5f485a Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Thu, 25 May 2023 12:56:14 +0300 Subject: [PATCH 07/17] updated build stuff --- requirements-devupgrade.txt | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 requirements-devupgrade.txt diff --git a/requirements-devupgrade.txt b/requirements-devupgrade.txt new file mode 100644 index 0000000..19fe6d8 --- /dev/null +++ b/requirements-devupgrade.txt @@ -0,0 +1,20 @@ +# Philip bleeding edge config +asgiref==3.6.0 +beautifulsoup4==4.12.2 +black==23.1.0 +click==8.1.3 +coverage==7.1.0 +Django==4.2 +docutils==0.19 +isort==5.12.0 +mypy-extensions==1.0.0 +packaging==23.0 +pathspec==0.11.0 +Pillow==9.4.0 +platformdirs==3.0.0 +pytz==2022.7 +ruff==0.0.245 +soupsieve==2.4.1 +sqlparse==0.4.3 +Unidecode==1.3.6 +piexif==1.1.3 \ No newline at end of file From 33d279b95a2a8b3d8d0ac713eea8f83b9a569833 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 17 Jul 2023 19:05:22 +0300 Subject: [PATCH 08/17] rearrange variant debian, ubuntu dev environemnts --- .../localsettings2023-04-05-cleansed.py | 0 .../localsettings2023-04-05-secret.py | 160 +++++++++++++++++ .../localsettingsserver2023-01-cleansed.py | 0 .../settings2023-02-10.py | 0 .../settings2023-04-23.py | 0 .../{debian => debian-server}/apache2.conf | 0 _deploy/{debian => debian-server}/envvars | 0 .../localsettingsserver-old.py | 0 .../localsettingsserver2023-01-secret.py | 164 ++++++++++++++++++ _deploy/debian-server/requirements-server.txt | 23 +++ _deploy/{debian => debian-server}/serversetup | 0 .../sitecustomize.py | 0 ...his-is-the-live-expo.survex.com-system.txt | 0 .../wookey-exposerver-recipe.txt | 0 _deploy/potatohut/localsettingspotatohut.py | 79 --------- _deploy/readme.txt | 18 ++ _deploy/wsl/localsettingsWSL.py | 7 +- 17 files changed, 369 insertions(+), 82 deletions(-) rename _deploy/{debian => debian-laptops}/localsettings2023-04-05-cleansed.py (100%) create mode 100644 _deploy/debian-laptops/localsettings2023-04-05-secret.py rename _deploy/{debian => debian-laptops}/localsettingsserver2023-01-cleansed.py (100%) rename _deploy/{debian => debian-laptops}/settings2023-02-10.py (100%) rename _deploy/{debian => debian-laptops}/settings2023-04-23.py (100%) rename _deploy/{debian => debian-server}/apache2.conf (100%) rename _deploy/{debian => debian-server}/envvars (100%) rename _deploy/{debian => debian-server}/localsettingsserver-old.py (100%) create mode 100644 _deploy/debian-server/localsettingsserver2023-01-secret.py create mode 100644 _deploy/debian-server/requirements-server.txt rename _deploy/{debian => debian-server}/serversetup (100%) rename _deploy/{debian => debian-server}/sitecustomize.py (100%) rename _deploy/{debian => debian-server}/this-is-the-live-expo.survex.com-system.txt (100%) rename _deploy/{debian => debian-server}/wookey-exposerver-recipe.txt (100%) delete mode 100644 _deploy/potatohut/localsettingspotatohut.py create mode 100644 _deploy/readme.txt diff --git a/_deploy/debian/localsettings2023-04-05-cleansed.py b/_deploy/debian-laptops/localsettings2023-04-05-cleansed.py similarity index 100% rename from _deploy/debian/localsettings2023-04-05-cleansed.py rename to _deploy/debian-laptops/localsettings2023-04-05-cleansed.py diff --git a/_deploy/debian-laptops/localsettings2023-04-05-secret.py b/_deploy/debian-laptops/localsettings2023-04-05-secret.py new file mode 100644 index 0000000..f73b512 --- /dev/null +++ b/_deploy/debian-laptops/localsettings2023-04-05-secret.py @@ -0,0 +1,160 @@ +import os +import sys +import urllib.parse +from pathlib import Path + +"""Settings for a troggle installation which may vary among different +installations: for development or deployment, in a docker image or +python virtual environment (venv), on ubuntu, debian or in Windows +System for Linux (WSL), on the main server or in the potato hut, +using SQLite or mariaDB. + +It sets the directory locations for the major parts of the system so +that e.g. expofiles can be on a different filesystem. + +This file is included at the end of the main troggle/settings.py file so that +it overwrites defaults in that file. + +NOTE this file is vastly out of sync with troggle/_deploy/wsl/localsettings.py +which is the most recent version used in active maintenance. There should be +essential differences, but there and many, many non-essential differences which +should be eliminated for clarity and to use modern idioms. 8 March 2023. +""" + +print(" * importing troggle/localsettings.py") + +# DO NOT check this file into the git repo - it contains real passwords. + +EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever +#SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + 'NAME' : 'troggle', # Or path to database file if using sqlite3. + 'USER' : 'expo', # Not used with sqlite3. + 'PASSWORD' : 'uFqP56B4XleeyIW', # Not used with sqlite3. + 'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. + 'PORT' : '', # Set to empty string for default. Not used with sqlite3. + } +} + + +EXPOUSER = 'expo' +EXPOUSERPASS = '161:gosser' +EXPOADMINUSER = 'expoadmin' +EXPOADMINUSERPASS = 'gosser:161' +EXPOUSER_EMAIL = 'wookey@wookware.org' +EXPOADMINUSER_EMAIL = 'wookey@wookware.org' + +REPOS_ROOT_PATH = '/home/expo/' +sys.path.append(REPOS_ROOT_PATH) +sys.path.append(REPOS_ROOT_PATH + 'troggle') +# Define the path to the django app (troggle in this case) +PYTHON_PATH = REPOS_ROOT_PATH + 'troggle/' + + +PHOTOS_YEAR = "2023" +# add in 358 when they don't make it crash horribly +NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"] + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + PYTHON_PATH + "templates" + ], + 'OPTIONS': { + 'debug': 'DEBUG', + 'context_processors': [ + # django.template.context_processors.csrf, # is always enabled and cannot be removed, sets csrf_token + 'django.contrib.auth.context_processors.auth', # knowledge of logged-on user & permissions + 'core.context.troggle_context', # in core/troggle.py + 'django.template.context_processors.debug', + #'django.template.context_processors.request', # copy of current request, added in trying to make csrf work + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', # includes a variable MEDIA_URL + 'django.template.context_processors.static', # includes a variable STATIC_URL + 'django.template.context_processors.tz', + 'django.contrib.messages.context_processors.messages', + ], + 'loaders': [ + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', #For each app, inc admin, in INSTALLED_APPS, loader looks for /templates + # insert your own TEMPLATE_LOADERS here + ] + }, + }, +] + +PUBLIC_SITE = True + +# This should be False for normal running +DEBUG = False +CACHEDPAGES = True # experimental page cache for a handful of page types + + +# executables: +CAVERN = 'cavern' # for parsing .svx files and producing .3d files +SURVEXPORT = 'survexport' # for parsing .3d files and producing .pos files + +PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) +LIBDIR = Path(REPOS_ROOT_PATH) / 'lib' / PV + +EXPOWEB = Path(REPOS_ROOT_PATH + 'expoweb/') +SURVEYS = REPOS_ROOT_PATH +SURVEY_SCANS = REPOS_ROOT_PATH + 'expofiles/surveyscans/' +FILES = REPOS_ROOT_PATH + 'expofiles' +PHOTOS_ROOT = REPOS_ROOT_PATH + 'expofiles/photos/' + +TROGGLE_PATH = Path(__file__).parent +TEMPLATE_PATH = TROGGLE_PATH / 'templates' +MEDIA_ROOT = TROGGLE_PATH / 'media' +JSLIB_ROOT = TROGGLE_PATH / 'media' / 'jslib' # used for CaveViewer JS utility + + +CAVEDESCRIPTIONS = EXPOWEB / "cave_data" +ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data" + + +PYTHON_PATH = REPOS_ROOT_PATH + 'troggle/' + + +#URL_ROOT = 'http://expo.survex.com/' +URL_ROOT = '/' +DIR_ROOT = Path("") #this should end in / if a value is given +EXPOWEB_URL = '/' +SURVEYS_URL = '/survey_scans/' + +REPOS_ROOT_PATH = Path(REPOS_ROOT_PATH) + +SURVEX_DATA = REPOS_ROOT_PATH / "loser" +DRAWINGS_DATA = REPOS_ROOT_PATH / "drawings" + + +EXPOFILES = REPOS_ROOT_PATH / "expofiles" +SCANS_ROOT = EXPOFILES / "surveyscans" +PHOTOS_ROOT = EXPOFILES / "photos" + +#EXPOFILES = urllib.parse.urljoin(REPOS_ROOT_PATH, 'expofiles/') +PHOTOS_URL = urllib.parse.urljoin(URL_ROOT, '/photos/') + +# MEDIA_URL is used by urls.py in a regex. See urls.py & core/views_surveys.py +MEDIA_URL = '/site_media/' + + +STATIC_URL = urllib.parse.urljoin(URL_ROOT , '/static/') # used by Django admin pages. Do not delete. +JSLIB_URL = urllib.parse.urljoin(URL_ROOT , '/javascript/') # always fails, try to revive it ? +# STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py + +#TINY_MCE_MEDIA_ROOT = STATIC_ROOT + '/tiny_mce/' # not needed while TinyMCE not installed +#TINY_MCE_MEDIA_URL = STATIC_URL + '/tiny_mce/' # not needed while TinyMCE not installed + +LOGFILE = '/var/log/troggle/troggle.log' +IMPORTLOGFILE = '/var/log/troggle/import.log' + +# Sanitise these to be strings as Django seems to be particularly sensitive to crashing if they aren't +STATIC_URL = str(STATIC_URL) + "/" +MEDIA_URL = str(MEDIA_URL) + "/" + +print(" + finished importing troggle/localsettings.py") diff --git a/_deploy/debian/localsettingsserver2023-01-cleansed.py b/_deploy/debian-laptops/localsettingsserver2023-01-cleansed.py similarity index 100% rename from _deploy/debian/localsettingsserver2023-01-cleansed.py rename to _deploy/debian-laptops/localsettingsserver2023-01-cleansed.py diff --git a/_deploy/debian/settings2023-02-10.py b/_deploy/debian-laptops/settings2023-02-10.py similarity index 100% rename from _deploy/debian/settings2023-02-10.py rename to _deploy/debian-laptops/settings2023-02-10.py diff --git a/_deploy/debian/settings2023-04-23.py b/_deploy/debian-laptops/settings2023-04-23.py similarity index 100% rename from _deploy/debian/settings2023-04-23.py rename to _deploy/debian-laptops/settings2023-04-23.py diff --git a/_deploy/debian/apache2.conf b/_deploy/debian-server/apache2.conf similarity index 100% rename from _deploy/debian/apache2.conf rename to _deploy/debian-server/apache2.conf diff --git a/_deploy/debian/envvars b/_deploy/debian-server/envvars similarity index 100% rename from _deploy/debian/envvars rename to _deploy/debian-server/envvars diff --git a/_deploy/debian/localsettingsserver-old.py b/_deploy/debian-server/localsettingsserver-old.py similarity index 100% rename from _deploy/debian/localsettingsserver-old.py rename to _deploy/debian-server/localsettingsserver-old.py diff --git a/_deploy/debian-server/localsettingsserver2023-01-secret.py b/_deploy/debian-server/localsettingsserver2023-01-secret.py new file mode 100644 index 0000000..d99e61d --- /dev/null +++ b/_deploy/debian-server/localsettingsserver2023-01-secret.py @@ -0,0 +1,164 @@ +import os +import sys +import urllib.parse +from pathlib import Path + +"""Settings for a troggle installation which may vary among different +installations: for development or deployment, in a docker image or +python virtual environment (venv), on ubuntu, debian or in Windows +System for Linux (WSL), on the main server or in the potato hut, +using SQLite or mariaDB. + +It sets the directory locations for the major parts of the system so +that e.g. expofiles can be on a different filesystem, or /javascript/ can be in +a system-wide location rather than just a local directory. + +This file is included at the end of the main troggle/settings.py file so that +it overwrites defaults in that file. + +Read https://realpython.com/python-pathlib/ +Read https://adamj.eu/tech/2020/03/16/use-pathlib-in-your-django-project/ +""" + +print(" * importing troggle/localsettings.py") + +# DO NOT check this file into the git repo - it contains real passwords. + +EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever +#SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + 'NAME' : 'troggle', # Or path to database file if using sqlite3. + 'USER' : 'expo', # Not used with sqlite3. + 'PASSWORD' : 'uFqP56B4XleeyIW', # Not used with sqlite3. + 'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. + 'PORT' : '', # Set to empty string for default. Not used with sqlite3. + } +} + + +EXPOUSER = 'expo' +EXPOADMINUSER = 'expoadmin' +EXPOUSER_EMAIL = 'wookey@wookware.org' +EXPOADMINUSER_EMAIL = 'wookey@wookware.org' + +SECRET_KEY = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" +EXPOUSERPASS = "nope" +EXPOADMINUSERPASS = "nope" +EMAIL_HOST_PASSWORD = "nope" + +REPOS_ROOT_PATH = '/home/expo/' +sys.path.append(REPOS_ROOT_PATH) +sys.path.append(REPOS_ROOT_PATH + 'troggle') +# Define the path to the django app (troggle in this case) +PYTHON_PATH = REPOS_ROOT_PATH + 'troggle/' +PHOTOS_YEAR = "2022" +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + PYTHON_PATH + "templates" + ], + 'OPTIONS': { + 'debug': 'DEBUG', + 'context_processors': [ + # django.template.context_processors.csrf, # is always enabled and cannot be removed, sets csrf_token + 'django.contrib.auth.context_processors.auth', # knowledge of logged-on user & permissions + 'core.context.troggle_context', # in core/troggle.py + 'django.template.context_processors.debug', + #'django.template.context_processors.request', # copy of current request, added in trying to make csrf work + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', # includes a variable MEDIA_URL + 'django.template.context_processors.static', # includes a variable STATIC_URL + 'django.template.context_processors.tz', + 'django.contrib.messages.context_processors.messages', + ], + 'loaders': [ + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', #For each app, inc admin, in INSTALLED_APPS, loader looks for /templates + # insert your own TEMPLATE_LOADERS here + ] + }, + }, +] + +PUBLIC_SITE = True + +# This should be False for normal running +DEBUG = True +CACHEDPAGES = True # experimental page cache for a handful of page types + +SURVEX_DATA = REPOS_ROOT_PATH + 'loser/' +DRAWINGS_DATA = REPOS_ROOT_PATH + 'drawings/' + +# executables: +CAVERN = 'cavern' # for parsing .svx files and producing .3d files +SURVEXPORT = 'survexport' # for parsing .3d files and producing .pos files + +EXPOWEB = REPOS_ROOT_PATH + 'expoweb/' +#SURVEYS = REPOS_ROOT_PATH +SCANS_ROOT = REPOS_ROOT_PATH + 'expofiles/surveyscans/' +FILES = REPOS_ROOT_PATH + 'expofiles' +PHOTOS_ROOT = REPOS_ROOT_PATH + 'expofiles/photos/' + +TROGGLE_PATH = Path(__file__).parent +TEMPLATE_PATH = TROGGLE_PATH / 'templates' +MEDIA_ROOT = TROGGLE_PATH / 'media' +JSLIB_ROOT = TROGGLE_PATH / 'media' / 'jslib' # used for CaveViewer JS utility + + +CAVEDESCRIPTIONS = os.path.join(EXPOWEB, "cave_data") +ENTRANCEDESCRIPTIONS = os.path.join(EXPOWEB, "entrance_data") + +# CACHEDIR = REPOS_ROOT_PATH + 'expowebcache/' +# THREEDCACHEDIR = CACHEDIR + '3d/' +# THUMBNAILCACHE = CACHEDIR + 'thumbs' + +PYTHON_PATH = REPOS_ROOT_PATH + 'troggle/' +PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) +LIBDIR = Path(REPOS_ROOT_PATH) / 'lib' / PV + +#Note that all these *_URL constants are not actually used in urls.py, they should be.. +#URL_ROOT = 'http://expo.survex.com/' +URL_ROOT = '/' +DIR_ROOT = ''#this should end in / if a value is given +EXPOWEB_URL = '/' +SCANS_URL = '/survey_scans/' +EXPOFILES = urllib.parse.urljoin(REPOS_ROOT_PATH, 'expofiles/') +PHOTOS_URL = urllib.parse.urljoin(URL_ROOT, '/photos/') + +# MEDIA_URL is used by urls.py in a regex. See urls.py & core/views_surveys.py +MEDIA_URL = '/site_media/' + + +STATIC_URL = urllib.parse.urljoin(URL_ROOT , '/static/') # used by Django admin pages. Do not delete. +JSLIB_URL = urllib.parse.urljoin(URL_ROOT , '/javascript/') # always fails, try to revive it ? + +#TINY_MCE_MEDIA_ROOT = STATIC_ROOT + '/tiny_mce/' # not needed while TinyMCE not installed +#TINY_MCE_MEDIA_URL = STATIC_URL + '/tiny_mce/' # not needed while TinyMCE not installed + +LOGFILE = '/var/log/troggle/troggle.log' +IMPORTLOGFILE = '/var/log/troggle/import.log' + +# add in 358 when they don't make it crash horribly +NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"] + +# Sanitise these to be strings as all other code is expecting strings +# and we have not made the change to pathlib Path type in the other localsettings-* variants yet. +CAVEDESCRIPTIONS = os.fspath(CAVEDESCRIPTIONS) +ENTRANCEDESCRIPTIONS = os.fspath(ENTRANCEDESCRIPTIONS) +LOGFILE = os.fspath(LOGFILE) +#SURVEYS = os.fspath(SURVEYS) +EXPOWEB = os.fspath(EXPOWEB) +DRAWINGS_DATA = os.fspath(DRAWINGS_DATA) +SURVEX_DATA = os.fspath(SURVEX_DATA) +REPOS_ROOT_PATH = os.fspath(REPOS_ROOT_PATH) +TEMPLATE_PATH = os.fspath(TROGGLE_PATH) +MEDIA_ROOT = os.fspath(MEDIA_ROOT) +JSLIB_ROOT = os.fspath(JSLIB_ROOT) +SCANS_ROOT = os.fspath(SCANS_ROOT) +LIBDIR = os.fspath(LIBDIR) + +print(" + finished importing troggle/localsettings.py") \ No newline at end of file diff --git a/_deploy/debian-server/requirements-server.txt b/_deploy/debian-server/requirements-server.txt new file mode 100644 index 0000000..1a5abb1 --- /dev/null +++ b/_deploy/debian-server/requirements-server.txt @@ -0,0 +1,23 @@ +#This requirements txt matches the libaries as of 2023-07-09 on expo.survex.com + +#Nb on the server asgiref==3.3.0, however this conflicts with the Django==3.2.12 requirement +asgiref==3.3.2 +Django==3.2.12 +docutils==0.16 +packaging==20.9 +Pillow==8.1.2 +pytz==2021.1 +sqlparse==0.4.1 +Unidecode==1.2.0 +beautifulsoup4==4.9.3 +piexif==1.1.3 + +#Not installed on expo.survex.com +#black==23.3 +#click==8.1.3 +#coverage==7.2 +#isort==5.12.0 +#mypy-extensions==1.0.0 +#pathspec==0.11 +#platformdirs==3.8 +#ruff==0.0.245 diff --git a/_deploy/debian/serversetup b/_deploy/debian-server/serversetup similarity index 100% rename from _deploy/debian/serversetup rename to _deploy/debian-server/serversetup diff --git a/_deploy/debian/sitecustomize.py b/_deploy/debian-server/sitecustomize.py similarity index 100% rename from _deploy/debian/sitecustomize.py rename to _deploy/debian-server/sitecustomize.py diff --git a/_deploy/debian/this-is-the-live-expo.survex.com-system.txt b/_deploy/debian-server/this-is-the-live-expo.survex.com-system.txt similarity index 100% rename from _deploy/debian/this-is-the-live-expo.survex.com-system.txt rename to _deploy/debian-server/this-is-the-live-expo.survex.com-system.txt diff --git a/_deploy/debian/wookey-exposerver-recipe.txt b/_deploy/debian-server/wookey-exposerver-recipe.txt similarity index 100% rename from _deploy/debian/wookey-exposerver-recipe.txt rename to _deploy/debian-server/wookey-exposerver-recipe.txt diff --git a/_deploy/potatohut/localsettingspotatohut.py b/_deploy/potatohut/localsettingspotatohut.py deleted file mode 100644 index 184f0ab..0000000 --- a/_deploy/potatohut/localsettingspotatohut.py +++ /dev/null @@ -1,79 +0,0 @@ -import sys - -# This is an example file. Copy it to localsettings.py, set the -# password and _don't_ check that file back to the repo as it exposes -# your/our password to the world! - -# This will ALL NEED TO BE CHANGED to match localsettingsWSL / python3 / Django v1.18.29 -# This WILL NOT WORK as it is for an earlier version of Django - - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - 'NAME': 'troggle', # Or path to database file if using sqlite3. - 'USER': 'expo', # Not used with sqlite3. - 'PASSWORD': 'notarealpassword', # Not used with sqlite3. - 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT': '', # Set to empty string for default. Not used with sqlite3. - } -} - -REPOS_ROOT_PATH = '/home/expo/' -sys.path.append(REPOS_ROOT_PATH) -sys.path.append(REPOS_ROOT_PATH + 'troggle') - -PUBLIC_SITE = True -FIX_PERMISSIONS = ["sudo", "/usr/local/bin/fix_permissions"] - -SURVEX_DATA = REPOS_ROOT_PATH + 'loser/' -DRAWINGS_DATA = REPOS_ROOT_PATH + 'drawings/' -THREEDCACHEDIR = REPOS_ROOT_PATH + 'expowebcache/3d/' - -CAVERN = 'cavern' -THREEDTOPOS = '3dtopos' -EXPOWEB = REPOS_ROOT_PATH + 'expoweb/' -SURVEYS = REPOS_ROOT_PATH -SURVEY_SCANS = REPOS_ROOT_PATH + 'expoimages/' -FILES = REPOS_ROOT_PATH + 'expoimages' - - -PYTHON_PATH = '/home/expo/troggle/' - -URL_ROOT = "http://expo/" -DIR_ROOT = ''#this should end in / if a value is given -EXPOWEB_URL = 'http://expo/' -SURVEYS_URL = 'http://expo/survey_scans/' -MEDIA_URL = '/' + DIR_ROOT + 'site_media/' - -STATIC_URL = URL_ROOT -STATIC_ROOT = DIR_ROOT - -MEDIA_ROOT = REPOS_ROOT_PATH + '/troggle/media/' -MEDIA_ADMIN_DIR = '/usr/lib/python2.7/site-packages/django/contrib/admin/media/' - -JSLIB_PATH = '/usr/share/javascript/' - -TINY_MCE_MEDIA_ROOT = '/usr/share/tinymce/www/' -TINY_MCE_MEDIA_URL = URL_ROOT + DIR_ROOT + 'tinymce_media/' - -TEMPLATE_DIRS = ( - PYTHON_PATH + "templates", - # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. -) - -LOGFILE = '/home/expo/troggle/troggle_log.txt' - -FEINCMS_ADMIN_MEDIA='/site_media/feincms/' - -EMAIL_HOST = "smtp.gmail.com" - -EMAIL_HOST_USER = "cuccexpo@gmail.com" - -EMAIL_HOST_PASSWORD = "insert-real-email-password-here" - -EMAIL_PORT=587 - -EMAIL_USE_TLS = True diff --git a/_deploy/readme.txt b/_deploy/readme.txt new file mode 100644 index 0000000..0986324 --- /dev/null +++ b/_deploy/readme.txt @@ -0,0 +1,18 @@ +2023-07-17 Philip Sargent + +Trying to sort out configurations as we got into a bit of a mess on +Expo in the last couple of weeks with two (notionally identical Debian +Bullseye) expo laptops Crowley (which has local troggle installed and +can run locally) and Aziraphale (has local copy of troggle repo but is +not configured to run locally), Martin Green's laptop (Ubuntu 22.04.2), +Philip's Barbie laptop Ubuntu 22.04.3). And of course the server itself +expo.survex.com which is running Debian Bullseye. But most development +recently had been done on Philip's two other machines, desktop and PC, +both running Ubuntu on WSL on Windows and both using venv environments, +which Crowley also does. + +- settings.py +is common to all configurations, +but these are all different: +- localsettings.py +- requirements.txt \ No newline at end of file diff --git a/_deploy/wsl/localsettingsWSL.py b/_deploy/wsl/localsettingsWSL.py index ffead19..2824125 100644 --- a/_deploy/wsl/localsettingsWSL.py +++ b/_deploy/wsl/localsettingsWSL.py @@ -47,6 +47,7 @@ PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) REPOS_ROOT_PATH = Path(__file__).parent.parent LIBDIR = REPOS_ROOT_PATH / "lib" / PV + TROGGLE_PATH = Path(__file__).parent TEMPLATE_PATH = TROGGLE_PATH / "templates" MEDIA_ROOT = TROGGLE_PATH / "media" @@ -64,9 +65,9 @@ NOTABLECAVESHREFS = ["290", "291", "264", "258", "204", "359", "76", "107"] # PYTHON_PATH = os.fspath(PYTHON_PATH) PYTHON_PATH = REPOS_ROOT_PATH / "troggle" LOGFILE = PYTHON_PATH / "troggle.log" +SQLITEDB = PYTHON_PATH / "troggle.sqlite" +KMZ_ICONS_PATH = PYTHON_PATH / "kmz_icons" -#sys.path.append(os.fspath(REPOS_ROOT_PATH)) -#sys.path.append(os.fspath(PYTHON_PATH)) # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). @@ -99,7 +100,7 @@ DBSQLITE = { "default": { "ENGINE": "django.db.backends.sqlite3", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. #'NAME' : 'troggle.sqlite', - "NAME": "/home/philip/p11d4/troggle.sqlite", + "NAME": str(SQLITEDB), "USER": "expo", # Not used with sqlite3. "PASSWORD": "sekrit", # Not used with sqlite3. "HOST": "", # Set to empty string for localhost. Not used with sqlite3. From f76c22e843b85ba0ad99d0db9dd31f2969892d97 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 17 Jul 2023 19:25:40 +0300 Subject: [PATCH 09/17] moving dev configs to _deploy/ --- .../debian-laptops/os-trog.sh | 0 .../debian-laptops/venv-trog-crowley.sh | 0 .../localsettingsubuntu.py | 0 .../wsl/requirements-devupgrade.txt | 0 venv-trog.sh => _deploy/wsl/venv-trog.sh | 0 .../xubuntu/pre-push-barbie.sh | 0 core/TESTS/tests-cuyc-ignored.py | 2 +- localsettingsWSL.py | 180 ----------------- localsettingsXubuntu.py | 187 ------------------ 9 files changed, 1 insertion(+), 368 deletions(-) rename os-trog.sh => _deploy/debian-laptops/os-trog.sh (100%) rename venv-trog-crowley.sh => _deploy/debian-laptops/venv-trog-crowley.sh (100%) mode change 100755 => 100644 rename _deploy/{ubuntu => ubuntu-dev}/localsettingsubuntu.py (100%) rename requirements-devupgrade.txt => _deploy/wsl/requirements-devupgrade.txt (100%) rename venv-trog.sh => _deploy/wsl/venv-trog.sh (100%) rename pre-push-barbie.sh => _deploy/xubuntu/pre-push-barbie.sh (100%) mode change 100755 => 100644 delete mode 100644 localsettingsWSL.py delete mode 100644 localsettingsXubuntu.py diff --git a/os-trog.sh b/_deploy/debian-laptops/os-trog.sh similarity index 100% rename from os-trog.sh rename to _deploy/debian-laptops/os-trog.sh diff --git a/venv-trog-crowley.sh b/_deploy/debian-laptops/venv-trog-crowley.sh old mode 100755 new mode 100644 similarity index 100% rename from venv-trog-crowley.sh rename to _deploy/debian-laptops/venv-trog-crowley.sh diff --git a/_deploy/ubuntu/localsettingsubuntu.py b/_deploy/ubuntu-dev/localsettingsubuntu.py similarity index 100% rename from _deploy/ubuntu/localsettingsubuntu.py rename to _deploy/ubuntu-dev/localsettingsubuntu.py diff --git a/requirements-devupgrade.txt b/_deploy/wsl/requirements-devupgrade.txt similarity index 100% rename from requirements-devupgrade.txt rename to _deploy/wsl/requirements-devupgrade.txt diff --git a/venv-trog.sh b/_deploy/wsl/venv-trog.sh similarity index 100% rename from venv-trog.sh rename to _deploy/wsl/venv-trog.sh diff --git a/pre-push-barbie.sh b/_deploy/xubuntu/pre-push-barbie.sh old mode 100755 new mode 100644 similarity index 100% rename from pre-push-barbie.sh rename to _deploy/xubuntu/pre-push-barbie.sh diff --git a/core/TESTS/tests-cuyc-ignored.py b/core/TESTS/tests-cuyc-ignored.py index ac4e5b3..976c97f 100644 --- a/core/TESTS/tests-cuyc-ignored.py +++ b/core/TESTS/tests-cuyc-ignored.py @@ -26,7 +26,7 @@ from django.test import Client, SimpleTestCase, TestCase, TransactionTestCase class ImportTest(TestCase): def test_import_imports(self): - ed to go through all modules and copy all imports here + #ed to go through all modules and copy all imports here from io import StringIO from cuy.club.models import (Article, Event, Member, Webpage, diff --git a/localsettingsWSL.py b/localsettingsWSL.py deleted file mode 100644 index 42a2a1c..0000000 --- a/localsettingsWSL.py +++ /dev/null @@ -1,180 +0,0 @@ -import sys -from pathlib import Path - -"""Settings for a troggle installation which may vary among different -installations: for development or deployment, in a docker image or -python virtual environment (venv), on ubuntu, debian or in Windows -System for Linux (WSL), on the main server or in the potato hut, -using SQLite or mariaDB. - -It sets the directory locations for the major parts of the system so -that e.g. expofiles can be on a different filesystem, or /javascript/ can be in -a system-wide location rather than just a local directory. - -This file is included at the end of the main troggle/settings.py file so that -it overwrites defaults in that file. - -Read https://realpython.com/python-pathlib/ -Read https://adamj.eu/tech/2020/03/16/use-pathlib-in-your-django-project/ -""" - -print(" * importing troggle/localsettings.py") - -# ----------------------------------------------------------------- -# THINK before you push this to a repo -# - have you checked that credentials.py is in .gitignore ? -# - we don't want to have to change the expo system password ! -# ----------------------------------------------------------------- -# default values, real secrets imported from credentials.py - -SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py" -EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py" -EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py" -EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py" - -EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate. -# SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely - -SERVERPORT = "8000" # not needed - -PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) - -# Troggle does a lot of file-handling. This is very error-prone when using primitive methods, -# so we use pathlib which has been standard since python 3.4 -# If pathlib is new to you, you will need to read https://realpython.com/python-pathlib/ - -# --------------------- MEDIA redirections BEGIN --------------------- -REPOS_ROOT_PATH = Path(__file__).parent.parent -LIBDIR = REPOS_ROOT_PATH / "lib" / PV - -TROGGLE_PATH = Path(__file__).parent -TEMPLATE_PATH = TROGGLE_PATH / "templates" -MEDIA_ROOT = TROGGLE_PATH / "media" -JSLIB_ROOT = TROGGLE_PATH / "media" / "jslib" # used for CaveViewer JS utility - -EXPOFILES = REPOS_ROOT_PATH / "expofiles" - -SCANS_ROOT = EXPOFILES / "surveyscans" -PHOTOS_ROOT = Path("/mnt/d/EXPO/PHOTOS") -PHOTOS_YEAR = "2023" -NOTABLECAVESHREFS = ["290", "291", "264", "258", "204", "359", "76", "107"] - - -PYTHON_PATH = REPOS_ROOT_PATH / "troggle" -LOGFILE = PYTHON_PATH / "troggle.log" - -# URL that handles the media served from MEDIA_ROOT. Make sure to use a -# trailing slash if there is a path component (optional in other cases). -MEDIA_URL = "/site-media/" - -DIR_ROOT = Path("") # this should end in / if a value is given -URL_ROOT = "/" -# URL_ROOT = 'http://localhost:'+ SERVERPORT +'/' - -# Note that these constants are not actually used in urls.py, they should be.. -# and they all need to end with / so using 'Path' doesn't work.. -MEDIA_URL = Path(URL_ROOT, "/site_media/") -PHOTOS_URL = Path(URL_ROOT, "/photos/") - - -STATIC_URL = Path(URL_ROOT, "/static/") # used by Django admin pages. Do not delete. -JSLIB_URL = Path(URL_ROOT, "/javascript/") # used for CaveViewer JS utility - -# STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py -# --------------------- MEDIA redirections END --------------------- - -PUBLIC_SITE = True -DEBUG = True # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES ! -CACHEDPAGES = True # experimental page cache for a handful of page types - -# executables: -CAVERN = "cavern" # for parsing .svx files and producing .3d files -SURVEXPORT = "survexport" # for parsing .3d files and producing .pos files - -DBSQLITE = { - "default": { - "ENGINE": "django.db.backends.sqlite3", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - "NAME": "/home/philip/p11d4/troggle.sqlite", - # 'NAME' : ':memory:', - "USER": "expo", # Not used with sqlite3. - "PASSWORD": "sekrit", # Not used with sqlite3. - "HOST": "", # Set to empty string for localhost. Not used with sqlite3. - "PORT": "", # Set to empty string for default. Not used with sqlite3. - } -} -DBMARIADB = { - "default": { - "ENGINE": "django.db.backends.mysql", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - "NAME": "troggle", # Or path to database file if using sqlite3. - "USER": "expo", - "PASSWORD": "my-secret-password-schwatzmooskogel", - "HOST": "", # Set to empty string for localhost. Not used with sqlite3. - "PORT": "", # Set to empty string for default. Not used with sqlite3. - } -} - -# default database for me is squlite -DBSWITCH = "sqlite" - -if DBSWITCH == "sqlite": - DATABASES = DBSQLITE -if DBSWITCH == "mariadb": - DATABASES = DBMARIADB - - - -TEMPLATES = [ - { - "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [TEMPLATE_PATH], - "OPTIONS": { - "debug": "DEBUG", - "context_processors": [ - # django.template.context_processors.csrf, # is always enabled and cannot be removed, sets csrf_token - "django.contrib.auth.context_processors.auth", # knowledge of logged-on user & permissions - "core.context.troggle_context", # in core/troggle.py - only used in expedition.html - "django.template.context_processors.debug", - "django.template.context_processors.i18n", - "django.template.context_processors.media", # includes a variable MEDIA_URL - "django.template.context_processors.static", # includes a variable STATIC_URL used by admin pages - "django.template.context_processors.tz", - "django.template.context_processors.request", # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar. - "django.contrib.messages.context_processors.messages", - ], - "loaders": [ - "django.template.loaders.filesystem.Loader", # default lcation is troggle/templates/ - "django.template.loaders.app_directories.Loader", # needed for admin 'app' - ], - }, - }, -] - -EXPOUSER = "expo" -EXPOUSER_EMAIL = "philip.sargent@gmail.com" -EXPOADMINUSER = "expoadmin" -EXPOADMINUSER_EMAIL = "philip.sargent@gmail.com" - -EMAIL_HOST = "smtp-auth.mythic-beasts.com" -EMAIL_HOST_USER = "django-test@klebos.net" # Philip Sargent really -EMAIL_PORT = 587 -EMAIL_USE_TLS = True -DEFAULT_FROM_EMAIL = "django-test@klebos.net" - -SURVEX_DATA = REPOS_ROOT_PATH / "loser" -DRAWINGS_DATA = REPOS_ROOT_PATH / "drawings" -EXPOWEB = REPOS_ROOT_PATH / "expoweb" -CAVEDESCRIPTIONS = EXPOWEB / "cave_data" -ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data" - -EXPOWEB_URL = "" -# SCANS_URL = '/survey_scans/' # defunct, removed. - -sys.path.append(str(REPOS_ROOT_PATH)) -sys.path.append(str(PYTHON_PATH)) -#TINY_MCE_MEDIA_ROOT = STATIC_ROOT + '/tiny_mce/' # not needed while TinyMCE not installed -#TINY_MCE_MEDIA_URL = STATIC_URL + '/tiny_mce/' # not needed while TinyMCE not installed - -# Sanitise these to be strings as Django seems to be particularly sensitive to crashing if they aren't -STATIC_URL = str(STATIC_URL) + "/" -MEDIA_URL = str(MEDIA_URL) + "/" - diff --git a/localsettingsXubuntu.py b/localsettingsXubuntu.py deleted file mode 100644 index 511bef8..0000000 --- a/localsettingsXubuntu.py +++ /dev/null @@ -1,187 +0,0 @@ -import os -import sys -import urllib.parse -from pathlib import Path - -"""Settings for a troggle installation which may vary among different -installations: for development or deployment, in a docker image or -python virtual environment (venv), on ubuntu, debian or in Windows -System for Linux (WSL), on the main server or in the potato hut, -using SQLite or mariaDB. - -It sets the directory locations for the major parts of the system so -that e.g. expofiles can be on a different filesystem, or /javascript/ can be in -a system-wide location rather than just a local directory. - -This file is included at the end of the main troggle/settings.py file so that -it overwrites defaults in that file. - -Read https://realpython.com/python-pathlib/ -Read https://adamj.eu/tech/2020/03/16/use-pathlib-in-your-django-project/ -""" - -print(" * importing troggle/localsettings.py") - -#----------------------------------------------------------------- -# THINK before you push this to a repo -# - have you checked that credentials.py is in .gitignore ? -# - we don't want to have to change the expo system password ! -#----------------------------------------------------------------- -# default values, real secrets imported from credentials.py - -SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py" -EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py" -EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py" -EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py" - -EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate. -# SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely - -SERVERPORT = '8000' # not needed - -PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) - -# Troggle does a lot of file-handling. This is very error-prone when using primitive methods, -# so we use pathlib which has been standard since python 3.4 -# If pathlib is new to you, you will need to read https://realpython.com/python-pathlib/ - -# --------------------- MEDIA redirections BEGIN --------------------- -REPOS_ROOT_PATH = Path(__file__).parent.parent -LIBDIR = REPOS_ROOT_PATH / 'lib' / PV - -TROGGLE_PATH = Path(__file__).parent -TEMPLATE_PATH = TROGGLE_PATH / "templates" -MEDIA_ROOT = TROGGLE_PATH / "media" -JSLIB_ROOT = TROGGLE_PATH / "media" / "jslib" # used for CaveViewer JS utility - -# EXPOFILES = Path('/media/philip/sd-huge1/cucc-expo/expofiles/') -EXPOFILES = REPOS_ROOT_PATH / "expofiles" - -SCANS_ROOT = EXPOFILES / 'surveyscans' -PHOTOS_ROOT = EXPOFILES / 'photos' -PHOTOS_YEAR = "2023" - -# URL that handles the media served from MEDIA_ROOT. Make sure to use a -# trailing slash if there is a path component (optional in other cases). -MEDIA_URL = '/site-media/' - -DIR_ROOT = ''#this should end in / if a value is given -URL_ROOT = '/' -# URL_ROOT = 'http://localhost:'+ SERVERPORT +'/' - -#Note that these constants are not actually used in urls.py, they should be.. -MEDIA_URL = urllib.parse.urljoin(URL_ROOT , '/site_media/') -SCANS_URL = urllib.parse.urljoin(URL_ROOT , '/survey_scans/') -PHOTOS_URL = urllib.parse.urljoin(URL_ROOT , '/photos/') -SVX_URL = urllib.parse.urljoin(URL_ROOT , '/survex/') - - -STATIC_URL = urllib.parse.urljoin(URL_ROOT , '/static/') # used by Django admin pages. Do not delete. -JSLIB_URL = urllib.parse.urljoin(URL_ROOT , '/javascript/') # used for CaveViewer JS utility - -#STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py -# --------------------- MEDIA redirections END --------------------- - -PUBLIC_SITE = True -DEBUG = True # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES ! -CACHEDPAGES = True # experimental page cache for a handful of page types - -# executables: -CAVERN = 'cavern' # for parsing .svx files and producing .3d files -SURVEXPORT = 'survexport' # for parsing .3d files and producing .pos files - -DBSQLITE = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - 'NAME' : 'troggle.sqlite', -# 'NAME' : ':memory:', - 'USER' : 'expo', # Not used with sqlite3. - 'PASSWORD' : 'sekrit', # Not used with sqlite3. - 'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT' : '', # Set to empty string for default. Not used with sqlite3. - } -} -DBMARIADB = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - 'NAME' : 'troggle', # Or path to database file if using sqlite3. - 'USER' : 'expo', - 'PASSWORD' : 'my-secret-password-schwatzmooskogel', - 'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT' : '', # Set to empty string for default. Not used with sqlite3. - } -} - -# default database for me is squlite -DBSWITCH = "sqlite" - -if DBSWITCH == "sqlite": - DATABASES = DBSQLITE -if DBSWITCH == "mariadb": - DATABASES = DBMARIADB - -NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"] - -PYTHON_PATH = REPOS_ROOT_PATH / 'troggle' -sys.path.append(os.fspath(REPOS_ROOT_PATH)) -sys.path.append(os.fspath(PYTHON_PATH)) - -LOGFILE = PYTHON_PATH / 'troggle.log' -PYTHON_PATH = os.fspath(PYTHON_PATH) - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - TEMPLATE_PATH - ], - 'OPTIONS': { - 'debug': 'DEBUG', - 'context_processors': [ - # django.template.context_processors.csrf, # is always enabled and cannot be removed, sets csrf_token - 'django.contrib.auth.context_processors.auth', # knowledge of logged-on user & permissions - 'core.context.troggle_context', # in core/troggle.py - only used in expedition.html - 'django.template.context_processors.debug', - 'django.template.context_processors.i18n', - 'django.template.context_processors.media', # includes a variable MEDIA_URL - 'django.template.context_processors.static', # includes a variable STATIC_URL used by admin pages - 'django.template.context_processors.tz', - 'django.template.context_processors.request', # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar. - 'django.contrib.messages.context_processors.messages', - ], - 'loaders': [ - 'django.template.loaders.filesystem.Loader', # default lcation is troggle/templates/ - 'django.template.loaders.app_directories.Loader', # needed for admin 'app' - ] - }, - }, -] - -EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py" -EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py" - -EXPOUSER = 'expo' -EXPOUSER_EMAIL = 'philip.sargent@gmail.com' -EXPOADMINUSER = 'expoadmin' -EXPOADMINUSER_EMAIL = 'philip.sargent@gmail.com' - -EMAIL_HOST = "smtp-auth.mythic-beasts.com" -EMAIL_HOST_USER = "django-test@klebos.net" # Philip Sargent really -EMAIL_PORT=587 -EMAIL_USE_TLS = True -DEFAULT_FROM_EMAIL = 'django-test@klebos.net' - -SURVEX_DATA = REPOS_ROOT_PATH / "loser" -DRAWINGS_DATA = REPOS_ROOT_PATH / "drawings" - -EXPOWEB = REPOS_ROOT_PATH / "expoweb" -#SURVEYS = REPOS_ROOT_PATH -CAVEDESCRIPTIONS = EXPOWEB / "cave_data" -ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data" -EXPOWEB_URL = '' -# SCANS_URL = '/survey_scans/' # defunct, removed. - -# Sanitise these to be strings as all other code is expecting strings -# and we have not made the change to pathlib Path type in the other localsettings-* variants yet. - - From 3ffb07371d2f6b586a0c87c66c284263bd32450f Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sat, 22 Jul 2023 22:26:50 +0300 Subject: [PATCH 10/17] warning message to people who have got here before they are ready, and link to docm --- templates/editcave.html | 49 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/templates/editcave.html b/templates/editcave.html index 45f752b..099eb42 100644 --- a/templates/editcave.html +++ b/templates/editcave.html @@ -1,10 +1,57 @@ {% extends "cavebase.html" %} -{% block title %}Edit Cave - {{cave.official_name|safe}} - {{cave.kataster_number}}{% endblock %} + +{% block title %} +{% if cave.official_name %} +Edit Cave - {{cave.official_name|safe}} - {{cave.kataster_number}} +{% else %} +New Cave Creation Form +{% endif %} +{% endblock %} + {% block extraheaders %} {% include 'html_editor_scripts_css.html' %} {% endblock %} {% block content %} +{% if cave.official_name %}

Edit Cave - {{cave.official_name|safe}} {{cave.reference}}

+

The fields on this form, two of which are vital, are documented explicitly on the page +cave entry fields +and these same fields appear in the alternative file upload process as described in +new cave file. +

As it says there, +"There are a lot of fields on the 'new/edit cave' page and the 'new/edit entrance' page and some of them are a quite mysterious. +This page explains what they mean." +{% else %} +

New Cave Creation

+
+ +

WARNING +

This form is the SIXTH step in +the expo online cave recording system. +

The whole process starts at step ONE here: Great, I have discovered a new cave.... +

But even if you are happy and confident that you have understood steps 1 to 5, please go through the recommended procedure +of editing an existing cave entry first, before trying to create a totally new one. +This is described under the section "Recommended procedure" on the page CaveEntry +especially sub-heading "Edit Cave form" and please go through the example edit process for caves +Plateauhöhle 171 and Plateauhöhle 172, as described there. + +

+If you have not done all the preceeding steps, cancel this form and go back and do them now. +Click here. + + +

This form contains many esoteric and non-obvious data requirements, which will give you a +frustrating time if you try to persist without being familiar with how all the bits fit together. +

The fields on this form, two of which are vital, are documented explicitly on the page +cave entry fields +and these same fields appear in the alternative file upload process as described in +new cave file. +

As it says there, +"There are a lot of fields on the 'new/edit cave' page and the 'new/edit entrance' page and some of them are a quite mysterious. +This page explains what they mean." + +

+{% endif %} {% include 'html_editor_pop_ups.html' %}

{{message}}

{% csrf_token %} From 3ea014ec577fef12b946c6120b326e8471009d01 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sun, 23 Jul 2023 22:01:01 +0300 Subject: [PATCH 11/17] add new *team role --- parsers/survex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsers/survex.py b/parsers/survex.py index b3db9e7..29e3fff 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -271,7 +271,7 @@ class LoadingSurvex: rx_names = re.compile(r"(?i)names") rx_flagsnot = re.compile(r"not\s") rx_linelen = re.compile(r"[\d\-+.]+$") - instruments = "(bitch|bodger|bolt|bolter|bolting|book|clino|comp|compass|consultant|disto|distox|distox2|dog|dogsbody|drawing|drill|gps|helper|inst|instr|instrument|monkey|nagging|nail|nail_polish|nail_polish_bitch|nail_polish_monkey|nail_varnish|nail_varnish_bitch|note|paint|photo|pic|point|polish|powerdrill|rig|rigger|rigging|sketch|slacker|something|tape|topodroid|unknown|useless|varnish|waiting_patiently)" + instruments = "(bitch|bodger|bolt|bolter|bolting|book|clino|comp|compass|consultant|disto|distox|distox2|dog|dogsbody|drawing|drill|gps|helper|inst|instr|instrument|monkey|nagging|nail|nail_polish|nail_polish_bitch|nail_polish_monkey|nail_varnish|nail_varnish_bitch|note|paint|photo|pic|point|polish|powerdrill|rig|rigger|rigging|shoot|sketch|slacker|something|tape|topodroid|unknown|useless|varnish|waiting_patiently)" rx_teammem = re.compile(r"(?i)" + instruments + "?(?:es|s)?\s+(.*)$") rx_teamold = re.compile(r"(?i)(.*)\s+" + instruments + "?(?:es|s)?$") rx_teamabs = re.compile(r"(?i)^\s*(" + instruments + ")?(?:es|s)?\s*$") From 9473b22bd9aa71a23d23c4a0f6f9c88a3b7666f5 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sun, 23 Jul 2023 23:30:19 +0300 Subject: [PATCH 12/17] detect non-ISO dates in JSON and from user and fix --- core/models/wallets.py | 84 +++++++++++++++++++++++++------------- core/views/wallets_edit.py | 24 ++--------- parsers/imports.py | 2 +- parsers/scans.py | 1 + 4 files changed, 60 insertions(+), 51 deletions(-) diff --git a/core/models/wallets.py b/core/models/wallets.py index 9da5695..29ca9e2 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -15,6 +15,42 @@ from django.urls import reverse YEAR_RANGE = (1975, 2050) +def make_valid_date(date): + """Take whatever garbage some fool has typed in and try to make it into a valid ISO-format date + """ + datestr = date.replace(".", "-") + try: + samedate = datetime.date.fromisoformat(datestr) + except ValueError: + # Could be in std euro format e.g. 14/07/2023 + match = re.search(r'(\d{1,2})/(\d{1,2})/(\d{2,4})', datestr) + if match: + d = int(match.group(1)) + m = int(match.group(2)) + y = int(match.group(3)) + if y<2000: + y = y + 2000 + try: + samedate = datetime.date(y, m, d) + print(f"- - Warning, not in ISO format. '{datestr=}' but we coped: {samedate.isoformat()} ") + return samedate + except: + print(f"! - Fail, tried to decompose date in dd/mm/yyyy format but failed: {datestr=} ") + return None + # probably a single digit day number. HACKUS MAXIMUS. + datestr = datestr[:-1] + "0" + datestr[-1] + # datestr = f"{datestr:02d}" + print(f"! - ValueError, trying.. {datestr=} ") + try: + samedate = datetime.date.fromisoformat(datestr) + except: + try: + samedate = datetime.date.fromisoformat(datestr[:10]) + except: + print(f"! - ValueError, FAILED {datestr=} ") + samedate = None + return samedate + class Wallet(models.Model): """We do not keep the JSON values in the database, we query them afresh each time, but we will change this when we need to do a Django query on e.g. personame @@ -33,7 +69,9 @@ class Wallet(models.Model): def get_json(self): """Read the JSON file for the wallet and do stuff - Do it every time it is queried, to be sure the result is fresh""" + Do it every time it is queried, to be sure the result is fresh + + import DataIssue locally to prevent import cycle problem""" # jsonfile = Path(self.fpath, 'contents.json') # Get from git repo instead @@ -42,7 +80,7 @@ class Wallet(models.Model): fp = Path(self.fpath) wname = fp.name wyear = fp.parent.name - wurl = f"/walletedit/{self.walletname}" # .replace('#', ':') + wurl = f"/walletedit/{self.walletname}".replace('#', ':') if len(wyear) != 4 or len(wname) !=6: # no contents.json for old-style wallets @@ -52,8 +90,10 @@ class Wallet(models.Model): jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json" if not Path(jsonfile).is_file(): - print(f'{jsonfile} is not a file {wyear=} {wname=} ') - # Should this be a DataIssue? + message = f"! {jsonfile} is not a file {wyear=} {wname=} " + from troggle.core.models.troggle import DataIssue + print(message) + DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) return None else: with open(jsonfile) as json_f: @@ -62,33 +102,19 @@ class Wallet(models.Model): except: message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file" print(message) - # Should this be a DataIssue? + DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) return None + if waldata["date"]: - datestr = waldata["date"].replace(".", "-") - try: - thisdate = datetime.date.fromisoformat(datestr) - except ValueError: - # probably a single digit day number. HACKUS MAXIMUS. - # clearly we need to fix this when we first import date strings.. - datestr = datestr[:-1] + "0" + datestr[-1] - print(f" - {datestr=} ") - try: - thisdate = datetime.date.fromisoformat(datestr) - self.walletdate = thisdate - self.save() - try: - waldata["date"] = thisdate.isoformat() - except: - message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file" - from troggle.core.models.troggle import DataIssue - - DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl) - except: - message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file" - from troggle.core.models.troggle import DataIssue - - DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl) + thisdate = make_valid_date(waldata["date"]) + if thisdate: + self.walletdate = thisdate + self.save() + waldata["date"] = thisdate.isoformat() + else: + message = f"! {str(self.walletname)} Date format not ISO {waldata['date']}. Failed to load from {jsonfile} JSON file" + from troggle.core.models.troggle import DataIssue + DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) return waldata def year(self): diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py index 7cf2251..9273fed 100644 --- a/core/views/wallets_edit.py +++ b/core/views/wallets_edit.py @@ -18,7 +18,7 @@ from troggle.core.models.caves import Cave from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole from troggle.core.models.troggle import DataIssue, Expedition -from troggle.core.models.wallets import Wallet, YEAR_RANGE +from troggle.core.models.wallets import Wallet, YEAR_RANGE, make_valid_date from troggle.core.views.auth import login_required_if_public from troggle.core.views.caves import getCave @@ -32,7 +32,7 @@ from troggle.parsers.scans import contentsjson """ todo = """ -- Nasty bug in navingating to 'previous wallet' when we have a 2-year gap in expos +- Nasty bug in navigating to 'previous wallet' when we have a 2-year gap in expos The xxxx#00 wallet is not getting edited correctly. Something is off by one somewhere.. - Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import @@ -496,25 +496,7 @@ def walletedit(request, path=None): or thing == "[]" or thing is None) - def make_valid_date(date): - datestr = date.replace(".", "-") - try: - samedate = datetime.date.fromisoformat(datestr) - except ValueError: - # probably a single digit day number. HACKUS MAXIMUS. - # clearly we need to fix this when we first import date strings.. - datestr = datestr[:-1] + "0" + datestr[-1] - # datestr = f"{datestr:02d}" - print(f"! - ValueError, trying.. {datestr=} ") - try: - samedate = datetime.date.fromisoformat(datestr) - except: - try: - samedate = datetime.date.fromisoformat(datestr[:10]) - except: - print(f"! - ValueError, FAILED {datestr=} ") - samedate = None - return samedate + def scan_survexblocks(svxfile): """Scans for *team people attached to all the survex blocks in this svxfile diff --git a/parsers/imports.py b/parsers/imports.py index 50b45cf..9191194 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -27,7 +27,7 @@ def import_people(): troggle.parsers.people.load_people_expos() def import_surveyscans(): - print("-- Importing Survey Scans") + print("-- Importing Survey Scans and Wallets") with transaction.atomic(): troggle.parsers.scans.load_all_scans() diff --git a/parsers/scans.py b/parsers/scans.py index 3f042da..9929209 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -38,6 +38,7 @@ def load_all_scans(): Wallet.objects.all().delete() print(" - deleting all Wallet and SingleScan objects") DataIssue.objects.filter(parser="scans").delete() + DataIssue.objects.filter(parser="wallets").delete() # These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet. valids = [ From 9f2b77bf1dbf09fcfa88fcfd5e5863b3e83f890c Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 24 Jul 2023 13:14:42 +0300 Subject: [PATCH 13/17] More compact parsing report to terminal --- parsers/logbooks.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/parsers/logbooks.py b/parsers/logbooks.py index e701183..8d2f969 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -57,7 +57,7 @@ LOGBOOK_PARSER_SETTINGS = { LOGBOOKS_DIR = "years" # subfolder of settings.EXPOWEB ENTRIES = { - "2023": 27, + "2023": 32, "2022": 90, "2019": 55, "2018": 95, @@ -305,8 +305,8 @@ def parser_html(year, expedition, txt, seq=""): endmatch = re.match(r"(?i)(?s).*([\s\S]*?)(?= 0: + print(f"\n - {year} endpara:\n'{endpara}'") endpath = Path(settings.EXPOWEB, LOGBOOKS_DIR, year, "endmatter.html") with open(endpath, "w") as end: end.write(endpara + "\n") @@ -560,7 +560,8 @@ def parse_logbook_for_expedition(expedition, blog=False): if logbook_parseable: # -------------------- parser = globals()[parsefunc] - print(f" - {year} parsing with {parsefunc} - {lb}") + # print(f" - {year} parsing with {parsefunc} - {lb}") + print(" .", end="") logentries = parser(year, expedition, txt, sq) # this launches the right parser # -------------------- @@ -665,7 +666,7 @@ def LoadLogbooks(): logentries = parse_logbook_for_expedition(b, blog=True) # loads the blog logbook for one expo allentries += logentries - print(f"total {len(allentries):,} log entries parsed in all expeditions") + print(f"\ntotal {len(allentries):,} log entries parsed in all expeditions") mem = get_process_memory() print(f" - MEM: {mem:7.2f} MB in use, {mem-mem1:7.2f} MB more", file=sys.stderr) duration = time.time() - start @@ -682,6 +683,7 @@ def LoadLogbooks(): for expo in expos: expo.save() # to save logbook name property mem = get_process_memory() + print(f"total {len(allentries):,} log entries saved into database") print(f" - MEM: {mem:7.2f} MB in use, {mem-mem1:7.2f} MB more", file=sys.stderr) duration = time.time() - start print(f" - TIME: {duration:7.2f} s", file=sys.stderr) From 64fa602a070900f643142453d51d5f8f6282fb8d Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 24 Jul 2023 14:24:39 +0300 Subject: [PATCH 14/17] ignore /subsections/ files --- parsers/survex.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/parsers/survex.py b/parsers/survex.py index 29e3fff..8ba7866 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1550,9 +1550,9 @@ class LoadingSurvex: nonlocal blockcount blockcount += 1 - if blockcount % 20 == 0: + if blockcount % 40 == 0: print(".", file=sys.stderr, end="") - if blockcount % 800 == 0: + if blockcount % 1600 == 0: print("\n", file=sys.stderr, end="") mem = get_process_memory() print(f" - MEM: {mem:7.2f} MB in use", file=sys.stderr) @@ -2186,7 +2186,7 @@ def FindAndLoadSurvex(survexblockroot): ) unseensroot = re.sub(r"\.svx$", "", UNSEENS) - excpts = ["surface/terrain", "kataster/kataster-boundaries", "template", "docs", unseensroot] + excpts = ["surface/terrain", "kataster/kataster-boundaries", "template", "docs", "deprecated", "subsections", unseensroot] removals = [] for x in unseens: for o in excpts: @@ -2202,7 +2202,8 @@ def FindAndLoadSurvex(survexblockroot): file=sys.stderr, ) check_team_cache() - print(" -- Now loading the previously-omitted survex files.", file=sys.stderr) + print(f" -- Now loading the previously-omitted survex files.", file=sys.stderr) + print(f" - (except the exceptions: {excpts})", file=sys.stderr) with open(Path(settings.SURVEX_DATA, UNSEENS), "w") as u: u.write( @@ -2498,7 +2499,11 @@ def MakeFileRoot(fn): print(f" - Making/finding a new root survexfile for this import: {fn}") fileroot = SurvexFile(path=fn, cave=cave) - fileroot.survexdirectory = SurvexDirectory.objects.get(id=1) # default + try: + fileroot.survexdirectory = SurvexDirectory.objects.get(id=1) # default + except: + fileroot.survexdirectory = None + if cave: # But setting the SurvexDirectory does work ! From 3bd7684d4b41e72a336f7ae0e738f7c3ea9ff522 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 24 Jul 2023 14:24:53 +0300 Subject: [PATCH 15/17] formatting terminal o/p --- parsers/logbooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 8d2f969..84d0f5a 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -666,7 +666,7 @@ def LoadLogbooks(): logentries = parse_logbook_for_expedition(b, blog=True) # loads the blog logbook for one expo allentries += logentries - print(f"\ntotal {len(allentries):,} log entries parsed in all expeditions") + print(f"\n - {len(allentries):,} log entries parsed in all expeditions") mem = get_process_memory() print(f" - MEM: {mem:7.2f} MB in use, {mem-mem1:7.2f} MB more", file=sys.stderr) duration = time.time() - start @@ -683,7 +683,7 @@ def LoadLogbooks(): for expo in expos: expo.save() # to save logbook name property mem = get_process_memory() - print(f"total {len(allentries):,} log entries saved into database") + print(f" - {len(allentries):,} log entries saved into database") print(f" - MEM: {mem:7.2f} MB in use, {mem-mem1:7.2f} MB more", file=sys.stderr) duration = time.time() - start print(f" - TIME: {duration:7.2f} s", file=sys.stderr) From c31615b1ff48273e0e2a03139cd8eff33fbf788e Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 24 Jul 2023 15:26:36 +0300 Subject: [PATCH 16/17] clean up after Mark changed things a bit --- parsers/survex.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/parsers/survex.py b/parsers/survex.py index 8ba7866..2be2217 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -435,23 +435,34 @@ class LoadingSurvex: self.currentdate = self.inheritdate # unecessary duplication # Not an error, so not put in DataIssues, but is printed to debug output message = ( - f"- No *date. INHERITING date from ({survexblock.parent})-{survexblock.parent.survexfile.path} to ({survexblock})-{survexblock.survexfile.path} {self.inheritdate:%Y-%m-%d}" + f"- No *date. INHERITING date '{self.inheritdate:%Y-%m-%d}' from ({survexblock.parent})-{survexblock.parent.survexfile.path} to ({survexblock})-{survexblock.survexfile.path} {self.inheritdate:%Y-%m-%d}" ) print(self.insp + message) # stash_data_issue( # parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) # child # ) if survexblock.survexfile != survexblock.parent.survexfile: - # This is noteworthy, however. - message = ( - f"- Warning *date INHERITED from DIFFERENT file:\n ({survexblock.parent})-{survexblock.parent.survexfile.path} to ({survexblock})-{survexblock.survexfile.path} {self.inheritdate:%Y-%m-%d}\n {self.stackbegin} {self.inheritdate:%Y-%m-%d}" - ) - print(self.insp + message) - stash_data_issue( - parser="survex", message=message, url=None, sb=(survexblock.parent.survexfile.path) # PARENT - ) - - return self.inheritdate + # This is noteworthy, however. + + if survexblock.parent.name == "rootblock": + # Not a sensible thing to inherit a date from, even if a date exists, which it shouldn't... + message = ( + f"- No *date. But not sensible to inherit from rootblock. From ({survexblock.parent})-{survexblock.parent.survexfile.path} to ({survexblock})-{survexblock.survexfile.path} {self.inheritdate:%Y-%m-%d}" + ) + print(self.insp + message) + # stash_data_issue( + # parser="survex", message=message, url=None, sb=(survexblock.survexfile.path) + # ) + return + else: + message = ( + f"- Warning *date '{self.inheritdate:%Y-%m-%d}' INHERITED from DIFFERENT file:\n ({survexblock.parent})-{survexblock.parent.survexfile.path} to ({survexblock})-{survexblock.survexfile.path} {self.inheritdate:%Y-%m-%d}\n {self.stackbegin} {self.inheritdate:%Y-%m-%d}" + ) + print(self.insp + message) + stash_data_issue( + parser="survex", message=message, url=None, sb=(survexblock.parent.survexfile.path) # PARENT + ) + return self.inheritdate else: # This is not an error in the Expo dataset. # Many files just holding *include lines do not have dates. @@ -2203,7 +2214,7 @@ def FindAndLoadSurvex(survexblockroot): ) check_team_cache() print(f" -- Now loading the previously-omitted survex files.", file=sys.stderr) - print(f" - (except the exceptions: {excpts})", file=sys.stderr) + print(f" - (except: {excpts})", file=sys.stderr) with open(Path(settings.SURVEX_DATA, UNSEENS), "w") as u: u.write( From 31c815eeb02cce67c5c5586c7a2573c9a3c93eff Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 24 Jul 2023 17:33:39 +0300 Subject: [PATCH 17/17] bugfix in error message --- parsers/caves.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsers/caves.py b/parsers/caves.py index 668ac40..f8b39e9 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -483,7 +483,7 @@ def read_cave(filename, cave=None): cave=cave, entrance_letter=letter, entrance=entrance ) except: - message = f' ! Entrance setting failure, slug:"{slug}" #entrances:{len(entrances)} {entrance} letter:"{letter}" cave:"{cave}" filename:"cave_data/{filename}"' + message = f' ! Entrance setting failure, slug:"{slug}" #entrances:{len(entrances)} {e} letter:"{letter}" cave:"{cave}" filename:"cave_data/{filename}"' DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.url}_edit/") print(message) def reload_entrances():