From 8e982bd6bee63d89c286b7d393bfb2b8536e078d Mon Sep 17 00:00:00 2001 From: "Philip Sargent (laptop)" Date: Tue, 2 May 2023 14:27:55 +0300 Subject: [PATCH 1/4] Fixing new dev machine setup --- README.txt | 6 +- _deploy/wsl/localsettingsWSL.py | 5 +- deprecations.txt | 2 +- export/os-trog.sh | 70 +++++++++++++ export/venv-trog.sh | 172 ++++++++++++++++++++++++++++++++ requirements-p10d3.txt | 0 requirements-p11d32.txt | 0 requirements-p11d4.txt | 16 +++ requirements.txt | 4 +- venv-trog.sh | 25 ++--- 10 files changed, 277 insertions(+), 23 deletions(-) create mode 100644 export/os-trog.sh create mode 100644 export/venv-trog.sh delete mode 100644 requirements-p10d3.txt delete mode 100644 requirements-p11d32.txt create mode 100644 requirements-p11d4.txt diff --git a/README.txt b/README.txt index 32c858a..41cde78 100644 --- a/README.txt +++ b/README.txt @@ -1,4 +1,4 @@ -Updated 3 February 2023 +Updated 2 May 2023 Troggle is an application for caving expedition data management, originally created for use on Cambridge University Caving Club (CUCC)expeditions @@ -21,7 +21,9 @@ http://expo.survex.com/handbook/computing/onlinesystems.html Troggle setup ============= -0. read the very extensive online documentation and stop reading this README. +0. read the very extensive online documentation and stop reading this README... +well, come back to this README after you have read the HTML pages. Not everything has been transferred. + http://expo.survex.com/handbook/troggle/troglaptop.html http://expo.survex.com/handbook/troggle/serverconfig.html http://expo.survex.com/handbook/troggle/trogdangoup.html diff --git a/_deploy/wsl/localsettingsWSL.py b/_deploy/wsl/localsettingsWSL.py index 8308c6e..42a2a1c 100644 --- a/_deploy/wsl/localsettingsWSL.py +++ b/_deploy/wsl/localsettingsWSL.py @@ -52,11 +52,10 @@ TEMPLATE_PATH = TROGGLE_PATH / "templates" MEDIA_ROOT = TROGGLE_PATH / "media" JSLIB_ROOT = TROGGLE_PATH / "media" / "jslib" # used for CaveViewer JS utility -# FILES = Path('/mnt/d/expofiles/') EXPOFILES = REPOS_ROOT_PATH / "expofiles" SCANS_ROOT = EXPOFILES / "surveyscans" -PHOTOS_ROOT = EXPOFILES / "photos" +PHOTOS_ROOT = Path("/mnt/d/EXPO/PHOTOS") PHOTOS_YEAR = "2023" NOTABLECAVESHREFS = ["290", "291", "264", "258", "204", "359", "76", "107"] @@ -95,7 +94,7 @@ SURVEXPORT = "survexport" # for parsing .3d files and producing .pos files DBSQLITE = { "default": { "ENGINE": "django.db.backends.sqlite3", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - "NAME": "/home/philip/p11d5/troggle.sqlite", + "NAME": "/home/philip/p11d4/troggle.sqlite", # 'NAME' : ':memory:', "USER": "expo", # Not used with sqlite3. "PASSWORD": "sekrit", # Not used with sqlite3. diff --git a/deprecations.txt b/deprecations.txt index 4da520e..638aa68 100644 --- a/deprecations.txt +++ b/deprecations.txt @@ -1,2 +1,2 @@ -/home/philip/p11d3/lib/python3.11/site-packages/django/conf/__init__.py:267: RemovedInDjango50Warning: The USE_L10N setting is deprecated. Starting with Django 5.0, localized formatting of data will always be enabled. For example Django will display numbers and dates using the format of the current locale. +/home/philip/p11d4/lib/python3.11/site-packages/django/conf/__init__.py:267: RemovedInDjango50Warning: The USE_L10N setting is deprecated. Starting with Django 5.0, localized formatting of data will always be enabled. For example Django will display numbers and dates using the format of the current locale. warnings.warn(USE_L10N_DEPRECATED_MSG, RemovedInDjango50Warning) diff --git a/export/os-trog.sh b/export/os-trog.sh new file mode 100644 index 0000000..c6ff0e0 --- /dev/null +++ b/export/os-trog.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Run this in a terminal in the troggle directory: 'bash os-trog.sh' +# On WSL, do Shift-click in the file explorer on the troggle folder to open a Linux command line +# 'Open Linux shell here' +echo 'Run this in a terminal in the troggle directory: "bash venv-trog.sh"' +cat /etc/os-release +# Expects an Ubuntu 22.04 relatively clean install. + +#sudo apt install python-is-python3 -y +python --version : ensure python is an alias for python3 not python2.7 +sudo apt update -y +sudo apt dist-upgrade -y +sudo apt autoremove -y +sudo apt install sqlite3 -y +sudo apt install python3-pip -y + +# this installs a shed-load of other stuff: binutils etc.sudo apt install survex-aven +sudo apt install git openssh-client -y +# On a clean debian 11 (bullseye) installation with Xfce & ssh, + + #on ubuntu 20.04: + #Package sftp is not available, but is referred to by another package. + #This may mean that the package is missing, has been obsoleted, or + #is only available from another source + #E: Package 'sftp' has no installation candidate + + +# On Ubuntu 20.04, with python10, the pip install fails. +# So you need to get the pip from source +# sudo curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 +# but really you should be using 22.04 +# and also, isf using debian, +# sudo python3.10 -m pip install -U virtualenv + +# as debian does not install everything that ubuntu does, you need: +# sudo useradd expo +# sudo usermod -a -G sudo expo # to put expo in sudoers group, re-login required + +# default since 22.04 +# sudo apt install python3.10 +# sudo apt install python3.10-venv -y +# sudo apt install python3.10-dev -y +# sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 + +# sudo apt install mariadb-server -y +# sudo apt install libmariadb-dev -y + +sudo python -m pip install --upgrade pip + +sudo apt install sftp -y + + +# Go to https://expo.survex.com/handbook/troggle/troglaptop.html#dbtools +# sudo service mysql start + +echo '### git config --global user.email "you@example.com" +echo '### git config --global user.name "Your Name" + +echo '###' +echo '### Currently set version of python' +python --version + +echo '###' +echo '### Now YOU have to configure the git settings for YOURSELF (not "expo")' + +echo '###' +echo '### NOW INSTALLING tunnel and therion, go and have a cup of tea. Or a 3-course meal.' +echo '###' +sudo apt install tunnelx therion -y + diff --git a/export/venv-trog.sh b/export/venv-trog.sh new file mode 100644 index 0000000..a47ab87 --- /dev/null +++ b/export/venv-trog.sh @@ -0,0 +1,172 @@ +#!/bin/bash +# footled lots to make this work with python 3.10 & 3.11 and WSL1 and WSL2 on Ubuntu 22.04 +# Run this in a terminal in the troggle directory: 'bash venv-trog.sh' +echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.sh"' + +# Expects an Ubuntu 22.04 relatively clean install. +# If you have not already installed these on your clean Ubuntu install DO THIS FIRST +# use the script os-trog.sh + +# If you are using Debian, then stick with the default version of python +# If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.11 + + +# NOW we set up troggle +PYTHON=python3.11 +VENAME=p11d3 # python3.x and django 3.2 +echo "** You are logged in as `id -u -n`" +echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/" +echo "The 150MB venv will created in /home/`id -u -n`/$VENAME/" +TROGDIR=$(cd $(dirname $0) && pwd) +echo "-- Troggle folder (this script location): ${TROGDIR}" + +if [ -d requirements.txt ]; then + echo "-- No requirements.txt found. Copy it from your most recent installation." + exit 1 +fi +echo ## Using requirements.txt : +cat requirements.txt +echo ## + + +$PYTHON --version + +# NOTE that when using a later or earlier verison of python, you MUST also +# use the allowed version of Pillow, see https://pillow.readthedocs.io/en/latest/installation.html + +# NOW set up link from expo user folder +# needed for WSL2 +echo Creating links from Linux filesystem user +# These links only need making once, for many venv +cd ~ + +if [ ! -d $VENAME ]; then + echo "## Creating venv $VENAME. (If this fails with a pip error, you need to ensure you have python3.11-venv installed and/or use a Ubuntu window)" + $PYTHON -m venv $VENAME +else + echo "## /$VENAME/ already exists ! Delete it first." + exit 1 +fi + + # Activate the virtual env and see what the default packages are +echo "### Activating $VENAME" + +cd $VENAME +echo "-- now in: ${PWD}" +source bin/activate +echo "### Activated." + # update local version of pip, more recent than OS version + # debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3 + + # update local version of setuptools, more recent than OS version, needed for packages without wheels + +echo "### installing later version of pip inside $VENAME" +$PYTHON -m pip install --upgrade pip +$PYTHON -m pip install --upgrade setuptools + +PIP=pip + +$PIP list > original-pip.list +$PIP freeze >original.txt + +# we are in /home/$USER/$VENAME/ +ln -s ${TROGDIR} troggle +ln -s ${TROGDIR}/../expoweb expoweb +ln -s ${TROGDIR}/../loser loser +ln -s ${TROGDIR}/../drawings drawings + +# fudge for philip's machine +if [ -d ${TROGDIR}/../expofiles ]; then + ln -s ${TROGDIR}/../expofiles expofiles +else + if [ ! -d /mnt/f/expofiles ]; then + sudo mkdir /mnt/f + sudo mount -t drvfs F: /mnt/f + else + ln -s /mnt/f/expofiles expofiles + fi +fi + +echo "### Setting file permissions.. may take a while.." +git config --global --add safe.directory '*' +sudo chmod -r 0777 * + +echo "### links to expoweb, troggle etc. complete:" +ls -tla +echo "###" +echo "### now installing ${TROGDIR}/requirements.txt" +echo "###" + +# NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE +# seen on wsl2 as well as wsl1 +# which ALSO ruins EXISTING permissions ! +# Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04 + +$PIP install -r ${TROGDIR}/requirements.txt +echo '### install from requirements.txt completed.' +echo '### ' + +$PIP freeze > requirements.txt +# so that we can track requirements more easily with git +# because we do not install these with pip, but they are listed by the freeze command +# Now find out what we actually installed by subtracting the stuff venv installed anyway +sort original.txt > 1 +sort requirements.txt >2 +comm -3 1 2 --check-order | awk '{ print $1}'>fresh-requirements.txt +rm 1 +rm 2 + +cp requirements.txt requirements-$VENAME.txt +cp requirements-$VENAME.txt troggle/requirements-$VENAME.txt + +$PIP list > installed-pip.list +$PIP list -o > installed-pip-o.list + +REQ=installation-record +mkdir $REQ +mv requirements-$VENAME.txt $REQ +mv original.txt $REQ +mv requirements.txt $REQ +mv original-pip.list $REQ +mv installed-pip.list $REQ +mv installed-pip-o.list $REQ +cp fresh-requirements.txt ../requirements.txt +mv fresh-requirements.txt $REQ +cp troggle/`basename "$0"` $REQ + + +$PYTHON --version +python --version +echo "Django version:`django-admin --version`" + +echo "### Now do + '[sudo service mysql start]' + '[sudo service mariadb restart]' + '[sudo mysql_secure_installation]' + 'cd ~/$VENAME' + 'source bin/activate' + 'cd troggle' + 'django-admin' + 'python manage.py check' +## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html +## you need to follow the Linux instructions. + 'ssh expo@expo.survex.com' + +## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine +## the tests may ALSO fail because of ssh and permissions errors +# Ran 85 tests in 83.492s +# FAILED (failures=5) +## So you will need to run +$sudo chown -Rhv philip:philip ~/$VENAME (if your username is philip) +# and then REBOOT (or at least, exit WSL and terminate and restart WSL) +# because this chmod only takes effect then. + + 'python manage.py test -v 2' + './pre-run.sh' (runs the tests again) + + 'python databaseReset.py reset $VENAME' + 'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)' + " +if [ ! -d /mnt/f/expofiles ]; then +echo '### No valid expofiles directory. Fix this before any tests will work. +fi \ No newline at end of file diff --git a/requirements-p10d3.txt b/requirements-p10d3.txt deleted file mode 100644 index e69de29..0000000 diff --git a/requirements-p11d32.txt b/requirements-p11d32.txt deleted file mode 100644 index e69de29..0000000 diff --git a/requirements-p11d4.txt b/requirements-p11d4.txt new file mode 100644 index 0000000..4f24dad --- /dev/null +++ b/requirements-p11d4.txt @@ -0,0 +1,16 @@ +asgiref==3.6.0 +black==23.1.0 +click==8.1.3 +coverage==7.1.0 +Django==4.2 +docutils==0.19 +isort==5.12.0 +mypy-extensions==1.0.0 +packaging==23.0 +pathspec==0.11.0 +Pillow==9.4.0 +platformdirs==3.0.0 +pytz==2022.7 +ruff==0.0.245 +sqlparse==0.4.3 +Unidecode==1.3.6 diff --git a/requirements.txt b/requirements.txt index c304f0f..4f24dad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ asgiref==3.6.0 black==23.1.0 click==8.1.3 coverage==7.1.0 -Django==4.2b1 +Django==4.2 docutils==0.19 isort==5.12.0 mypy-extensions==1.0.0 @@ -14,5 +14,3 @@ pytz==2022.7 ruff==0.0.245 sqlparse==0.4.3 Unidecode==1.3.6 -docutils==0.19 - diff --git a/venv-trog.sh b/venv-trog.sh index a47ab87..2767570 100644 --- a/venv-trog.sh +++ b/venv-trog.sh @@ -3,17 +3,16 @@ # Run this in a terminal in the troggle directory: 'bash venv-trog.sh' echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.sh"' -# Expects an Ubuntu 22.04 relatively clean install. +# Expects an Ubuntu 22.04 (or 20.04) relatively clean install. # If you have not already installed these on your clean Ubuntu install DO THIS FIRST # use the script os-trog.sh # If you are using Debian, then stick with the default version of python # If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.11 - # NOW we set up troggle PYTHON=python3.11 -VENAME=p11d3 # python3.x and django 3.2 +VENAME=p11d4 # python3.x and django 4.2 echo "** You are logged in as `id -u -n`" echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/" echo "The 150MB venv will created in /home/`id -u -n`/$VENAME/" @@ -21,14 +20,13 @@ TROGDIR=$(cd $(dirname $0) && pwd) echo "-- Troggle folder (this script location): ${TROGDIR}" if [ -d requirements.txt ]; then - echo "-- No requirements.txt found. Copy it from your most recent installation." + echo "-- No requirements.txt found. You should be in the /troggle/ folder. Copy it from your most recent installation." exit 1 fi echo ## Using requirements.txt : cat requirements.txt echo ## - $PYTHON --version # NOTE that when using a later or earlier verison of python, you MUST also @@ -79,17 +77,17 @@ ln -s ${TROGDIR}/../drawings drawings if [ -d ${TROGDIR}/../expofiles ]; then ln -s ${TROGDIR}/../expofiles expofiles else - if [ ! -d /mnt/f/expofiles ]; then - sudo mkdir /mnt/f - sudo mount -t drvfs F: /mnt/f + if [ ! -d /mnt/d/expofiles ]; then + sudo mkdir /mnt/d + sudo mount -t drvfs D: /mnt/d else - ln -s /mnt/f/expofiles expofiles + ln -s /mnt/d/expofiles expofiles fi fi echo "### Setting file permissions.. may take a while.." git config --global --add safe.directory '*' -sudo chmod -r 0777 * +sudo chmod -R 777 * echo "### links to expoweb, troggle etc. complete:" ls -tla @@ -154,8 +152,7 @@ echo "### Now do ## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine ## the tests may ALSO fail because of ssh and permissions errors -# Ran 85 tests in 83.492s -# FAILED (failures=5) + ## So you will need to run $sudo chown -Rhv philip:philip ~/$VENAME (if your username is philip) # and then REBOOT (or at least, exit WSL and terminate and restart WSL) @@ -167,6 +164,6 @@ $sudo chown -Rhv philip:philip ~/$VENAME (if your username is philip) 'python databaseReset.py reset $VENAME' 'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)' " -if [ ! -d /mnt/f/expofiles ]; then -echo '### No valid expofiles directory. Fix this before any tests will work. +if [ ! -d /mnt/d/expofiles ]; then +echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work. fi \ No newline at end of file From 175307773b9f87da8507c70bc70fa586f73abd57 Mon Sep 17 00:00:00 2001 From: "Philip Sargent (laptop)" Date: Tue, 2 May 2023 15:28:28 +0300 Subject: [PATCH 2/4] More fixes for laptop --- core/TESTS/test_imports.py | 2 +- core/TESTS/test_logins.py | 2 +- core/TESTS/test_parsers.py | 3 +-- core/TESTS/tests.py | 3 +-- venv-trog.sh | 13 +++++++------ 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/core/TESTS/test_imports.py b/core/TESTS/test_imports.py index 28bd65e..2b23dd4 100644 --- a/core/TESTS/test_imports.py +++ b/core/TESTS/test_imports.py @@ -8,7 +8,7 @@ The simple redirections to files which exist, e.g. in /expoweb/ /expofiles/ /expofiles/documents/ -etc. will test fine. +etc. using parameters in localsettings such as PHOTOS_ROOT will test fine. But paths like this: /survey_scans/ diff --git a/core/TESTS/test_logins.py b/core/TESTS/test_logins.py index ef9478b..8596205 100644 --- a/core/TESTS/test_logins.py +++ b/core/TESTS/test_logins.py @@ -103,7 +103,7 @@ class PostTests(TestCase): self.expedition = e def test_file_permissions(self): - """Expect to be allowed to write to expofiles + """Expect to be allowed to write to SCANS_ROOT, DRAWINGS_DATA, SURVEX_DATA, EXPOWEB Need to login first. """ c = self.client diff --git a/core/TESTS/test_parsers.py b/core/TESTS/test_parsers.py index 70f19cd..00b565e 100644 --- a/core/TESTS/test_parsers.py +++ b/core/TESTS/test_parsers.py @@ -6,8 +6,7 @@ so any path that relies on data being in the database will fail. The simple redirections to files which exist, e.g. in /expoweb/ -/expofiles/ -/expofiles/documents/ +/photos/ etc. will test fine. But paths like this: diff --git a/core/TESTS/tests.py b/core/TESTS/tests.py index 078a048..00a5021 100644 --- a/core/TESTS/tests.py +++ b/core/TESTS/tests.py @@ -6,8 +6,7 @@ so any path that relies on data being in the database will fail. The simple redirections to files which exist, e.g. in /expoweb/ -/expofiles/ -/expofiles/documents/ +/photos/ etc. will test fine. But paths like this: diff --git a/venv-trog.sh b/venv-trog.sh index 2767570..42fac41 100644 --- a/venv-trog.sh +++ b/venv-trog.sh @@ -72,17 +72,18 @@ ln -s ${TROGDIR} troggle ln -s ${TROGDIR}/../expoweb expoweb ln -s ${TROGDIR}/../loser loser ln -s ${TROGDIR}/../drawings drawings +#ln -s ${TROGDIR}/../expofiles expofiles # fudge for philip's machine +if [ ! -d /mnt/d/EXPO ]; then + sudo mkdir /mnt/d + sudo mount -t drvfs D: /mnt/d +fi + if [ -d ${TROGDIR}/../expofiles ]; then ln -s ${TROGDIR}/../expofiles expofiles else - if [ ! -d /mnt/d/expofiles ]; then - sudo mkdir /mnt/d - sudo mount -t drvfs D: /mnt/d - else - ln -s /mnt/d/expofiles expofiles - fi + ln -s /mnt/d/EXPO/expofiles expofiles fi echo "### Setting file permissions.. may take a while.." From ef467ad481eb5ebf120c6806a18529d1981696f6 Mon Sep 17 00:00:00 2001 From: "Philip Sargent (laptop)" Date: Tue, 2 May 2023 17:42:58 +0300 Subject: [PATCH 3/4] Making the dummy entrance thing clear --- core/forms.py | 2 +- core/views/caves.py | 9 ++++++--- parsers/caves.py | 35 ++++++++++++++++++++++++----------- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/core/forms.py b/core/forms.py index b68e259..a2ea45b 100644 --- a/core/forms.py +++ b/core/forms.py @@ -61,7 +61,7 @@ class CaveForm(ModelForm): survex_file = forms.CharField( required=False, label="Survex file eg. caves-1623/000/000.svx", widget=forms.TextInput(attrs={"size": "45"}) ) - url = forms.CharField(required=True, label="URL eg. 1623/000/000.html", widget=forms.TextInput(attrs={"size": "45"})) + url = forms.CharField(required=True, label="URL eg. 1623/000/000 (no .html)", widget=forms.TextInput(attrs={"size": "45"})) length = forms.CharField(required=False, label="Length (m)") depth = forms.CharField(required=False, label="Depth (m)") extent = forms.CharField(required=False, label="Extent (m)") diff --git a/core/views/caves.py b/core/views/caves.py index 1ad8dcf..b5a13f4 100644 --- a/core/views/caves.py +++ b/core/views/caves.py @@ -281,12 +281,15 @@ def cavepage(request, karea, subpath): accessed by kataster area number specifically OR accessed by cave.url specifically set in data, e.g. - "1623/000/000.html" <= cave-data/1623-000.html + "1623/000/000" <= cave-data/1623-000.html "1623/41/115.htm" <= cave-data/1623-115.html so we have to query the database to fine the URL as we cannot rely on the url actually telling us the cave by inspection. + + NOTE that old caves have ".html" (or ".htm") in the URL as they used to be actual files. But since 2006 these URLs + refer to virtual pages generated on the fly by troggle, so the".html" is confusing and redundant. - There are A LOT OF URLS to e.g. /1623/161/l/rl89a.htm which are IMAGES and html files - in cave descriptions. These need to be handled HERE + There are also A LOT OF URLS to e.g. /1623/161/l/rl89a.htm which are IMAGES and real html files + in cave descriptions. These need to be handled HERE too (accident of history). """ kpath = karea + subpath # print(f" ! cavepage:'{kpath}' kataster area:'{karea}' rest of path:'{subpath}'") diff --git a/parsers/caves.py b/parsers/caves.py index 8a611ec..06b77c9 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -64,25 +64,38 @@ def dummy_entrance(k, slug, msg="DUMMY"): message = f" ! {k:11s} {msg}-{slug} {k} entrance create failure" DataIssue.objects.create(parser="entrances", message=message, url=f"{slug}") print(message) - raise + raise # caught and handled by calling routine. def set_dummy_entrance(id, slug, cave, msg="DUMMY"): """Called only when reading the cave and entrance html files - Entrance field either missing or holds a null string instead of a filename in a cave_data file.""" + + Called when the Entrance field in a cave_data file is either missing or + holds a null string instead of a filename. + + Previously, the lack of an entrance where an entrance was expected, caused troggle to crash in several places. + But it is more robust now, so this is not necessary... we hope. + + Also, Cave and Entrance editing now expects there to be a real file (since April 2023), so creating this + dummy is actually harmful. So this is commented out, pending removal after further experience. + + global variable entrances_xslug is simply a cache of references to Entrance objects + to speed things up when parsing a lot of caves and entrances. All DB actions are time-consuming + so + """ global entrances_xslug + message = f" - Note: Missing Entrance for entrance '{id}' on cave '{cave}'" + DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.url}") try: - #Note the below line, just creates a dastabase entry, sets the global variable entrances_xslug and not a file, this then breaks entrance editing by the website. I am not sure if we want entrances automagically created. Therefore I have commented it out. Perhaps entrances_xslug is being saved with the worng filename, breaking edit_caves. MJG - #entrance = dummy_entrance(id, slug, msg="DUMMY") - #entrances_xslug[slug] = entrance - #CaveAndEntrance.objects.update_or_create(cave=cave, entrance_letter="", entrance=entrance) - #message = f" - Note: Dummy Entrance successfully set for entrance {id} on cave {cave}" - message = f" - Note: Missing Entrance for entrance {id} on cave {cave}" - DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.url}") - # print(message) + # I am not sure if we want entrances automagically created. + # Therefore I have commented it out. MJG + + # entrance = dummy_entrance(id, slug, msg="DUMMY") + # entrances_xslug[slug] = entrance + # CaveAndEntrance.objects.update_or_create(cave=cave, entrance_letter="", entrance=entrance) + pass except: - # raise message = f' ! Entrance Dummy setting failure, slug:"{slug}" cave id :"{id}" ' DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.url}") print(message) From 7f0e7d8fa3e4ce324a17151d5f76e8d3fdfa262b Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Tue, 2 May 2023 23:26:53 +0300 Subject: [PATCH 4/4] Ubuntu 22.04 + python3.11 --- os-trog.sh | 14 +++++++------- requirements-p10d4.txt | 17 +++++++++++++++++ venv-trog.sh | 3 ++- 3 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 requirements-p10d4.txt diff --git a/os-trog.sh b/os-trog.sh index c6ff0e0..6d32ab7 100644 --- a/os-trog.sh +++ b/os-trog.sh @@ -6,24 +6,25 @@ echo 'Run this in a terminal in the troggle directory: "bash venv-trog.sh"' cat /etc/os-release # Expects an Ubuntu 22.04 relatively clean install. -#sudo apt install python-is-python3 -y +sudo apt install python-is-python3 -y python --version : ensure python is an alias for python3 not python2.7 sudo apt update -y sudo apt dist-upgrade -y sudo apt autoremove -y sudo apt install sqlite3 -y sudo apt install python3-pip -y +sudo apt install python3.10-venv -y +sudo apt install python3.11-venv python3.11-dev -y # this installs a shed-load of other stuff: binutils etc.sudo apt install survex-aven +sudo apt install openssh-server -y +# sudo apt install sftp -y sudo apt install git openssh-client -y # On a clean debian 11 (bullseye) installation with Xfce & ssh, - #on ubuntu 20.04: + #on ubuntu 20.04 and 22.04 #Package sftp is not available, but is referred to by another package. - #This may mean that the package is missing, has been obsoleted, or - #is only available from another source - #E: Package 'sftp' has no installation candidate - + # this is why we install openssh-server because it includes it. # On Ubuntu 20.04, with python10, the pip install fails. # So you need to get the pip from source @@ -47,7 +48,6 @@ sudo apt install git openssh-client -y sudo python -m pip install --upgrade pip -sudo apt install sftp -y # Go to https://expo.survex.com/handbook/troggle/troglaptop.html#dbtools diff --git a/requirements-p10d4.txt b/requirements-p10d4.txt new file mode 100644 index 0000000..aafb7f6 --- /dev/null +++ b/requirements-p10d4.txt @@ -0,0 +1,17 @@ +asgiref==3.6.0 +black==23.1.0 +click==8.1.3 +coverage==7.1.0 +Django==4.2 +docutils==0.19 +isort==5.12.0 +mypy-extensions==1.0.0 +packaging==23.0 +pathspec==0.11.0 +Pillow==9.4.0 +platformdirs==3.0.0 +pytz==2022.7 +ruff==0.0.245 +sqlparse==0.4.3 +tomli==2.0.1 +Unidecode==1.3.6 diff --git a/venv-trog.sh b/venv-trog.sh index 42fac41..2133bef 100644 --- a/venv-trog.sh +++ b/venv-trog.sh @@ -27,6 +27,7 @@ echo ## Using requirements.txt : cat requirements.txt echo ## + $PYTHON --version # NOTE that when using a later or earlier verison of python, you MUST also @@ -166,5 +167,5 @@ $sudo chown -Rhv philip:philip ~/$VENAME (if your username is philip) 'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)' " if [ ! -d /mnt/d/expofiles ]; then -echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work. + echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.' fi \ No newline at end of file