From 98e7cf6d1b8785691d1e855813a74b4ac6fd51c2 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@gmail.com>
Date: Tue, 26 Sep 2023 15:20:59 +0300
Subject: [PATCH] test stuff for Django 5.0a (alpha)

---
 README.txt                                    |   3 +
 _deploy/wsl/requirements-p11d4.txt            |  21 --
 _deploy/wsl/requirements-p11d42.txt           |  21 ++
 _deploy/wsl/requirements-p11d5.txt            |  28 ++-
 _deploy/wsl/venv-trog-d4.sh                   | 184 ++++++++++++++++++
 _deploy/wsl/{venv-trog.sh => venv-trog-d5.sh} |  55 ++++--
 6 files changed, 262 insertions(+), 50 deletions(-)
 delete mode 100644 _deploy/wsl/requirements-p11d4.txt
 create mode 100644 _deploy/wsl/requirements-p11d42.txt
 create mode 100644 _deploy/wsl/venv-trog-d4.sh
 rename _deploy/wsl/{venv-trog.sh => venv-trog-d5.sh} (81%)

diff --git a/README.txt b/README.txt
index 41cde78..cd9333c 100644
--- a/README.txt
+++ b/README.txt
@@ -8,6 +8,9 @@ Troggle has been forked into two projects. The original one is maintained by Aar
 and was used for Erebus caves in Antarctica. 
 The CUCC variant uses files as the definitive data, not the database, and lives at http://expo.survex.com/repositories/troggle/.git/
 
+The versions have diverged markedly, not just in the software but also in the implicit convnetions of how the directory structures of the survex files, the drawings and
+the scans are arranged.
+
 For the server setup, see /_deploy/debian/wookey-exposerver-recipe.txt
 and see http://expo.survex.com/handbook/troggle/serverconfig.html
 
diff --git a/_deploy/wsl/requirements-p11d4.txt b/_deploy/wsl/requirements-p11d4.txt
deleted file mode 100644
index 009a3c9..0000000
--- a/_deploy/wsl/requirements-p11d4.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-asgiref==3.6
-beautifulsoup4=4.12
-black==23.1.0
-chardet=5.1
-click==8.1.3
-coverage==7.1.0
-deptry=0.12
-Django==4.2
-docutils==0.19
-isort==5.12
-mypy-extensions==1.0.0
-packaging==23.0
-pathspec==0.11
-Pillow==9.4.0
-platformdirs==3.0
-pytz==2022.7
-ruff==0.0.245
-setuptools=67.7
-soupsieve=2.5
-sqlparse==0.4
-Unidecode==1.3
diff --git a/_deploy/wsl/requirements-p11d42.txt b/_deploy/wsl/requirements-p11d42.txt
new file mode 100644
index 0000000..5f5eed7
--- /dev/null
+++ b/_deploy/wsl/requirements-p11d42.txt
@@ -0,0 +1,21 @@
+asgiref==3.6.0
+beautifulsoup4==4.12.0
+black==23.1.0
+chardet==5.1.0
+click==8.1.0
+coverage==7.1.0
+deptry==0.12.0
+Django==4.2
+docutils==0.19
+isort==5.12.0
+mypy-extensions==1.0.0
+packaging==23.0
+pathspec==0.11.0
+Pillow==9.4.0
+platformdirs==3.0.0
+pytz==2022.7
+ruff==0.0.245
+soupsieve==2.5
+sqlparse==0.4.0
+Unidecode==1.3.0
+piexif==1.1
diff --git a/_deploy/wsl/requirements-p11d5.txt b/_deploy/wsl/requirements-p11d5.txt
index 604ffcd..ee1d50c 100644
--- a/_deploy/wsl/requirements-p11d5.txt
+++ b/_deploy/wsl/requirements-p11d5.txt
@@ -1,9 +1,21 @@
-asgiref==3.5.2
-coverage==6.5.0
-Django==3.2.16
+# to be used with pre-release Djangi install which installs other stuff too
+beautifulsoup4==4.12
+piexif==1.1
+black==23.1
+chardet==5.1
+click==8.1
+coverage==7.1
+deptry==0.12
 docutils==0.19
-Pillow==9.3.0
-pytz==2022.6
-sqlparse==0.4.3
-typing_extensions==4.4.0
-Unidecode==1.3.6
+isort==5.12
+mypy-extensions==1.0
+packaging==23.0
+pathspec==0.11
+Pillow==9.4.0
+platformdirs==3.0
+pytz==2022.7
+ruff==0.0.245
+setuptools==67.7
+soupsieve==2.5
+Unidecode==1.3
+piexif==1.1
diff --git a/_deploy/wsl/venv-trog-d4.sh b/_deploy/wsl/venv-trog-d4.sh
new file mode 100644
index 0000000..97e99b7
--- /dev/null
+++ b/_deploy/wsl/venv-trog-d4.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+# footled lots to make this work with python 3.10 & 3.11 and WSL1 and WSL2 on Ubuntu 22.04
+# Run this in a terminal in the troggle directory: 'bash venv-trog.sh'
+echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.sh"'
+
+# Expects an Ubuntu 22.04 (or 20.04) relatively clean install.
+# If you have not already installed these on your clean Ubuntu install DO THIS FIRST
+# use the script os-trog.sh
+
+# If you are using Debian, then stick with the default version of python
+# If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.11
+
+# NOW we set up troggle
+PYTHON=python3.11
+VENAME=p11d42 # python3.x and django 4.2
+echo "** You are logged in as `id -u -n`"
+echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/"
+echo "The 150MB venv will created in  /home/`id -u -n`/$VENAME/"
+TROGDIR=$(cd $(dirname $0) && pwd)
+echo "-- Troggle folder (this script location): ${TROGDIR}"
+
+REQUIRE=requirements-$VENAME.txt
+
+if [ -d $REQUIRE ]; then
+  echo "-- No ${REQUIRE} found. You should be in the /troggle/ folder. Copy it from your most recent installation."
+  exit 1
+fi
+echo ## Using $REQUIRE :
+cat $REQUIRE
+echo ##
+
+
+$PYTHON --version
+
+# NOTE that when using a later or earlier verison of python, you MUST also
+# use the allowed version of Pillow, see https://pillow.readthedocs.io/en/latest/installation.html
+
+# NOW set up link from expo user folder
+# needed for WSL2
+echo Creating links from Linux filesystem user 
+# These links only need making once, for many venv
+cd ~
+
+if [ ! -d $VENAME ]; then
+  echo "## Creating venv $VENAME. (If this fails with a pip error, you need to ensure you have python3.11-venv installed and/or use a Ubuntu window)"
+  $PYTHON -m venv $VENAME
+else 
+  echo "## /$VENAME/ already exists ! Delete it first."
+  exit 1
+fi
+
+  # Activate the virtual env and see what the default packages are
+echo "### Activating $VENAME"
+
+cd $VENAME
+echo "-- now in: ${PWD}"
+ls -tlarg
+source bin/activate
+echo $VIRTUAL_ENV 
+if [ -d ~/$VENAME/bin ]; then
+    echo "### Activating."
+else
+  echo "## ~/$VENAME/bin does not exist. FAILed to create venv properly."
+  exit 1
+fi
+  # update local version of pip, more recent than OS version
+  # debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3
+  
+  # update local version of setuptools, more recent than OS version, needed for packages without wheels
+  
+echo "### installing later version of pip inside $VENAME"
+$PYTHON -m pip install --upgrade pip
+$PYTHON -m pip install --upgrade setuptools
+
+PIP=pip
+
+$PIP list > original-pip.list
+$PIP freeze >original.txt
+
+# we are in  /home/$USER/$VENAME/
+ln -s ${TROGDIR} troggle
+ln -s ${TROGDIR}/../expoweb expoweb
+ln -s ${TROGDIR}/../loser loser
+ln -s ${TROGDIR}/../drawings drawings
+#ln -s ${TROGDIR}/../expofiles expofiles
+
+# fudge for philip's machine
+if [ ! -d /mnt/d/EXPO ]; then
+  sudo mkdir /mnt/d
+  sudo mount -t drvfs D: /mnt/d
+fi
+  
+if [  -d ${TROGDIR}/../expofiles ]; then
+  ln -s ${TROGDIR}/../expofiles expofiles
+else
+  ln -s /mnt/d/EXPO/expofiles expofiles
+fi
+
+echo "### Setting file permissions.. may take a while.."
+git config --global --add safe.directory '*'
+sudo chmod -R 777 *
+
+echo "### links to expoweb, troggle etc. complete:"
+ls -tla
+echo "###"
+echo "### now installing ${TROGDIR}/${REQUIRE}"
+echo "###"
+cat ${TROGDIR}/${REQUIRE}
+
+# NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE
+# seen on wsl2 as well as wsl1 
+# which ALSO ruins EXISTING permissions !
+# Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04
+
+read -p "Press any key to resume ..."
+$PIP install -r ${TROGDIR}/${REQUIRE}
+echo "### install from ${TROGDIR}/${REQUIRE} completed."
+echo '### '
+
+$PIP install --pre django
+
+$PIP freeze > $REQUIRE  
+# so that we can track requirements more easily with git
+# because we do not install these with pip, but they are listed by the freeze command
+# Now find out what we actually installed by subtracting the stuff venv installed anyway
+sort original.txt > 1
+sort $REQUIRE >2
+comm -3 1 2 --check-order | awk '{ print $1}'>fresh-$REQUIRE
+rm 1
+rm 2
+
+# cp $REQUIRE requirements-$VENAME.txt 
+cp $REQUIRE troggle/$REQUIRE
+
+$PIP list > installed-pip.list
+$PIP list -o > installed-pip-o.list
+
+REQ=installation-record
+mkdir $REQ 
+
+mv original.txt $REQ
+mv $REQUIRE $REQ
+mv original-pip.list $REQ
+mv installed-pip.list $REQ
+mv installed-pip-o.list $REQ
+cp fresh-$REQUIRE ../$REQUIRE
+mv fresh-$REQUIRE $REQ
+cp troggle/`basename "$0"` $REQ
+
+
+$PYTHON --version
+python --version
+echo "Django version:`django-admin --version`"
+
+echo "### Now do 
+  '[sudo service mysql start]'
+  '[sudo service mariadb restart]'
+  '[sudo mysql_secure_installation]'
+  'cd ~/$VENAME'
+  'source bin/activate'
+  'cd troggle'
+  'django-admin'
+  'python manage.py check'
+## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html
+## you need to follow the Linux instructions.
+  'ssh expo@expo.survex.com'
+
+## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine
+## the tests may ALSO fail because of ssh and permissions errors 
+
+## So you will need to run 
+$sudo chown -Rhv philip:philip ~/$VENAME  (if your username is philip)
+# and then REBOOT (or at least, exit WSL and terminate and restart WSL)
+# because this chmod only takes effect then.
+
+   'python manage.py test -v 2'
+   './pre-run.sh' (runs the tests again)
+   
+   'python databaseReset.py reset $VENAME'
+   'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
+   "
+# if [ ! -d /mnt/d/expofiles ]; then
+  # echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.'
+# fi
\ No newline at end of file
diff --git a/_deploy/wsl/venv-trog.sh b/_deploy/wsl/venv-trog-d5.sh
similarity index 81%
rename from _deploy/wsl/venv-trog.sh
rename to _deploy/wsl/venv-trog-d5.sh
index 2133bef..6ec82a0 100644
--- a/_deploy/wsl/venv-trog.sh
+++ b/_deploy/wsl/venv-trog-d5.sh
@@ -12,19 +12,21 @@ echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.s
 
 # NOW we set up troggle
 PYTHON=python3.11
-VENAME=p11d4 # python3.x and django 4.2
+VENAME=p11d5 # python3.x and django 4.2
 echo "** You are logged in as `id -u -n`"
 echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/"
 echo "The 150MB venv will created in  /home/`id -u -n`/$VENAME/"
 TROGDIR=$(cd $(dirname $0) && pwd)
 echo "-- Troggle folder (this script location): ${TROGDIR}"
 
-if [ -d requirements.txt ]; then
-  echo "-- No requirements.txt found. You should be in the /troggle/ folder. Copy it from your most recent installation."
+REQUIRE=requirements-$VENAME.txt
+
+if [ -d $REQUIRE ]; then
+  echo "-- No ${REQUIRE} found. You should be in the /troggle/ folder. Copy it from your most recent installation."
   exit 1
 fi
-echo ## Using requirements.txt :
-cat requirements.txt
+echo ## Using $REQUIRE :
+cat $REQUIRE
 echo ##
 
 
@@ -52,8 +54,15 @@ echo "### Activating $VENAME"
 
 cd $VENAME
 echo "-- now in: ${PWD}"
+ls -tlarg
 source bin/activate
-echo "### Activated."
+echo $VIRTUAL_ENV 
+if [ -d ~/$VENAME/bin ]; then
+    echo "### Activating."
+else
+  echo "## ~/$VENAME/bin does not exist. FAILed to create venv properly."
+  exit 1
+fi
   # update local version of pip, more recent than OS version
   # debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3
   
@@ -94,44 +103,48 @@ sudo chmod -R 777 *
 echo "### links to expoweb, troggle etc. complete:"
 ls -tla
 echo "###"
-echo "### now installing ${TROGDIR}/requirements.txt"
+echo "### now installing ${TROGDIR}/${REQUIRE}"
 echo "###"
+cat ${TROGDIR}/${REQUIRE}
 
 # NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE
 # seen on wsl2 as well as wsl1 
 # which ALSO ruins EXISTING permissions !
 # Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04
 
-$PIP install -r ${TROGDIR}/requirements.txt
-echo '### install from requirements.txt completed.'
+read -p "Press any key to resume ..."
+$PIP install -r ${TROGDIR}/${REQUIRE}
+echo "### install from ${TROGDIR}/${REQUIRE} completed."
 echo '### '
 
-$PIP freeze > requirements.txt  
+$PIP install --pre django
+
+$PIP freeze > $REQUIRE  
 # so that we can track requirements more easily with git
 # because we do not install these with pip, but they are listed by the freeze command
 # Now find out what we actually installed by subtracting the stuff venv installed anyway
 sort original.txt > 1
-sort requirements.txt >2
-comm -3 1 2 --check-order | awk '{ print $1}'>fresh-requirements.txt
+sort $REQUIRE >2
+comm -3 1 2 --check-order | awk '{ print $1}'>fresh-$REQUIRE
 rm 1
 rm 2
 
-cp requirements.txt requirements-$VENAME.txt 
-cp requirements-$VENAME.txt troggle/requirements-$VENAME.txt
+# cp $REQUIRE requirements-$VENAME.txt 
+cp $REQUIRE troggle/$REQUIRE
 
 $PIP list > installed-pip.list
 $PIP list -o > installed-pip-o.list
 
 REQ=installation-record
 mkdir $REQ 
-mv requirements-$VENAME.txt $REQ
+
 mv original.txt $REQ
-mv requirements.txt $REQ
+mv $REQUIRE $REQ
 mv original-pip.list $REQ
 mv installed-pip.list $REQ
 mv installed-pip-o.list $REQ
-cp fresh-requirements.txt ../requirements.txt
-mv fresh-requirements.txt $REQ
+cp fresh-$REQUIRE ../$REQUIRE
+mv fresh-$REQUIRE $REQ
 cp troggle/`basename "$0"` $REQ
 
 
@@ -166,6 +179,6 @@ $sudo chown -Rhv philip:philip ~/$VENAME  (if your username is philip)
    'python databaseReset.py reset $VENAME'
    'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
    "
-if [ ! -d /mnt/d/expofiles ]; then
-  echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.'
-fi
\ No newline at end of file
+# if [ ! -d /mnt/d/expofiles ]; then
+  # echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.'
+# fi
\ No newline at end of file