From b191678570e7a8f0f26b1278222e89dbed076752 Mon Sep 17 00:00:00 2001
From: Your Name <Philip Sargent>
Date: Wed, 22 Nov 2023 16:54:49 +0000
Subject: [PATCH] working on django 5

---
 _deploy/xubuntu/os-trog.sh      | 102 +++++++++++++++++
 _deploy/xubuntu/venv-trog-d5.sh | 192 ++++++++++++++++++++++++++++++++
 requirements-p12d5.txt          |  23 ++++
 venv-trog-d5.sh                 | 192 ++++++++++++++++++++++++++++++++
 4 files changed, 509 insertions(+)
 create mode 100755 _deploy/xubuntu/os-trog.sh
 create mode 100755 _deploy/xubuntu/venv-trog-d5.sh
 create mode 100755 requirements-p12d5.txt
 create mode 100755 venv-trog-d5.sh

diff --git a/_deploy/xubuntu/os-trog.sh b/_deploy/xubuntu/os-trog.sh
new file mode 100755
index 0000000..6f83494
--- /dev/null
+++ b/_deploy/xubuntu/os-trog.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+# Run this in a terminal in the troggle directory: 'bash os-trog.sh'
+# On WSL, do Shift-click in the file explorer on the troggle folder to open a Linux command line
+# 'Open Linux shell here'
+echo 'Run this in a terminal in the troggle directory: "bash venv-trog.sh"'
+cat /etc/os-release
+# Expects an Ubuntu 22.04 relatively clean install.
+
+sudo apt install python-is-python3 -y
+python --version : ensure python is an alias for python3 not python2.7
+sudo apt update -y
+sudo apt dist-upgrade -y
+sudo apt autoremove -y
+sudo apt install sqlite3 -y
+sudo apt install python3-pip  -y
+
+# this installs a shed-load of other stuff: binutils etc.sudo apt install survex-aven
+sudo apt install git openssh-client -y
+# On a clean debian 11 (bullseye) installation with Xfce & ssh, 
+
+    #on ubuntu 20.04:
+    #Package sftp is not available, but is referred to by another package.
+    #This may mean that the package is missing, has been obsoleted, or
+    #is only available from another source
+    #E: Package 'sftp' has no installation candidate
+    
+    
+# On Ubuntu 20.04, with python10, the pip install fails.
+# So you need to get the pip from source
+# sudo curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
+# but really you should be using 22.04
+# and also, isf using debian, 
+# sudo python3.10 -m pip install -U virtualenv
+
+# do not actually use this any more
+sudo useradd expo
+sudo usermod -a -G sudo expo # to put expo in sudoers group, re-login required
+
+# as debian does not install everything that ubuntu does, you need:
+sudo apt install python3-venv -y
+sudo apt install python3-dev -y
+
+# default since 22.04 
+# sudo apt install python3.10
+sudo apt install python3.11-venv -y
+sudo apt install python3.11-dev -y
+sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1
+
+sudo apt install mariadb-server -y 
+sudo apt install libmariadb-dev -y
+
+sudo python -m pip install --upgrade pip
+
+sudo apt install sftp -y
+echo '###'
+echo '### NOW INSTALLING tunnel and therion, go and have a cup of tea. Or a 3-course meal.'
+echo '###'
+sudo apt install tunnelx therion -y
+sudo apt install gedit
+
+# Go to https://expo.survex.com/handbook/troggle/troglaptop.html#dbtools
+# sudo service mysql start
+
+echo "### python 3.12"
+sudo add-apt-repository ppa:deadsnakes/ppa -y
+sudo apt update
+sudo apt install python3.12-full -y
+sudo apt install python3.12-distutils -y
+
+sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.12 7
+sudo update-alternatives --config python
+sudo rm /usr/bin/python3
+sudo ln -s /etc/alternatives/python /usr/bin/python3
+sudo apt dist-upgrade
+
+#this next crashes, why?
+#sudo python -m pip install --upgrade pip
+
+sudo apt install gpsprune qgis gedit tig gitg meld rsync
+
+git config --global user.email "you@example.com"
+git config --global user.name "Your Name"
+git config --global pull.rebase true
+
+echo '###'
+echo '### Currently set version of python'
+python --version
+
+echo '###'
+echo '### Now YOU have to configure the git settings for YOURSELF (not "expo")'
+echo '### because you can't clone the repos without a key
+# cd ~/expo
+
+git clone ssh://expo@expo.survex.com/home/expo/troggle
+git clone ssh://expo@expo.survex.com/home/expo/loser
+git clone ssh://expo@expo.survex.com/home/expo/expoweb
+git clone ssh://expo@expo.survex.com/home/expo/drawings
+
+mkdir expofiles
+rsync -azv --delete-after --prune-empty-dirs --exclude="photos" --exclude="video" expo@expo.survex.com:expofiles/ expofiles
+rsync  -azv --exclude="*.jpg.xml" --exclude="*.jpeg.xml" --exclude="*.JPG.xml" expo@expo.s^Cvex.com:expofiles/photos/ expofiles/photos
+
diff --git a/_deploy/xubuntu/venv-trog-d5.sh b/_deploy/xubuntu/venv-trog-d5.sh
new file mode 100755
index 0000000..e70a49a
--- /dev/null
+++ b/_deploy/xubuntu/venv-trog-d5.sh
@@ -0,0 +1,192 @@
+#!/bin/bash
+# footled lots to make this work with python 3.10 & 3.11 and WSL1 and WSL2 on Ubuntu 22.04
+# Run this in a terminal in the troggle directory: 'bash venv-trog.sh'
+echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.sh"'
+
+# Expects an Ubuntu 22.04 (or 20.04) relatively clean install.
+# If you have not already installed these on your clean Ubuntu install DO THIS FIRST
+# use the script os-trog.sh
+
+# If you are using Debian, then stick with the default version of python
+# If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.11
+
+# NOW we set up troggle
+PYTHON=python3.12
+VENAME=p12d5 # python3.x and django 4.2
+echo "** You are logged in as `id -u -n`"
+echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/"
+echo "The 150MB venv will created in  /home/`id -u -n`/$VENAME/"
+TROGDIR=$(cd $(dirname $0) && pwd)
+echo "-- Troggle folder (this script location): ${TROGDIR}"
+
+REQUIRE=requirements-$VENAME.txt
+
+if [ -d $REQUIRE ]; then
+  echo "-- No ${REQUIRE} found. You should be in the /troggle/ folder. Copy it from your most recent installation."
+  exit 1
+fi
+echo ## Using $REQUIRE :
+cat $REQUIRE
+echo ##
+
+
+$PYTHON --version
+
+# NOTE that when using a later or earlier verison of python, you MUST also
+# use the allowed version of Pillow, see https://pillow.readthedocs.io/en/latest/installation.html
+
+# NOW set up link from expo user folder
+# needed for WSL2
+echo Creating links from Linux filesystem user 
+# These links only need making once, for many venv
+cd ~
+
+if [ ! -d $VENAME ]; then
+  echo "## Creating venv $VENAME. (If this fails with a pip error, you need to ensure you have python3.11-venv installed and/or use a Ubuntu window)"
+  $PYTHON -m venv $VENAME
+else 
+  echo "## /$VENAME/ already exists ! Delete it first."
+  exit 1
+fi
+
+  # Activate the virtual env and see what the default packages are
+echo "### Activating $VENAME"
+
+cd $VENAME
+echo "-- now in: ${PWD}"
+ls -tlarg
+source bin/activate
+echo $VIRTUAL_ENV 
+if [ -d ~/$VENAME/bin ]; then
+    echo "### Activating."
+else
+  echo "## ~/$VENAME/bin does not exist. FAILed to create venv properly."
+  exit 1
+fi
+  # update local version of pip, more recent than OS version
+  # debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3
+  
+  # update local version of setuptools, more recent than OS version, needed for packages without wheels
+  
+echo "### installing later version of pip inside $VENAME"
+$PYTHON -m pip install --upgrade pip
+$PYTHON -m pip install --upgrade setuptools
+
+PIP=pip
+
+$PIP list > original-pip.list
+$PIP freeze >original.txt
+
+# we are in  /home/$USER/$VENAME/
+ln -s ${TROGDIR} troggle
+ln -s ${TROGDIR}/../expoweb expoweb
+ln -s ${TROGDIR}/../loser loser
+ln -s ${TROGDIR}/../drawings drawings
+#ln -s ${TROGDIR}/../expofiles expofiles
+
+# fudge for philip's machine
+if [ ! -d /mnt/d/EXPO ]; then
+  sudo mkdir /mnt/d
+  sudo mount -t drvfs D: /mnt/d
+fi
+  
+if [  -d ${TROGDIR}/../expofiles ]; then
+  ln -s ${TROGDIR}/../expofiles expofiles
+else
+  ln -s /mnt/d/EXPO/expofiles expofiles
+fi
+
+echo "### Setting file permissions.. may take a while.."
+git config --global --add safe.directory '*'
+sudo chmod -R 777 *
+
+echo "### links to expoweb, troggle etc. complete:"
+ls -tla
+echo "###"
+echo "### now installing ${TROGDIR}/${REQUIRE}"
+echo "###"
+cat ${TROGDIR}/${REQUIRE}
+
+# NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE
+# seen on wsl2 as well as wsl1 
+# which ALSO ruins EXISTING permissions !
+# Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04
+
+read -p "Press any key to resume ..."
+$PIP install -r ${TROGDIR}/${REQUIRE}
+echo "### install from ${TROGDIR}/${REQUIRE} completed."
+echo '### '
+
+# this installs pre-release django 5.0
+$PIP install --pre django
+
+$PIP freeze > $REQUIRE  
+# so that we can track requirements more easily with git
+# because we do not install these with pip, but they are listed by the freeze command
+# Now find out what we actually installed by subtracting the stuff venv installed anyway
+sort original.txt > 1
+sort $REQUIRE >2
+comm -3 1 2 --check-order | awk '{ print $1}'>fresh-$REQUIRE
+rm 1
+rm 2
+
+# cp $REQUIRE requirements-$VENAME.txt 
+cp $REQUIRE troggle/$REQUIRE
+
+$PIP list > installed-pip.list
+$PIP list -o > installed-pip-o.list
+
+REQ=installation-record
+mkdir $REQ 
+
+mv original.txt $REQ
+mv $REQUIRE $REQ
+mv original-pip.list $REQ
+mv installed-pip.list $REQ
+mv installed-pip-o.list $REQ
+cp fresh-$REQUIRE ../$REQUIRE
+mv fresh-$REQUIRE $REQ
+cp troggle/`basename "$0"` $REQ
+
+# install VS code
+sudo apt install software-properties-common apt-transport-https wget gpg
+wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg
+sudo install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg
+sudo sh -c 'echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" > /etc/apt/sources.list.d/vscode.list'
+sudo apt update
+sudo apt install code
+
+$PYTHON --version
+python --version
+echo "Django version:`django-admin --version`"
+
+echo "### Now do 
+  '[sudo service mysql start]'
+  '[sudo service mariadb restart]'
+  '[sudo mysql_secure_installation]'
+  'cd ~/$VENAME'
+  'source bin/activate'
+  'cd troggle'
+  'django-admin'
+  'python manage.py check'
+## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html
+## you need to follow the Linux instructions.
+  'ssh expo@expo.survex.com'
+
+## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine
+## the tests may ALSO fail because of ssh and permissions errors 
+
+## So you will need to run 
+$sudo chown -Rhv philip:philip ~/$VENAME  (if your username is philip)
+# and then REBOOT (or at least, exit WSL and terminate and restart WSL)
+# because this chmod only takes effect then.
+
+   'python manage.py test -v 2'
+   './pre-run.sh' (runs the tests again)
+   
+   'python databaseReset.py reset $VENAME'
+   'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
+   "
+# if [ ! -d /mnt/d/expofiles ]; then
+  # echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.'
+# fi
diff --git a/requirements-p12d5.txt b/requirements-p12d5.txt
new file mode 100755
index 0000000..fe10a93
--- /dev/null
+++ b/requirements-p12d5.txt
@@ -0,0 +1,23 @@
+asgiref==3.7.2
+setuptools==69.0.2
+sqlparse==0.4.4
+asgiref==3.6.0
+BeautifulSoup4==4.12.0
+black==23.1.0
+chardet==5.1.0
+click==8.1.0
+coverage==7.1.0
+deptry==0.12.0
+docutils==0.19
+isort==5.12.0
+mypy-extensions==1.0.0
+packaging==23.0
+pathspec==0.11.0
+piexif==1.1.3
+Pillow==9.4.0
+platformdirs==3.0.0
+pytz==2022.7
+ruff==0.0.245
+soupsieve==2.5
+sqlparse==0.4.0
+Unidecode==1.3.0
diff --git a/venv-trog-d5.sh b/venv-trog-d5.sh
new file mode 100755
index 0000000..e70a49a
--- /dev/null
+++ b/venv-trog-d5.sh
@@ -0,0 +1,192 @@
+#!/bin/bash
+# footled lots to make this work with python 3.10 & 3.11 and WSL1 and WSL2 on Ubuntu 22.04
+# Run this in a terminal in the troggle directory: 'bash venv-trog.sh'
+echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.sh"'
+
+# Expects an Ubuntu 22.04 (or 20.04) relatively clean install.
+# If you have not already installed these on your clean Ubuntu install DO THIS FIRST
+# use the script os-trog.sh
+
+# If you are using Debian, then stick with the default version of python
+# If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.11
+
+# NOW we set up troggle
+PYTHON=python3.12
+VENAME=p12d5 # python3.x and django 4.2
+echo "** You are logged in as `id -u -n`"
+echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/"
+echo "The 150MB venv will created in  /home/`id -u -n`/$VENAME/"
+TROGDIR=$(cd $(dirname $0) && pwd)
+echo "-- Troggle folder (this script location): ${TROGDIR}"
+
+REQUIRE=requirements-$VENAME.txt
+
+if [ -d $REQUIRE ]; then
+  echo "-- No ${REQUIRE} found. You should be in the /troggle/ folder. Copy it from your most recent installation."
+  exit 1
+fi
+echo ## Using $REQUIRE :
+cat $REQUIRE
+echo ##
+
+
+$PYTHON --version
+
+# NOTE that when using a later or earlier verison of python, you MUST also
+# use the allowed version of Pillow, see https://pillow.readthedocs.io/en/latest/installation.html
+
+# NOW set up link from expo user folder
+# needed for WSL2
+echo Creating links from Linux filesystem user 
+# These links only need making once, for many venv
+cd ~
+
+if [ ! -d $VENAME ]; then
+  echo "## Creating venv $VENAME. (If this fails with a pip error, you need to ensure you have python3.11-venv installed and/or use a Ubuntu window)"
+  $PYTHON -m venv $VENAME
+else 
+  echo "## /$VENAME/ already exists ! Delete it first."
+  exit 1
+fi
+
+  # Activate the virtual env and see what the default packages are
+echo "### Activating $VENAME"
+
+cd $VENAME
+echo "-- now in: ${PWD}"
+ls -tlarg
+source bin/activate
+echo $VIRTUAL_ENV 
+if [ -d ~/$VENAME/bin ]; then
+    echo "### Activating."
+else
+  echo "## ~/$VENAME/bin does not exist. FAILed to create venv properly."
+  exit 1
+fi
+  # update local version of pip, more recent than OS version
+  # debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3
+  
+  # update local version of setuptools, more recent than OS version, needed for packages without wheels
+  
+echo "### installing later version of pip inside $VENAME"
+$PYTHON -m pip install --upgrade pip
+$PYTHON -m pip install --upgrade setuptools
+
+PIP=pip
+
+$PIP list > original-pip.list
+$PIP freeze >original.txt
+
+# we are in  /home/$USER/$VENAME/
+ln -s ${TROGDIR} troggle
+ln -s ${TROGDIR}/../expoweb expoweb
+ln -s ${TROGDIR}/../loser loser
+ln -s ${TROGDIR}/../drawings drawings
+#ln -s ${TROGDIR}/../expofiles expofiles
+
+# fudge for philip's machine
+if [ ! -d /mnt/d/EXPO ]; then
+  sudo mkdir /mnt/d
+  sudo mount -t drvfs D: /mnt/d
+fi
+  
+if [  -d ${TROGDIR}/../expofiles ]; then
+  ln -s ${TROGDIR}/../expofiles expofiles
+else
+  ln -s /mnt/d/EXPO/expofiles expofiles
+fi
+
+echo "### Setting file permissions.. may take a while.."
+git config --global --add safe.directory '*'
+sudo chmod -R 777 *
+
+echo "### links to expoweb, troggle etc. complete:"
+ls -tla
+echo "###"
+echo "### now installing ${TROGDIR}/${REQUIRE}"
+echo "###"
+cat ${TROGDIR}/${REQUIRE}
+
+# NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE
+# seen on wsl2 as well as wsl1 
+# which ALSO ruins EXISTING permissions !
+# Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04
+
+read -p "Press any key to resume ..."
+$PIP install -r ${TROGDIR}/${REQUIRE}
+echo "### install from ${TROGDIR}/${REQUIRE} completed."
+echo '### '
+
+# this installs pre-release django 5.0
+$PIP install --pre django
+
+$PIP freeze > $REQUIRE  
+# so that we can track requirements more easily with git
+# because we do not install these with pip, but they are listed by the freeze command
+# Now find out what we actually installed by subtracting the stuff venv installed anyway
+sort original.txt > 1
+sort $REQUIRE >2
+comm -3 1 2 --check-order | awk '{ print $1}'>fresh-$REQUIRE
+rm 1
+rm 2
+
+# cp $REQUIRE requirements-$VENAME.txt 
+cp $REQUIRE troggle/$REQUIRE
+
+$PIP list > installed-pip.list
+$PIP list -o > installed-pip-o.list
+
+REQ=installation-record
+mkdir $REQ 
+
+mv original.txt $REQ
+mv $REQUIRE $REQ
+mv original-pip.list $REQ
+mv installed-pip.list $REQ
+mv installed-pip-o.list $REQ
+cp fresh-$REQUIRE ../$REQUIRE
+mv fresh-$REQUIRE $REQ
+cp troggle/`basename "$0"` $REQ
+
+# install VS code
+sudo apt install software-properties-common apt-transport-https wget gpg
+wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg
+sudo install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg
+sudo sh -c 'echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" > /etc/apt/sources.list.d/vscode.list'
+sudo apt update
+sudo apt install code
+
+$PYTHON --version
+python --version
+echo "Django version:`django-admin --version`"
+
+echo "### Now do 
+  '[sudo service mysql start]'
+  '[sudo service mariadb restart]'
+  '[sudo mysql_secure_installation]'
+  'cd ~/$VENAME'
+  'source bin/activate'
+  'cd troggle'
+  'django-admin'
+  'python manage.py check'
+## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html
+## you need to follow the Linux instructions.
+  'ssh expo@expo.survex.com'
+
+## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine
+## the tests may ALSO fail because of ssh and permissions errors 
+
+## So you will need to run 
+$sudo chown -Rhv philip:philip ~/$VENAME  (if your username is philip)
+# and then REBOOT (or at least, exit WSL and terminate and restart WSL)
+# because this chmod only takes effect then.
+
+   'python manage.py test -v 2'
+   './pre-run.sh' (runs the tests again)
+   
+   'python databaseReset.py reset $VENAME'
+   'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
+   "
+# if [ ! -d /mnt/d/expofiles ]; then
+  # echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.'
+# fi