diff --git a/_deploy/xubuntu/os-trog.sh b/_deploy/xubuntu/os-trog.sh new file mode 100755 index 0000000..6f83494 --- /dev/null +++ b/_deploy/xubuntu/os-trog.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Run this in a terminal in the troggle directory: 'bash os-trog.sh' +# On WSL, do Shift-click in the file explorer on the troggle folder to open a Linux command line +# 'Open Linux shell here' +echo 'Run this in a terminal in the troggle directory: "bash venv-trog.sh"' +cat /etc/os-release +# Expects an Ubuntu 22.04 relatively clean install. + +sudo apt install python-is-python3 -y +python --version : ensure python is an alias for python3 not python2.7 +sudo apt update -y +sudo apt dist-upgrade -y +sudo apt autoremove -y +sudo apt install sqlite3 -y +sudo apt install python3-pip -y + +# this installs a shed-load of other stuff: binutils etc.sudo apt install survex-aven +sudo apt install git openssh-client -y +# On a clean debian 11 (bullseye) installation with Xfce & ssh, + + #on ubuntu 20.04: + #Package sftp is not available, but is referred to by another package. + #This may mean that the package is missing, has been obsoleted, or + #is only available from another source + #E: Package 'sftp' has no installation candidate + + +# On Ubuntu 20.04, with python10, the pip install fails. +# So you need to get the pip from source +# sudo curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 +# but really you should be using 22.04 +# and also, isf using debian, +# sudo python3.10 -m pip install -U virtualenv + +# do not actually use this any more +sudo useradd expo +sudo usermod -a -G sudo expo # to put expo in sudoers group, re-login required + +# as debian does not install everything that ubuntu does, you need: +sudo apt install python3-venv -y +sudo apt install python3-dev -y + +# default since 22.04 +# sudo apt install python3.10 +sudo apt install python3.11-venv -y +sudo apt install python3.11-dev -y +sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 + +sudo apt install mariadb-server -y +sudo apt install libmariadb-dev -y + +sudo python -m pip install --upgrade pip + +sudo apt install sftp -y +echo '###' +echo '### NOW INSTALLING tunnel and therion, go and have a cup of tea. Or a 3-course meal.' +echo '###' +sudo apt install tunnelx therion -y +sudo apt install gedit + +# Go to https://expo.survex.com/handbook/troggle/troglaptop.html#dbtools +# sudo service mysql start + +echo "### python 3.12" +sudo add-apt-repository ppa:deadsnakes/ppa -y +sudo apt update +sudo apt install python3.12-full -y +sudo apt install python3.12-distutils -y + +sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.12 7 +sudo update-alternatives --config python +sudo rm /usr/bin/python3 +sudo ln -s /etc/alternatives/python /usr/bin/python3 +sudo apt dist-upgrade + +#this next crashes, why? +#sudo python -m pip install --upgrade pip + +sudo apt install gpsprune qgis gedit tig gitg meld rsync + +git config --global user.email "you@example.com" +git config --global user.name "Your Name" +git config --global pull.rebase true + +echo '###' +echo '### Currently set version of python' +python --version + +echo '###' +echo '### Now YOU have to configure the git settings for YOURSELF (not "expo")' +echo '### because you can't clone the repos without a key +# cd ~/expo + +git clone ssh://expo@expo.survex.com/home/expo/troggle +git clone ssh://expo@expo.survex.com/home/expo/loser +git clone ssh://expo@expo.survex.com/home/expo/expoweb +git clone ssh://expo@expo.survex.com/home/expo/drawings + +mkdir expofiles +rsync -azv --delete-after --prune-empty-dirs --exclude="photos" --exclude="video" expo@expo.survex.com:expofiles/ expofiles +rsync -azv --exclude="*.jpg.xml" --exclude="*.jpeg.xml" --exclude="*.JPG.xml" expo@expo.s^Cvex.com:expofiles/photos/ expofiles/photos + diff --git a/_deploy/xubuntu/venv-trog-d5.sh b/_deploy/xubuntu/venv-trog-d5.sh new file mode 100755 index 0000000..e70a49a --- /dev/null +++ b/_deploy/xubuntu/venv-trog-d5.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# footled lots to make this work with python 3.10 & 3.11 and WSL1 and WSL2 on Ubuntu 22.04 +# Run this in a terminal in the troggle directory: 'bash venv-trog.sh' +echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.sh"' + +# Expects an Ubuntu 22.04 (or 20.04) relatively clean install. +# If you have not already installed these on your clean Ubuntu install DO THIS FIRST +# use the script os-trog.sh + +# If you are using Debian, then stick with the default version of python +# If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.11 + +# NOW we set up troggle +PYTHON=python3.12 +VENAME=p12d5 # python3.x and django 4.2 +echo "** You are logged in as `id -u -n`" +echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/" +echo "The 150MB venv will created in /home/`id -u -n`/$VENAME/" +TROGDIR=$(cd $(dirname $0) && pwd) +echo "-- Troggle folder (this script location): ${TROGDIR}" + +REQUIRE=requirements-$VENAME.txt + +if [ -d $REQUIRE ]; then + echo "-- No ${REQUIRE} found. You should be in the /troggle/ folder. Copy it from your most recent installation." + exit 1 +fi +echo ## Using $REQUIRE : +cat $REQUIRE +echo ## + + +$PYTHON --version + +# NOTE that when using a later or earlier verison of python, you MUST also +# use the allowed version of Pillow, see https://pillow.readthedocs.io/en/latest/installation.html + +# NOW set up link from expo user folder +# needed for WSL2 +echo Creating links from Linux filesystem user +# These links only need making once, for many venv +cd ~ + +if [ ! -d $VENAME ]; then + echo "## Creating venv $VENAME. (If this fails with a pip error, you need to ensure you have python3.11-venv installed and/or use a Ubuntu window)" + $PYTHON -m venv $VENAME +else + echo "## /$VENAME/ already exists ! Delete it first." + exit 1 +fi + + # Activate the virtual env and see what the default packages are +echo "### Activating $VENAME" + +cd $VENAME +echo "-- now in: ${PWD}" +ls -tlarg +source bin/activate +echo $VIRTUAL_ENV +if [ -d ~/$VENAME/bin ]; then + echo "### Activating." +else + echo "## ~/$VENAME/bin does not exist. FAILed to create venv properly." + exit 1 +fi + # update local version of pip, more recent than OS version + # debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3 + + # update local version of setuptools, more recent than OS version, needed for packages without wheels + +echo "### installing later version of pip inside $VENAME" +$PYTHON -m pip install --upgrade pip +$PYTHON -m pip install --upgrade setuptools + +PIP=pip + +$PIP list > original-pip.list +$PIP freeze >original.txt + +# we are in /home/$USER/$VENAME/ +ln -s ${TROGDIR} troggle +ln -s ${TROGDIR}/../expoweb expoweb +ln -s ${TROGDIR}/../loser loser +ln -s ${TROGDIR}/../drawings drawings +#ln -s ${TROGDIR}/../expofiles expofiles + +# fudge for philip's machine +if [ ! -d /mnt/d/EXPO ]; then + sudo mkdir /mnt/d + sudo mount -t drvfs D: /mnt/d +fi + +if [ -d ${TROGDIR}/../expofiles ]; then + ln -s ${TROGDIR}/../expofiles expofiles +else + ln -s /mnt/d/EXPO/expofiles expofiles +fi + +echo "### Setting file permissions.. may take a while.." +git config --global --add safe.directory '*' +sudo chmod -R 777 * + +echo "### links to expoweb, troggle etc. complete:" +ls -tla +echo "###" +echo "### now installing ${TROGDIR}/${REQUIRE}" +echo "###" +cat ${TROGDIR}/${REQUIRE} + +# NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE +# seen on wsl2 as well as wsl1 +# which ALSO ruins EXISTING permissions ! +# Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04 + +read -p "Press any key to resume ..." +$PIP install -r ${TROGDIR}/${REQUIRE} +echo "### install from ${TROGDIR}/${REQUIRE} completed." +echo '### ' + +# this installs pre-release django 5.0 +$PIP install --pre django + +$PIP freeze > $REQUIRE +# so that we can track requirements more easily with git +# because we do not install these with pip, but they are listed by the freeze command +# Now find out what we actually installed by subtracting the stuff venv installed anyway +sort original.txt > 1 +sort $REQUIRE >2 +comm -3 1 2 --check-order | awk '{ print $1}'>fresh-$REQUIRE +rm 1 +rm 2 + +# cp $REQUIRE requirements-$VENAME.txt +cp $REQUIRE troggle/$REQUIRE + +$PIP list > installed-pip.list +$PIP list -o > installed-pip-o.list + +REQ=installation-record +mkdir $REQ + +mv original.txt $REQ +mv $REQUIRE $REQ +mv original-pip.list $REQ +mv installed-pip.list $REQ +mv installed-pip-o.list $REQ +cp fresh-$REQUIRE ../$REQUIRE +mv fresh-$REQUIRE $REQ +cp troggle/`basename "$0"` $REQ + +# install VS code +sudo apt install software-properties-common apt-transport-https wget gpg +wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg +sudo install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg +sudo sh -c 'echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" > /etc/apt/sources.list.d/vscode.list' +sudo apt update +sudo apt install code + +$PYTHON --version +python --version +echo "Django version:`django-admin --version`" + +echo "### Now do + '[sudo service mysql start]' + '[sudo service mariadb restart]' + '[sudo mysql_secure_installation]' + 'cd ~/$VENAME' + 'source bin/activate' + 'cd troggle' + 'django-admin' + 'python manage.py check' +## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html +## you need to follow the Linux instructions. + 'ssh expo@expo.survex.com' + +## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine +## the tests may ALSO fail because of ssh and permissions errors + +## So you will need to run +$sudo chown -Rhv philip:philip ~/$VENAME (if your username is philip) +# and then REBOOT (or at least, exit WSL and terminate and restart WSL) +# because this chmod only takes effect then. + + 'python manage.py test -v 2' + './pre-run.sh' (runs the tests again) + + 'python databaseReset.py reset $VENAME' + 'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)' + " +# if [ ! -d /mnt/d/expofiles ]; then + # echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.' +# fi diff --git a/requirements-p12d5.txt b/requirements-p12d5.txt new file mode 100755 index 0000000..fe10a93 --- /dev/null +++ b/requirements-p12d5.txt @@ -0,0 +1,23 @@ +asgiref==3.7.2 +setuptools==69.0.2 +sqlparse==0.4.4 +asgiref==3.6.0 +BeautifulSoup4==4.12.0 +black==23.1.0 +chardet==5.1.0 +click==8.1.0 +coverage==7.1.0 +deptry==0.12.0 +docutils==0.19 +isort==5.12.0 +mypy-extensions==1.0.0 +packaging==23.0 +pathspec==0.11.0 +piexif==1.1.3 +Pillow==9.4.0 +platformdirs==3.0.0 +pytz==2022.7 +ruff==0.0.245 +soupsieve==2.5 +sqlparse==0.4.0 +Unidecode==1.3.0 diff --git a/venv-trog-d5.sh b/venv-trog-d5.sh new file mode 100755 index 0000000..e70a49a --- /dev/null +++ b/venv-trog-d5.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# footled lots to make this work with python 3.10 & 3.11 and WSL1 and WSL2 on Ubuntu 22.04 +# Run this in a terminal in the troggle directory: 'bash venv-trog.sh' +echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.sh"' + +# Expects an Ubuntu 22.04 (or 20.04) relatively clean install. +# If you have not already installed these on your clean Ubuntu install DO THIS FIRST +# use the script os-trog.sh + +# If you are using Debian, then stick with the default version of python +# If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.11 + +# NOW we set up troggle +PYTHON=python3.12 +VENAME=p12d5 # python3.x and django 4.2 +echo "** You are logged in as `id -u -n`" +echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/" +echo "The 150MB venv will created in /home/`id -u -n`/$VENAME/" +TROGDIR=$(cd $(dirname $0) && pwd) +echo "-- Troggle folder (this script location): ${TROGDIR}" + +REQUIRE=requirements-$VENAME.txt + +if [ -d $REQUIRE ]; then + echo "-- No ${REQUIRE} found. You should be in the /troggle/ folder. Copy it from your most recent installation." + exit 1 +fi +echo ## Using $REQUIRE : +cat $REQUIRE +echo ## + + +$PYTHON --version + +# NOTE that when using a later or earlier verison of python, you MUST also +# use the allowed version of Pillow, see https://pillow.readthedocs.io/en/latest/installation.html + +# NOW set up link from expo user folder +# needed for WSL2 +echo Creating links from Linux filesystem user +# These links only need making once, for many venv +cd ~ + +if [ ! -d $VENAME ]; then + echo "## Creating venv $VENAME. (If this fails with a pip error, you need to ensure you have python3.11-venv installed and/or use a Ubuntu window)" + $PYTHON -m venv $VENAME +else + echo "## /$VENAME/ already exists ! Delete it first." + exit 1 +fi + + # Activate the virtual env and see what the default packages are +echo "### Activating $VENAME" + +cd $VENAME +echo "-- now in: ${PWD}" +ls -tlarg +source bin/activate +echo $VIRTUAL_ENV +if [ -d ~/$VENAME/bin ]; then + echo "### Activating." +else + echo "## ~/$VENAME/bin does not exist. FAILed to create venv properly." + exit 1 +fi + # update local version of pip, more recent than OS version + # debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3 + + # update local version of setuptools, more recent than OS version, needed for packages without wheels + +echo "### installing later version of pip inside $VENAME" +$PYTHON -m pip install --upgrade pip +$PYTHON -m pip install --upgrade setuptools + +PIP=pip + +$PIP list > original-pip.list +$PIP freeze >original.txt + +# we are in /home/$USER/$VENAME/ +ln -s ${TROGDIR} troggle +ln -s ${TROGDIR}/../expoweb expoweb +ln -s ${TROGDIR}/../loser loser +ln -s ${TROGDIR}/../drawings drawings +#ln -s ${TROGDIR}/../expofiles expofiles + +# fudge for philip's machine +if [ ! -d /mnt/d/EXPO ]; then + sudo mkdir /mnt/d + sudo mount -t drvfs D: /mnt/d +fi + +if [ -d ${TROGDIR}/../expofiles ]; then + ln -s ${TROGDIR}/../expofiles expofiles +else + ln -s /mnt/d/EXPO/expofiles expofiles +fi + +echo "### Setting file permissions.. may take a while.." +git config --global --add safe.directory '*' +sudo chmod -R 777 * + +echo "### links to expoweb, troggle etc. complete:" +ls -tla +echo "###" +echo "### now installing ${TROGDIR}/${REQUIRE}" +echo "###" +cat ${TROGDIR}/${REQUIRE} + +# NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE +# seen on wsl2 as well as wsl1 +# which ALSO ruins EXISTING permissions ! +# Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04 + +read -p "Press any key to resume ..." +$PIP install -r ${TROGDIR}/${REQUIRE} +echo "### install from ${TROGDIR}/${REQUIRE} completed." +echo '### ' + +# this installs pre-release django 5.0 +$PIP install --pre django + +$PIP freeze > $REQUIRE +# so that we can track requirements more easily with git +# because we do not install these with pip, but they are listed by the freeze command +# Now find out what we actually installed by subtracting the stuff venv installed anyway +sort original.txt > 1 +sort $REQUIRE >2 +comm -3 1 2 --check-order | awk '{ print $1}'>fresh-$REQUIRE +rm 1 +rm 2 + +# cp $REQUIRE requirements-$VENAME.txt +cp $REQUIRE troggle/$REQUIRE + +$PIP list > installed-pip.list +$PIP list -o > installed-pip-o.list + +REQ=installation-record +mkdir $REQ + +mv original.txt $REQ +mv $REQUIRE $REQ +mv original-pip.list $REQ +mv installed-pip.list $REQ +mv installed-pip-o.list $REQ +cp fresh-$REQUIRE ../$REQUIRE +mv fresh-$REQUIRE $REQ +cp troggle/`basename "$0"` $REQ + +# install VS code +sudo apt install software-properties-common apt-transport-https wget gpg +wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg +sudo install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg +sudo sh -c 'echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" > /etc/apt/sources.list.d/vscode.list' +sudo apt update +sudo apt install code + +$PYTHON --version +python --version +echo "Django version:`django-admin --version`" + +echo "### Now do + '[sudo service mysql start]' + '[sudo service mariadb restart]' + '[sudo mysql_secure_installation]' + 'cd ~/$VENAME' + 'source bin/activate' + 'cd troggle' + 'django-admin' + 'python manage.py check' +## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html +## you need to follow the Linux instructions. + 'ssh expo@expo.survex.com' + +## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine +## the tests may ALSO fail because of ssh and permissions errors + +## So you will need to run +$sudo chown -Rhv philip:philip ~/$VENAME (if your username is philip) +# and then REBOOT (or at least, exit WSL and terminate and restart WSL) +# because this chmod only takes effect then. + + 'python manage.py test -v 2' + './pre-run.sh' (runs the tests again) + + 'python databaseReset.py reset $VENAME' + 'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)' + " +# if [ ! -d /mnt/d/expofiles ]; then + # echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.' +# fi