fixing setup scripts for a new Ubuntu 24.04 install

_deploy/wsl/localsettingsWSL.py

@@ -25,19 +25,17 @@ print(" * importing troggle/localsettings.py")
 #  - have you checked that credentials.py is in .gitignore ?
 #  - we don't want to have to change the expo system password !
 # -----------------------------------------------------------------
-# default values,  real secrets will be imported from credentials.py in future
+# default values,  real secrets imported from credentials.py
 
 SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py"
 EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py"
 EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"
 EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"
 
-SQLITEFILE = "/home/philip/expo/troggle.sqlite"  #  can be ':memory:'
+EXPOFILESREMOTE = False  # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate.
-
-PHOTOSREMOTE = False  # if True, then re-routes urls in expofiles/photos to remote server. Not implemented yet
-EXPOFILESREMOTE = False  # if True, then re-routes urls in expofiles to remote server. Tests are then less accurate.
 # SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely
-SERVERPORT = "8000"  # not needed as it is the default
+
+SERVERPORT = "8000"  # not needed
 
 PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor)
 
@@ -97,7 +95,7 @@ SURVEXPORT = "survexport"  # for parsing .3d files and producing .pos files
 DBSQLITE = {
     "default": {
         "ENGINE": "django.db.backends.sqlite3",  # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
-        "NAME": SQLITEFILE,
+        "NAME": "/home/philip/p12d5/troggle.sqlite",
         #        'NAME' : ':memory:',
         "USER": "expo",  # Not used with sqlite3.
         "PASSWORD": "sekrit",  # Not used with sqlite3.

_deploy/wsl/pre-push.sh

@@ -0,0 +1,57 @@
+#! /bin/bash
+# create and sanitise files for pushing to repo
+# catatrophically forgot to sanitize localsettingsWSL.py - oops.
+
+#Make sure you have the WSL permissions system working, or you will push unsanitized files as this will fail
+# Philip Sargent 2022/04/12
+echo "** This copies file to _deploy/wsl/ !"
+cd ..
+source .venv/bin/activate
+cd troggle
+echo `pwd`
+echo deprecations.
+
+python -Wall manage.py check -v 3 2>deprecations.txt >/dev/null
+echo diffsettings.
+rm diffsettings.txt
+if test -f "diffsettings.txt"; then
+    echo "diffsettings.txt not deleted. You have a serious permissions problem. Aborting.."
+    exit
+fi
+python manage.py diffsettings | grep "###" > diffsettings.txt
+echo uv pip freeze.
+uv pip freeze > requirements.txt
+echo inspectdb.
+# this next line requires database setting to be troggle.sqlite:
+python manage.py inspectdb > troggle-inspectdb.py
+#egrep -in "unable|error" troggle-inspectdb.py
+echo remove passwords.
+cp localsettings.py localsettingsWSL.py
+sed -i '/EXPOUSERPASS/ s/^.*$/EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py"/' diffsettings.txt
+sed -i '/EXPOUSERPASS/ s/^.*$/EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py"/' localsettingsWSL.py
+echo "    reset: EXPOUSERPASS = \"nnn:gggggg\"  - real-expo-password---imported-from-localsettings.py"
+
+sed -i '/EXPOADMINUSERPASS/ s/^.*$/EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"/' diffsettings.txt
+sed -i '/EXPOADMINUSERPASS/ s/^.*$/EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"/' localsettingsWSL.py
+echo "    reset: EXPOUSERPASS = \"gggggg:nnn\"  - real-expo-password---imported-from-localsettings.py"
+
+sed -i '/EMAIL_HOST_PASSWORD/ s/^.*$/EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"/' diffsettings.txt
+sed -i '/EMAIL_HOST_PASSWORD/ s/^.*$/EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"/' localsettingsWSL.py
+echo "    reset: EMAIL_HOST_PASSWORD = \"real-email-password--imported-from-localsettings.py\""
+
+sed -i '/SECRET_KEY/ s/^.*$/SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py"/' diffsettings.txt
+sed -i '/SECRET_KEY/ s/^.*$/SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py"/' localsettingsWSL.py
+echo "    reset: SECRET_KEY = \"real-SECRET_KEY--imported-from-localsettings.py\""
+
+mv _deploy/wsl/localsettingsWSL.py _deploy/wsl/localsettingsWSL.py.bak
+mv localsettingsWSL.py _deploy/wsl
+cp *.sh _deploy/wsl
+deactivate
+#
+# Do these before final testing, *not* just before pushing:
+# in ./pre-run.sh
+# python reset-django.py
+# python manage.py makemigrations
+# python manage.py test
+# python manage.py inspectdb > troggle-inspectdb.py
+# egrep -i "unable|error" troggle-inspectdb.py

_deploy/wsl/pre-run.sh

@@ -0,0 +1,36 @@
+#! /bin/bash
+# Do these before final testing, *not* just before pushing:
+# Need to be in an ALREADY activated venv
+PYTHON="python"
+
+echo "** Run inspectdb:"
+$PYTHON manage.py inspectdb > troggle-inspectdb.py
+# egrep -in "unable|error" troggle-inspectdb.py
+echo ""
+# count non-blank lines of python and template HTML code
+# includes all variants of settings.py files
+
+# fix this as core/utils.py has 28,000 lines of numbers.
+find . -name \*.html -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v ":0$" | grep -v ".venv" | awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-templates.txt
+
+find . -name \*.py -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v ":0$" | grep -v ".venv" | grep -v "/migrations/" |grep -v "troggle-inspectdb.py"| awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-python.txt
+
+echo "** Run reset-django.py - which deletes the database"
+# This deletes the database so must run after generating troggle-inspectdb.py
+$PYTHON reset-django.py
+echo "** After cleanup deletion, remake all migrations."
+$PYTHON manage.py makemigrations >/dev/null
+$PYTHON manage.py migrate
+
+echo "** Now running self check"
+$PYTHON  manage.py check -v 3 --deploy 2>security-warnings.txt >/dev/null
+$PYTHON  manage.py check -v 3 --deploy
+
+echo "** Now running test suite"
+# $PYTHON manage.py test -v 1
+
+echo ""
+echo `tail -1 lines-of-python.txt`   non-comment lines of python. But core/utils.py has 28,000 lines of numbers.
+echo `tail -1 lines-of-templates.txt`   non-comment lines of HTML templates.
+
+echo '** If you have an error running manage.py, maybe you are not in an activated venv ?'

_deploy/wsl/venv-trog.sh

@@ -1,90 +1,35 @@
 #!/bin/bash
-# footled lots to make this work with python 3.10 & 3.11 and WSL1 and WSL2 on Ubuntu 22.04
+# now using uv, unbelieveably simpler.
 # Run this in a terminal in the troggle directory: 'bash venv-trog.sh'
-echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.sh"'
+echo '-- Run this in a terminal in the directory above the troggle directory: "bash troggle/venv-trog.sh"'
 
 # Expects an Ubuntu 22.04 with all the gubbins already installed
 # If you have not already installed these on your clean Ubuntu install DO THIS FIRST
-# use the script os-trog24.04.sh
+# use the script os-trog24.04.sh runniing it in /home/username/
+python --version
 
-# NOW we set up troggle
+echo "-- EXPO    folder [current directory]:    `pwd`"
-PYTHONSVR=python3.11
-PYTHONDEV=python3.13
-
-VSVR=svr # python3.11 and django 3.2
-VDEV=dev # python3.13 and django 5
-echo "** You are logged in as `id -u -n`"
-#echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/"
-echo "The 150MB venv will created in  /home/`id -u -n`/$VSVR/ and  /home/`id -u -n`/$VDEV/
 TROGDIR=$(cd $(dirname $0) && pwd)
-echo "-- Troggle folder (this script location): ${TROGDIR}"
+echo "-- Troggle folder [this script location]: ${TROGDIR}"
-exit
-REQUIRE=requirements-$VSVR.txt
 
-if [ -d $REQUIRE ]; then
+cp troggle/pyproject.toml .
-  echo "-- No ${REQUIRE} found. You should be in the /troggle/ folder. Copy it from your most recent installation."
+
-  exit 1
+uv self update
-fi
+uv sync
-echo ## Using $REQUIRE :
-cat $REQUIRE
-echo ##
 
 
-$PYTHON --version
+exit 1
+# this next is a bit wordy
+# uv python list --only-installed
 
-# NOTE that when using a later or earlier verison of python, you MUST also
+# we are in  /home/$USER/expo/
-# use the allowed version of Pillow, see https://pillow.readthedocs.io/en/latest/installation.html
+# ln -s ${TROGDIR} troggle
-
+# ln -s ${TROGDIR}/../expoweb expoweb
-# NOW set up link from expo user folder
+# ln -s ${TROGDIR}/../loser loser
-# needed for WSL2
+# ln -s ${TROGDIR}/../drawings drawings
-echo Creating links from Linux filesystem user 
-# These links only need making once, for many venv
-cd ~
-
-if [ ! -d $VSVR ]; then
-  echo "## Creating venv $VSVR. (If this fails with a pip error, you need to ensure you have python3.11-venv installed and/or use a Ubuntu window)"
-  $PYTHON -m venv $VSVR
-else 
-  echo "## /$VSVR/ already exists ! Delete it first."
-  exit 1
-fi
-
-  # Activate the virtual env and see what the default packages are
-echo "### Activating $VSVR"
-
-cd $VSVR
-echo "-- now in: ${PWD}"
-ls -tlarg
-source bin/activate
-echo $VIRTUAL_ENV 
-if [ -d ~/$VSVR/bin ]; then
-    echo "### Activating."
-else
-  echo "## ~/$VSVR/bin does not exist. FAILed to create venv properly."
-  exit 1
-fi
-  # update local version of pip, more recent than OS version
-  # debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3
-  
-  # update local version of setuptools, more recent than OS version, needed for packages without wheels
-  
-echo "### installing later version of pip inside $VSVR"
-$PYTHON -m pip install --upgrade pip
-$PYTHON -m pip install --upgrade setuptools
-
-PIP=pip
-
-$PIP list > original-pip.list
-$PIP freeze >original.txt
-
-# we are in  /home/$USER/$VSVR/
-ln -s ${TROGDIR} troggle
-ln -s ${TROGDIR}/../expoweb expoweb
-ln -s ${TROGDIR}/../loser loser
-ln -s ${TROGDIR}/../drawings drawings
 #ln -s ${TROGDIR}/../expofiles expofiles
 
-# fudge for philip's machine
+# fudge for philip's laptop prior to M2 SSD upgrade
 if [ ! -d /mnt/d/EXPO ]; then
   sudo mkdir /mnt/d
   sudo mount -t drvfs D: /mnt/d
@@ -96,88 +41,34 @@ else
   ln -s /mnt/d/EXPO/expofiles expofiles
 fi
 
-echo "### Setting file permissions.. may take a while.."
-git config --global --add safe.directory '*'
-sudo chmod -R 777 *
-
-echo "### links to expoweb, troggle etc. complete:"
-ls -tla
-echo "###"
-echo "### now installing ${TROGDIR}/${REQUIRE}"
-echo "###"
-cat ${TROGDIR}/${REQUIRE}
-
-# NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE
-# seen on wsl2 as well as wsl1 
-# which ALSO ruins EXISTING permissions !
-# Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04
-
 read -p "Press any key to resume ..."
-$PIP install -r ${TROGDIR}/${REQUIRE}
+PIP="uv pip"
-echo "### install from ${TROGDIR}/${REQUIRE} completed."
-echo '### '
 
-$PIP install --pre django
 
+REQUIRE="requirements.txt"
 $PIP freeze > $REQUIRE  
-# so that we can track requirements more easily with git
+$PIP list 
-# because we do not install these with pip, but they are listed by the freeze command
-# Now find out what we actually installed by subtracting the stuff venv installed anyway
-sort original.txt > 1
-sort $REQUIRE >2
-comm -3 1 2 --check-order | awk '{ print $1}'>fresh-$REQUIRE
-rm 1
-rm 2
 
-# cp $REQUIRE requirements-$VSVR.txt 
+echo "Django version:`uv run django-admin --version`"
-cp $REQUIRE troggle/$REQUIRE
-
-$PIP list > installed-pip.list
-$PIP list -o > installed-pip-o.list
-
-REQ=installation-record
-mkdir $REQ 
-
-mv original.txt $REQ
-mv $REQUIRE $REQ
-mv original-pip.list $REQ
-mv installed-pip.list $REQ
-mv installed-pip-o.list $REQ
-cp fresh-$REQUIRE ../$REQUIRE
-mv fresh-$REQUIRE $REQ
-cp troggle/`basename "$0"` $REQ
-
-
-$PYTHON --version
-python --version
-echo "Django version:`django-admin --version`"
 
 echo "### Now do 
   '[sudo service mysql start]'
   '[sudo service mariadb restart]'
   '[sudo mysql_secure_installation]'
-  'cd ~/$VSVR'
-  'source bin/activate'
   'cd troggle'
-  'django-admin'
+  'uv run django-admin'
-  'python manage.py check'
+  'uv run manage.py check'
 ## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html
 ## you need to follow the Linux instructions.
   'ssh expo@expo.survex.com'
 
 ## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine
-## the tests may ALSO fail because of ssh and permissions errors 
 
-## So you will need to run 
+   'uv run  manage.py test -v 2'
-$sudo chown -Rhv philip:philip ~/$VSVR  (if your username is philip)
-# and then REBOOT (or at least, exit WSL and terminate and restart WSL)
-# because this chmod only takes effect then.
-
-   'python manage.py test -v 2'
    './pre-run.sh' (runs the tests again)
    
-   'python databaseReset.py reset $VSVR'
+   'uv run  databaseReset.py reset INIT'
-   'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
+   'uv run  manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
    "
 # if [ ! -d /mnt/d/expofiles ]; then
   # echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.'

md5bash.sh

@@ -1,79 +0,0 @@
-#!/bin/bash
-
-# read in the input string from command line arguments
-input=$1
-
-# pad the input string with a single "1" bit
-padded_input="$input"$(echo -ne '\x80')
-
-# pad the input string to a multiple of 512 bits (64 bytes)
-while (( $(echo -n "$padded_input" | wc -c) % 64 != 56 ))
-do
-  padded_input="$padded_input"$(echo -ne '\x00')
-done
-
-# append the length of the input string (in bits) as a 64-bit little-endian integer
-length=$(echo -n "$input" | wc -c)
-length=$(echo "obase=16; $length * 8" | bc | xxd -p -c 16 | tac | tr -d '\n')
-while (( $(echo -n "$length" | wc -c) < 16 ))
-do
-  length="$length"0
-done
-padded_input="$padded_input"$(echo -ne "$length" | xxd -r -p)
-
-# initialize the buffer (A, B, C, D)
-A="67452301"
-B="efcdab89"
-C="98badcfe"
-D="10325476"
-
-# process the input in 512-bit (64-byte) chunks
-for (( i=0; i<$(echo -n "$padded_input" | wc -c)/64; i++ ))
-do
-  chunk=$(echo -n "$padded_input" | dd bs=64 skip=$i count=1 2>/dev/null | xxd -p -c 64)
-
-  # initialize the message schedule (M)
-  M=()
-  for (( j=0; j<16; j++ ))
-  do
-    word=$(echo -ne "${chunk:$j*8:8}" | xxd -r -p | od -An -tu4 -v)
-    M+=($word)
-  done
-  for (( j=16; j<64; j++ ))
-  do
-    word1=${M[j-15]}
-    s0=$(( (word1>>7 | word1<<25) ^ (word1>>18 | word1<<14) ^ (word1>>3) ))
-
-    word2=${M[j-2]}
-    s1=$(( (word2>>17 | word2<<15) ^ (word2>>19 | word2<<13) ^ (word2>>10) ))
-
-    M+=($((M[j-16] + s0 + M[j-7] + s1)))
-  done
-
-  # initialize the working variables
-  AA=$A
-  BB=$B
-  CC=$C
-  DD=$D
-
-  # round 1
-  for (( j=0; j<16; j++ ))
-  do
-    F=$(( (B & C) | (~B & D) ))
-    g=$j
-    dTemp=$((D))
-    D=$((C))
-    C=$((B))
-    B=$((B + ((A + F + M[g] + 0xd76aa478) & 0xffffffff)))
-    A=$((dTemp))
-  done
-
-  # round 2
-  for (( j=16; j<32; j++ ))
-  do
-    F=$(( (D & B) | (~D & C) ))
-    g=$(( (5*j + 1) % 16 ))
-    dTemp=$((D))
-    D=$((C))
-    C=$((B))
-   

pre-push.sh

@@ -45,6 +45,7 @@ echo "    reset: SECRET_KEY = \"real-SECRET_KEY--imported-from-localsettings.py\
 
 mv _deploy/wsl/localsettingsWSL.py _deploy/wsl/localsettingsWSL.py.bak
 mv localsettingsWSL.py _deploy/wsl
+cp *.sh _deploy/wsl
 deactivate
 #
 # Do these before final testing, *not* just before pushing:

requirements.txt

@@ -1,19 +1,32 @@
-asgiref==3.8.1
+annotated-types==0.6.0
-beautifulsoup4==4.12.3
+asgiref==3.7.2
-black==24.10.0
+beautifulsoup4==4.12.0
-click==8.1.7
+black==24.2.0
-coverage==7.6.9
+chardet==5.2.0
-deptry==0.21.1
+click==8.1.0
-django==5.1.4
+coverage==7.0.0
+deptry==0.12.0
+django==5.0.4
+docutils==0.20
+fawltydeps==0.15.0
+importlib-metadata==7.0.1
 isort==5.13.2
 mypy-extensions==1.0.0
-packaging==24.2
+packaging==23.0
-pathspec==0.12.1
+pathspec==0.12.0
 piexif==1.1.3
-pillow==11.0.0
+pillow==10.2.0
-platformdirs==4.3.6
+pip==24.0
-requirements-parser==0.11.0
+pip-requirements-parser==32.0.1
-soupsieve==2.6
+platformdirs==4.0.0
-sqlparse==0.5.3
+pydantic==2.6.2
-types-setuptools==75.6.0.20241126
+pydantic-core==2.16.3
+pyparsing==3.1.1
+pytz==2024.1
+ruff==0.2.0
+setuptools==69.1.0
+soupsieve==2.5
+sqlparse==0.4.4
+typing-extensions==4.9.0
 unidecode==1.3.8
+zipp==3.17.0