upgrading scripts to use uv

_deploy/wsl/localsettingsWSL.py

@@ -95,7 +95,7 @@ SURVEXPORT = "survexport"  # for parsing .3d files and producing .pos files
 DBSQLITE = {
     "default": {
         "ENGINE": "django.db.backends.sqlite3",  # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
-        "NAME": "/home/philip/p11d5/troggle.sqlite",
+        "NAME": "/home/philip/expo/troggle.sqlite",
         #        'NAME' : ':memory:',
         "USER": "expo",  # Not used with sqlite3.
         "PASSWORD": "sekrit",  # Not used with sqlite3.

pre-push.sh

@@ -1,12 +1,16 @@
-#! /bin/sh
+#! /bin/bash
 # create and sanitise files for pushing to repo
 # catatrophically forgot to sanitize localsettingsWSL.py - oops.
 
 #Make sure you have the WSL permissions system working, or you will push unsanitized files as this will fail
 # Philip Sargent 2022/04/12
 echo "** This copies file to _deploy/wsl/ !"
-echo
+cd ..
+source .venv/bin/activate
+cd troggle
+echo `pwd`
 echo deprecations.
+
 python -Wall manage.py check -v 3 2>deprecations.txt >/dev/null
 echo diffsettings.
 rm diffsettings.txt
@@ -15,8 +19,8 @@ if test -f "diffsettings.txt"; then
     exit
 fi
 python manage.py diffsettings | grep "###" > diffsettings.txt
-echo pip freeze.
+echo uv pip freeze.
-pip freeze > requirements.txt
+uv pip freeze > requirements.txt
 echo inspectdb.
 # this next line requires database setting to be troggle.sqlite:
 python manage.py inspectdb > troggle-inspectdb.py
@@ -41,6 +45,7 @@ echo "    reset: SECRET_KEY = \"real-SECRET_KEY--imported-from-localsettings.py\
 
 mv _deploy/wsl/localsettingsWSL.py _deploy/wsl/localsettingsWSL.py.bak
 mv localsettingsWSL.py _deploy/wsl
+deactivate
 #
 # Do these before final testing, *not* just before pushing:
 # in ./pre-run.sh

pre-run.sh

@@ -1,8 +1,11 @@
 #! /bin/sh
 # Do these before final testing, *not* just before pushing:
 # Philip Sargent 2020/06/20
+# now with uv
+PYTHON="uv run"
+cd ..
 echo "** Run inspectdb:"
-python3 manage.py inspectdb > troggle-inspectdb.py
+$PYTHON troggle/manage.py inspectdb > troggle-inspectdb.py
 # egrep -in "unable|error" troggle-inspectdb.py
 echo ""
 # count non-blank lines of python and template HTML code
@@ -13,17 +16,17 @@ find . -name \*.py -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v ":0$" | gr
 
 
 # This deletes the database so must run after generating troggle-inspectdb.py
-python3 reset-django.py
+$PYTHON troggle/reset-django.py
 echo "** After cleanup deletion, remake all migrations."
-python3 manage.py makemigrations >/dev/null
+$PYTHON troggle/manage.py makemigrations >/dev/null
-python3 manage.py migrate
+$PYTHON troggle/manage.py migrate
 
 echo "** Now running self check"
-python3  manage.py check -v 3 --deploy 2>security-warnings.txt >/dev/null
+$PYTHON  troggle/manage.py check -v 3 --deploy 2>security-warnings.txt >/dev/null
-python3  manage.py check -v 3 --deploy
+$PYTHON  troggle/manage.py check -v 3 --deploy
 
 echo "** Now running test suite"
-python3 manage.py test
+$PYTHON troggle/manage.py test -v 1
 
 echo ""
 echo `tail -1 lines-of-python.txt`   non-comment lines of python.

requirements-keep.txt

@@ -0,0 +1,44 @@
+For debian bullseye (10)
+python3-django (3.2.12)
+tinymce (from where?)
+mariadb-server
+apache2
+libapache2-mod-wsgi-py3
+
+
+python3-django-registration ?
+Django==1.7
+django-extensions==2.2.9
+django-registration==2.0
+django-tinymce==2.0.1
+six==1.14.0 
+Unidecode==1.1.1 python3-unidecode
+Pillow==7.1.2  python3-willow
+
+asgiref==3.7.2
+attrs==22.2.0
+beautifulsoup4==4.12.2
+black==23.11.0
+bs4==0.0.1
+click==8.1.3
+colorama==0.4.6
+coverage==6.5.0
+Django==5.0
+docutils==0.20
+interrogate==1.5.0
+isort==5.11.4
+mypy-extensions==0.4.3
+packaging==23.2
+pathspec==0.10.3
+piexif==1.1.3
+Pillow==10.1.0
+platformdirs==2.6.2
+py==1.11.0
+pytz==2022.6
+ruff==0.1.0
+soupsieve==2.5
+sqlparse==0.4.0
+tabulate==0.9.0
+toml==0.10.2
+typing_extensions==4.4.0
+Unidecode==1.3.6

requirements.txt

@@ -1,44 +1,19 @@
-For debian bullseye (10)
+asgiref==3.8.1
-python3-django (3.2.12)
+beautifulsoup4==4.12.3
-tinymce (from where?)
+black==24.10.0
-mariadb-server
+click==8.1.7
-apache2
+coverage==7.6.9
-libapache2-mod-wsgi-py3
+deptry==0.21.1
-
+django==5.1.4
-
+isort==5.13.2
-python3-django-registration ?
+mypy-extensions==1.0.0
-Django==1.7
+packaging==24.2
-django-extensions==2.2.9
+pathspec==0.12.1
-django-registration==2.0
-django-tinymce==2.0.1
-six==1.14.0 
-Unidecode==1.1.1 python3-unidecode
-Pillow==7.1.2  python3-willow
-
-asgiref==3.7.2
-attrs==22.2.0
-beautifulsoup4==4.12.2
-black==23.11.0
-bs4==0.0.1
-click==8.1.3
-colorama==0.4.6
-coverage==6.5.0
-Django==5.0
-docutils==0.20
-interrogate==1.5.0
-isort==5.11.4
-mypy-extensions==0.4.3
-packaging==23.2
-pathspec==0.10.3
 piexif==1.1.3
-Pillow==10.1.0
+pillow==11.0.0
-platformdirs==2.6.2
+platformdirs==4.3.6
-py==1.11.0
+requirements-parser==0.11.0
-pytz==2022.6
+soupsieve==2.6
-ruff==0.1.0
+sqlparse==0.5.3
-soupsieve==2.5
+types-setuptools==75.6.0.20241126
-sqlparse==0.4.0
+unidecode==1.3.8
-tabulate==0.9.0
-toml==0.10.2
-typing_extensions==4.4.0
-Unidecode==1.3.6

security-warnings.txt

@@ -1,10 +1,44 @@
-System check identified some issues:
+Traceback (most recent call last):
-
+  File "/home/philip/expo/troggle/manage.py", line 23, in <module>
-WARNINGS:
+    execute_from_command_line(sys.argv)
-?: (security.W001) You do not have 'django.middleware.security.SecurityMiddleware' in your MIDDLEWARE so the SECURE_HSTS_SECONDS, SECURE_CONTENT_TYPE_NOSNIFF, SECURE_REFERRER_POLICY, SECURE_CROSS_ORIGIN_OPENER_POLICY, and SECURE_SSL_REDIRECT settings will have no effect.
+    ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
-?: (security.W009) Your SECRET_KEY has less than 50 characters, less than 5 unique characters, or it's prefixed with 'django-insecure-' indicating that it was generated automatically by Django. Please generate a long and random value, otherwise many of Django's security-critical features will be vulnerable to attack.
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/core/management/__init__.py", line 442, in execute_from_command_line
-?: (security.W012) SESSION_COOKIE_SECURE is not set to True. Using a secure-only session cookie makes it more difficult for network traffic sniffers to hijack user sessions.
+    utility.execute()
-?: (security.W016) You have 'django.middleware.csrf.CsrfViewMiddleware' in your MIDDLEWARE, but you have not set CSRF_COOKIE_SECURE to True. Using a secure-only CSRF cookie makes it more difficult for network traffic sniffers to steal the CSRF token.
+    ~~~~~~~~~~~~~~~^^
-?: (security.W018) You should not have DEBUG set to True in deployment.
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/core/management/__init__.py", line 416, in execute
-
+    django.setup()
-System check identified 5 issues (0 silenced).
+    ~~~~~~~~~~~~^^
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/__init__.py", line 24, in setup
+    apps.populate(settings.INSTALLED_APPS)
+    ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/apps/registry.py", line 91, in populate
+    app_config = AppConfig.create(entry)
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/apps/config.py", line 123, in create
+    mod = import_module(mod_path)
+  File "/home/philip/.local/share/uv/python/cpython-3.13.1-linux-x86_64-gnu/lib/python3.13/importlib/__init__.py", line 88, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+           ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
+  File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
+  File "<frozen importlib._bootstrap_external>", line 1026, in exec_module
+  File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/contrib/auth/apps.py", line 8, in <module>
+    from .checks import check_middleware, check_models_permissions, check_user_model
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/contrib/auth/checks.py", line 9, in <module>
+    from .management import _get_builtin_permissions
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/contrib/auth/management/__init__.py", line 10, in <module>
+    from django.contrib.contenttypes.management import create_contenttypes
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/contrib/contenttypes/management/__init__.py", line 2, in <module>
+    from django.db import DEFAULT_DB_ALIAS, IntegrityError, migrations, router, transaction
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/db/migrations/__init__.py", line 2, in <module>
+    from .operations import *  # NOQA
+    ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/db/migrations/operations/__init__.py", line 2, in <module>
+    from .models import (
+    ...<15 lines>...
+    )
+  File "/home/philip/expo/troggle/.venv/lib/python3.13/site-packages/django/db/migrations/operations/models.py", line 3, in <module>
+    from django.db.migrations.state import ModelState
+ModuleNotFoundError: No module named 'django.db.migrations.state'

venv-trog.sh

@@ -1,19 +1,17 @@
 #!/bin/bash
-# footled lots to make this work with python 3.10 & 3.11 and WSL1 and WSL2 on Ubuntu 22.04
+# now using uv, unbelieveably simpler.
 # Run this in a terminal in the troggle directory: 'bash venv-trog.sh'
 echo '-- Run this in a terminal in the directory above the troggle directory: "bash troggle/venv-trog.sh"'
 
 # Expects an Ubuntu 22.04 with all the gubbins already installed
 # If you have not already installed these on your clean Ubuntu install DO THIS FIRST
-# use the script os-trog24.04.sh
+# use the script os-trog24.04.sh runniing it in /home/username/
 python --version
 
 echo "-- EXPO    folder [current directory]:    `pwd`"
-
 TROGDIR=$(cd $(dirname $0) && pwd)
 echo "-- Troggle folder [this script location]: ${TROGDIR}"
 
-
 cp troggle/pyproject.toml .
 
 uv self update
@@ -24,31 +22,11 @@ exit 1
 # this next is a bit wordy
 # uv python list --only-installed
 
-# NOW we set up troggle
+# we are in  /home/$USER/expo/
-PYTHONSVR=python3.11
+# ln -s ${TROGDIR} troggle
-PYTHONDEV=python3.13
+# ln -s ${TROGDIR}/../expoweb expoweb
-
+# ln -s ${TROGDIR}/../loser loser
-VSVR=svr # python3.11 and django 3.2
+# ln -s ${TROGDIR}/../drawings drawings
-VDEV=dev # python3.13 and django 5
-#echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/"
-# echo "-- venvs will created in  /home/`id -u -n`/expo/$VSVR/ " "and  /home/`id -u -n`/expo/$VDEV/"
-
-# This will install uv copies in .local as well as creating venv
-#uv venv $VSVR --python 3.11
-uv venv $VDEV --python 3.13
-# uv python list --only-installed
-
-
-exit 1
-
-
-
-
-# we are in  /home/$USER/$VSVR/
-ln -s ${TROGDIR} troggle
-ln -s ${TROGDIR}/../expoweb expoweb
-ln -s ${TROGDIR}/../loser loser
-ln -s ${TROGDIR}/../drawings drawings
 #ln -s ${TROGDIR}/../expofiles expofiles
 
 # fudge for philip's laptop prior to M2 SSD upgrade
@@ -63,88 +41,34 @@ else
   ln -s /mnt/d/EXPO/expofiles expofiles
 fi
 
-echo "### Setting file permissions.. may take a while.."
-git config --global --add safe.directory '*'
-sudo chmod -R 777 *
-
-echo "### links to expoweb, troggle etc. complete:"
-ls -tla
-echo "###"
-echo "### now installing ${TROGDIR}/${REQUIRE}"
-echo "###"
-cat ${TROGDIR}/${REQUIRE}
-
-# NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE
-# seen on wsl2 as well as wsl1 
-# which ALSO ruins EXISTING permissions !
-# Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04
-
 read -p "Press any key to resume ..."
-$PIP install -r ${TROGDIR}/${REQUIRE}
+PIP="uv pip"
-echo "### install from ${TROGDIR}/${REQUIRE} completed."
-echo '### '
 
-$PIP install --pre django
 
+REQUIRE="requirements.txt"
 $PIP freeze > $REQUIRE  
-# so that we can track requirements more easily with git
+$PIP list 
-# because we do not install these with pip, but they are listed by the freeze command
-# Now find out what we actually installed by subtracting the stuff venv installed anyway
-sort original.txt > 1
-sort $REQUIRE >2
-comm -3 1 2 --check-order | awk '{ print $1}'>fresh-$REQUIRE
-rm 1
-rm 2
 
-# cp $REQUIRE requirements-$VSVR.txt 
+echo "Django version:`uv run django-admin --version`"
-cp $REQUIRE troggle/$REQUIRE
-
-$PIP list > installed-pip.list
-$PIP list -o > installed-pip-o.list
-
-REQ=installation-record
-mkdir $REQ 
-
-mv original.txt $REQ
-mv $REQUIRE $REQ
-mv original-pip.list $REQ
-mv installed-pip.list $REQ
-mv installed-pip-o.list $REQ
-cp fresh-$REQUIRE ../$REQUIRE
-mv fresh-$REQUIRE $REQ
-cp troggle/`basename "$0"` $REQ
-
-
-$PYTHON --version
-python --version
-echo "Django version:`django-admin --version`"
 
 echo "### Now do 
   '[sudo service mysql start]'
   '[sudo service mariadb restart]'
   '[sudo mysql_secure_installation]'
-  'cd ~/$VSVR'
-  'source bin/activate'
   'cd troggle'
-  'django-admin'
+  'uv run django-admin'
-  'python manage.py check'
+  'uv run manage.py check'
 ## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html
 ## you need to follow the Linux instructions.
   'ssh expo@expo.survex.com'
 
 ## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine
-## the tests may ALSO fail because of ssh and permissions errors 
 
-## So you will need to run 
+   'uv run  manage.py test -v 2'
-$sudo chown -Rhv philip:philip ~/$VSVR  (if your username is philip)
-# and then REBOOT (or at least, exit WSL and terminate and restart WSL)
-# because this chmod only takes effect then.
-
-   'python manage.py test -v 2'
    './pre-run.sh' (runs the tests again)
    
-   'python databaseReset.py reset $VSVR'
+   'uv run  databaseReset.py reset INIT'
-   'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
+   'uv run  manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
    "
 # if [ ! -d /mnt/d/expofiles ]; then
   # echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.'