2
0
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-12-15 10:37:07 +00:00

fixing setup scripts for a new Ubuntu 24.04 install

This commit is contained in:
2024-12-20 16:03:28 +00:00
parent 011e6777c9
commit 1b6333fdef
7 changed files with 155 additions and 238 deletions

View File

@@ -25,19 +25,17 @@ print(" * importing troggle/localsettings.py")
# - have you checked that credentials.py is in .gitignore ?
# - we don't want to have to change the expo system password !
# -----------------------------------------------------------------
# default values, real secrets will be imported from credentials.py in future
# default values, real secrets imported from credentials.py
SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py"
EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py"
EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"
EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"
SQLITEFILE = "/home/philip/expo/troggle.sqlite" # can be ':memory:'
PHOTOSREMOTE = False # if True, then re-routes urls in expofiles/photos to remote server. Not implemented yet
EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote server. Tests are then less accurate.
EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate.
# SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely
SERVERPORT = "8000" # not needed as it is the default
SERVERPORT = "8000" # not needed
PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor)
@@ -97,7 +95,7 @@ SURVEXPORT = "survexport" # for parsing .3d files and producing .pos files
DBSQLITE = {
"default": {
"ENGINE": "django.db.backends.sqlite3", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
"NAME": SQLITEFILE,
"NAME": "/home/philip/p12d5/troggle.sqlite",
# 'NAME' : ':memory:',
"USER": "expo", # Not used with sqlite3.
"PASSWORD": "sekrit", # Not used with sqlite3.

57
_deploy/wsl/pre-push.sh Normal file
View File

@@ -0,0 +1,57 @@
#! /bin/bash
# create and sanitise files for pushing to repo
# catatrophically forgot to sanitize localsettingsWSL.py - oops.
#Make sure you have the WSL permissions system working, or you will push unsanitized files as this will fail
# Philip Sargent 2022/04/12
echo "** This copies file to _deploy/wsl/ !"
cd ..
source .venv/bin/activate
cd troggle
echo `pwd`
echo deprecations.
python -Wall manage.py check -v 3 2>deprecations.txt >/dev/null
echo diffsettings.
rm diffsettings.txt
if test -f "diffsettings.txt"; then
echo "diffsettings.txt not deleted. You have a serious permissions problem. Aborting.."
exit
fi
python manage.py diffsettings | grep "###" > diffsettings.txt
echo uv pip freeze.
uv pip freeze > requirements.txt
echo inspectdb.
# this next line requires database setting to be troggle.sqlite:
python manage.py inspectdb > troggle-inspectdb.py
#egrep -in "unable|error" troggle-inspectdb.py
echo remove passwords.
cp localsettings.py localsettingsWSL.py
sed -i '/EXPOUSERPASS/ s/^.*$/EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py"/' diffsettings.txt
sed -i '/EXPOUSERPASS/ s/^.*$/EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py"/' localsettingsWSL.py
echo " reset: EXPOUSERPASS = \"nnn:gggggg\" - real-expo-password---imported-from-localsettings.py"
sed -i '/EXPOADMINUSERPASS/ s/^.*$/EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"/' diffsettings.txt
sed -i '/EXPOADMINUSERPASS/ s/^.*$/EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"/' localsettingsWSL.py
echo " reset: EXPOUSERPASS = \"gggggg:nnn\" - real-expo-password---imported-from-localsettings.py"
sed -i '/EMAIL_HOST_PASSWORD/ s/^.*$/EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"/' diffsettings.txt
sed -i '/EMAIL_HOST_PASSWORD/ s/^.*$/EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"/' localsettingsWSL.py
echo " reset: EMAIL_HOST_PASSWORD = \"real-email-password--imported-from-localsettings.py\""
sed -i '/SECRET_KEY/ s/^.*$/SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py"/' diffsettings.txt
sed -i '/SECRET_KEY/ s/^.*$/SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py"/' localsettingsWSL.py
echo " reset: SECRET_KEY = \"real-SECRET_KEY--imported-from-localsettings.py\""
mv _deploy/wsl/localsettingsWSL.py _deploy/wsl/localsettingsWSL.py.bak
mv localsettingsWSL.py _deploy/wsl
cp *.sh _deploy/wsl
deactivate
#
# Do these before final testing, *not* just before pushing:
# in ./pre-run.sh
# python reset-django.py
# python manage.py makemigrations
# python manage.py test
# python manage.py inspectdb > troggle-inspectdb.py
# egrep -i "unable|error" troggle-inspectdb.py

36
_deploy/wsl/pre-run.sh Normal file
View File

@@ -0,0 +1,36 @@
#! /bin/bash
# Do these before final testing, *not* just before pushing:
# Need to be in an ALREADY activated venv
PYTHON="python"
echo "** Run inspectdb:"
$PYTHON manage.py inspectdb > troggle-inspectdb.py
# egrep -in "unable|error" troggle-inspectdb.py
echo ""
# count non-blank lines of python and template HTML code
# includes all variants of settings.py files
# fix this as core/utils.py has 28,000 lines of numbers.
find . -name \*.html -print0 | xargs -0 egrep -vc "#|^\s*$" | grep -v ":0$" | grep -v ".venv" | awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-templates.txt
find . -name \*.py -print0 | xargs -0 egrep -vc "#|^\s*$" | grep -v ":0$" | grep -v ".venv" | grep -v "/migrations/" |grep -v "troggle-inspectdb.py"| awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-python.txt
echo "** Run reset-django.py - which deletes the database"
# This deletes the database so must run after generating troggle-inspectdb.py
$PYTHON reset-django.py
echo "** After cleanup deletion, remake all migrations."
$PYTHON manage.py makemigrations >/dev/null
$PYTHON manage.py migrate
echo "** Now running self check"
$PYTHON manage.py check -v 3 --deploy 2>security-warnings.txt >/dev/null
$PYTHON manage.py check -v 3 --deploy
echo "** Now running test suite"
# $PYTHON manage.py test -v 1
echo ""
echo `tail -1 lines-of-python.txt` non-comment lines of python. But core/utils.py has 28,000 lines of numbers.
echo `tail -1 lines-of-templates.txt` non-comment lines of HTML templates.
echo '** If you have an error running manage.py, maybe you are not in an activated venv ?'

View File

@@ -1,90 +1,35 @@
#!/bin/bash
# footled lots to make this work with python 3.10 & 3.11 and WSL1 and WSL2 on Ubuntu 22.04
# now using uv, unbelieveably simpler.
# Run this in a terminal in the troggle directory: 'bash venv-trog.sh'
echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.sh"'
echo '-- Run this in a terminal in the directory above the troggle directory: "bash troggle/venv-trog.sh"'
# Expects an Ubuntu 22.04 with all the gubbins already installed
# If you have not already installed these on your clean Ubuntu install DO THIS FIRST
# use the script os-trog24.04.sh
# use the script os-trog24.04.sh runniing it in /home/username/
python --version
# NOW we set up troggle
PYTHONSVR=python3.11
PYTHONDEV=python3.13
VSVR=svr # python3.11 and django 3.2
VDEV=dev # python3.13 and django 5
echo "** You are logged in as `id -u -n`"
#echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/"
echo "The 150MB venv will created in /home/`id -u -n`/$VSVR/ and /home/`id -u -n`/$VDEV/
echo "-- EXPO folder [current directory]: `pwd`"
TROGDIR=$(cd $(dirname $0) && pwd)
echo "-- Troggle folder (this script location): ${TROGDIR}"
exit
REQUIRE=requirements-$VSVR.txt
echo "-- Troggle folder [this script location]: ${TROGDIR}"
if [ -d $REQUIRE ]; then
echo "-- No ${REQUIRE} found. You should be in the /troggle/ folder. Copy it from your most recent installation."
exit 1
fi
echo ## Using $REQUIRE :
cat $REQUIRE
echo ##
cp troggle/pyproject.toml .
uv self update
uv sync
$PYTHON --version
exit 1
# this next is a bit wordy
# uv python list --only-installed
# NOTE that when using a later or earlier verison of python, you MUST also
# use the allowed version of Pillow, see https://pillow.readthedocs.io/en/latest/installation.html
# NOW set up link from expo user folder
# needed for WSL2
echo Creating links from Linux filesystem user
# These links only need making once, for many venv
cd ~
if [ ! -d $VSVR ]; then
echo "## Creating venv $VSVR. (If this fails with a pip error, you need to ensure you have python3.11-venv installed and/or use a Ubuntu window)"
$PYTHON -m venv $VSVR
else
echo "## /$VSVR/ already exists ! Delete it first."
exit 1
fi
# Activate the virtual env and see what the default packages are
echo "### Activating $VSVR"
cd $VSVR
echo "-- now in: ${PWD}"
ls -tlarg
source bin/activate
echo $VIRTUAL_ENV
if [ -d ~/$VSVR/bin ]; then
echo "### Activating."
else
echo "## ~/$VSVR/bin does not exist. FAILed to create venv properly."
exit 1
fi
# update local version of pip, more recent than OS version
# debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3
# update local version of setuptools, more recent than OS version, needed for packages without wheels
echo "### installing later version of pip inside $VSVR"
$PYTHON -m pip install --upgrade pip
$PYTHON -m pip install --upgrade setuptools
PIP=pip
$PIP list > original-pip.list
$PIP freeze >original.txt
# we are in /home/$USER/$VSVR/
ln -s ${TROGDIR} troggle
ln -s ${TROGDIR}/../expoweb expoweb
ln -s ${TROGDIR}/../loser loser
ln -s ${TROGDIR}/../drawings drawings
# we are in /home/$USER/expo/
# ln -s ${TROGDIR} troggle
# ln -s ${TROGDIR}/../expoweb expoweb
# ln -s ${TROGDIR}/../loser loser
# ln -s ${TROGDIR}/../drawings drawings
#ln -s ${TROGDIR}/../expofiles expofiles
# fudge for philip's machine
# fudge for philip's laptop prior to M2 SSD upgrade
if [ ! -d /mnt/d/EXPO ]; then
sudo mkdir /mnt/d
sudo mount -t drvfs D: /mnt/d
@@ -96,88 +41,34 @@ else
ln -s /mnt/d/EXPO/expofiles expofiles
fi
echo "### Setting file permissions.. may take a while.."
git config --global --add safe.directory '*'
sudo chmod -R 777 *
echo "### links to expoweb, troggle etc. complete:"
ls -tla
echo "###"
echo "### now installing ${TROGDIR}/${REQUIRE}"
echo "###"
cat ${TROGDIR}/${REQUIRE}
# NOW THERE IS A PERMISSIONS FAILURE THAT DIDN'T HAPPEN BEFORE
# seen on wsl2 as well as wsl1
# which ALSO ruins EXISTING permissions !
# Guessing it is to do with pip not liking non-standard py 3.11 installation on Ubuntu 22.04
read -p "Press any key to resume ..."
$PIP install -r ${TROGDIR}/${REQUIRE}
echo "### install from ${TROGDIR}/${REQUIRE} completed."
echo '### '
PIP="uv pip"
$PIP install --pre django
REQUIRE="requirements.txt"
$PIP freeze > $REQUIRE
# so that we can track requirements more easily with git
# because we do not install these with pip, but they are listed by the freeze command
# Now find out what we actually installed by subtracting the stuff venv installed anyway
sort original.txt > 1
sort $REQUIRE >2
comm -3 1 2 --check-order | awk '{ print $1}'>fresh-$REQUIRE
rm 1
rm 2
$PIP list
# cp $REQUIRE requirements-$VSVR.txt
cp $REQUIRE troggle/$REQUIRE
$PIP list > installed-pip.list
$PIP list -o > installed-pip-o.list
REQ=installation-record
mkdir $REQ
mv original.txt $REQ
mv $REQUIRE $REQ
mv original-pip.list $REQ
mv installed-pip.list $REQ
mv installed-pip-o.list $REQ
cp fresh-$REQUIRE ../$REQUIRE
mv fresh-$REQUIRE $REQ
cp troggle/`basename "$0"` $REQ
$PYTHON --version
python --version
echo "Django version:`django-admin --version`"
echo "Django version:`uv run django-admin --version`"
echo "### Now do
'[sudo service mysql start]'
'[sudo service mariadb restart]'
'[sudo mysql_secure_installation]'
'cd ~/$VSVR'
'source bin/activate'
'cd troggle'
'django-admin'
'python manage.py check'
'uv run django-admin'
'uv run manage.py check'
## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html
## you need to follow the Linux instructions.
'ssh expo@expo.survex.com'
## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine
## the tests may ALSO fail because of ssh and permissions errors
## So you will need to run
$sudo chown -Rhv philip:philip ~/$VSVR (if your username is philip)
# and then REBOOT (or at least, exit WSL and terminate and restart WSL)
# because this chmod only takes effect then.
'python manage.py test -v 2'
'uv run manage.py test -v 2'
'./pre-run.sh' (runs the tests again)
'python databaseReset.py reset $VSVR'
'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
'uv run databaseReset.py reset INIT'
'uv run manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
"
# if [ ! -d /mnt/d/expofiles ]; then
# echo '### No valid expofiles directory on /mnt/d . Fix this before any tests will work.'

View File

@@ -1,79 +0,0 @@
#!/bin/bash
# read in the input string from command line arguments
input=$1
# pad the input string with a single "1" bit
padded_input="$input"$(echo -ne '\x80')
# pad the input string to a multiple of 512 bits (64 bytes)
while (( $(echo -n "$padded_input" | wc -c) % 64 != 56 ))
do
padded_input="$padded_input"$(echo -ne '\x00')
done
# append the length of the input string (in bits) as a 64-bit little-endian integer
length=$(echo -n "$input" | wc -c)
length=$(echo "obase=16; $length * 8" | bc | xxd -p -c 16 | tac | tr -d '\n')
while (( $(echo -n "$length" | wc -c) < 16 ))
do
length="$length"0
done
padded_input="$padded_input"$(echo -ne "$length" | xxd -r -p)
# initialize the buffer (A, B, C, D)
A="67452301"
B="efcdab89"
C="98badcfe"
D="10325476"
# process the input in 512-bit (64-byte) chunks
for (( i=0; i<$(echo -n "$padded_input" | wc -c)/64; i++ ))
do
chunk=$(echo -n "$padded_input" | dd bs=64 skip=$i count=1 2>/dev/null | xxd -p -c 64)
# initialize the message schedule (M)
M=()
for (( j=0; j<16; j++ ))
do
word=$(echo -ne "${chunk:$j*8:8}" | xxd -r -p | od -An -tu4 -v)
M+=($word)
done
for (( j=16; j<64; j++ ))
do
word1=${M[j-15]}
s0=$(( (word1>>7 | word1<<25) ^ (word1>>18 | word1<<14) ^ (word1>>3) ))
word2=${M[j-2]}
s1=$(( (word2>>17 | word2<<15) ^ (word2>>19 | word2<<13) ^ (word2>>10) ))
M+=($((M[j-16] + s0 + M[j-7] + s1)))
done
# initialize the working variables
AA=$A
BB=$B
CC=$C
DD=$D
# round 1
for (( j=0; j<16; j++ ))
do
F=$(( (B & C) | (~B & D) ))
g=$j
dTemp=$((D))
D=$((C))
C=$((B))
B=$((B + ((A + F + M[g] + 0xd76aa478) & 0xffffffff)))
A=$((dTemp))
done
# round 2
for (( j=16; j<32; j++ ))
do
F=$(( (D & B) | (~D & C) ))
g=$(( (5*j + 1) % 16 ))
dTemp=$((D))
D=$((C))
C=$((B))

View File

@@ -45,6 +45,7 @@ echo " reset: SECRET_KEY = \"real-SECRET_KEY--imported-from-localsettings.py\
mv _deploy/wsl/localsettingsWSL.py _deploy/wsl/localsettingsWSL.py.bak
mv localsettingsWSL.py _deploy/wsl
cp *.sh _deploy/wsl
deactivate
#
# Do these before final testing, *not* just before pushing:

View File

@@ -1,19 +1,32 @@
asgiref==3.8.1
beautifulsoup4==4.12.3
black==24.10.0
click==8.1.7
coverage==7.6.9
deptry==0.21.1
django==5.1.4
annotated-types==0.6.0
asgiref==3.7.2
beautifulsoup4==4.12.0
black==24.2.0
chardet==5.2.0
click==8.1.0
coverage==7.0.0
deptry==0.12.0
django==5.0.4
docutils==0.20
fawltydeps==0.15.0
importlib-metadata==7.0.1
isort==5.13.2
mypy-extensions==1.0.0
packaging==24.2
pathspec==0.12.1
packaging==23.0
pathspec==0.12.0
piexif==1.1.3
pillow==11.0.0
platformdirs==4.3.6
requirements-parser==0.11.0
soupsieve==2.6
sqlparse==0.5.3
types-setuptools==75.6.0.20241126
pillow==10.2.0
pip==24.0
pip-requirements-parser==32.0.1
platformdirs==4.0.0
pydantic==2.6.2
pydantic-core==2.16.3
pyparsing==3.1.1
pytz==2024.1
ruff==0.2.0
setuptools==69.1.0
soupsieve==2.5
sqlparse==0.4.4
typing-extensions==4.9.0
unidecode==1.3.8
zipp==3.17.0