From 293eb10ffd5edc8c569e29855bcdb32405ea151f Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sat, 25 Jun 2022 19:28:01 +0300 Subject: [PATCH 1/5] New OS and venv tool --- os-trog.sh | 29 +++++++++++++++++++++++++++++ requirements-p10d3.txt | 13 +++++++++++++ requirements.txt | 1 + venv-trog.sh | 16 +++++++++------- 4 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 os-trog.sh create mode 100644 requirements-p10d3.txt diff --git a/os-trog.sh b/os-trog.sh new file mode 100644 index 0000000..d145422 --- /dev/null +++ b/os-trog.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Run this in a terminal in the troggle directory: 'bash os-trog.sh' +echo 'Run this in a terminal in the troggle directory: "bash venv-trog.sh"' + +# Expects an Ubuntu 22.04 relatively clean install. + + +python --version : ensure python is an alias for python3 not python2.7 +sudo apt update +sudo apt dist-upgrade +sudo apt install sqlite3 +sudo apt install python3-pip # this installs a shed-load of other stuff: binutils etc. + + +# On a clean debian 11 (bullseye) installation with Xfce & ssh, +# as debian does not install everything that ubuntu does, you need: +sudo usermod -a -G sudo expo # to put expo in sudoers group, re-login required +sudo apt install python3.10 +sudo apt install python3.10-venv +sudo apt install python3.10-dev +sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 + +sudo apt -y install mariadb-server +sudo apt install libmariadb-dev + +sudo python -m pip install --upgrade pip + +# Go to https://expo.survex.com/handbook/troggle/troglaptop.html#dbtools +# sudo service mysql start diff --git a/requirements-p10d3.txt b/requirements-p10d3.txt new file mode 100644 index 0000000..82024e9 --- /dev/null +++ b/requirements-p10d3.txt @@ -0,0 +1,13 @@ +asgiref==3.3.4 +confusable-homoglyphs==3.2.0 +coverage==5.5 +Django==3.2 +docutils==0.14 +gunicorn==20.1.0 +mariadb==1.0.11 +Pillow==9.0.1 +pytz==2019.1 +reportlab==3.6.8 +sqlparse==0.2.4 +typing-extensions==3.7.4.3 +Unidecode==1.0.23 diff --git a/requirements.txt b/requirements.txt index ff5b804..d774a94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ reportlab==3.6.8 sqlparse==0.2.4 typing-extensions==3.7.4.3 Unidecode==1.0.23 +mariadb diff --git a/venv-trog.sh b/venv-trog.sh index 7c10473..0439008 100644 --- a/venv-trog.sh +++ b/venv-trog.sh @@ -2,17 +2,19 @@ # Run this in a terminal in the troggle directory: 'bash venv-trog.sh' echo 'Run this in a terminal in the troggle directory: "bash venv-trog.sh"' -# Expects an Ubuntu 20.04 relatively clean install. -# Running using SQLite 3.31.1 2020-01-27 19:55:54 +# Expects an Ubuntu 22.04 relatively clean install. +# Running using SQLite version 3.37.2 2022-01-06 13:25:41 # Many permissions conflicts under WSL2 mean more sudo needed # If you have not already installed these on your clean Ubuntu install DO THIS FIRST -# /usr/bin/python --version : ensure python is an alias for python3 not python2.7 +# use the script os-trog.sh + +# python --version : ensure python is an alias for python3 not python2.7 # sudo apt install sqlite3 # sudo apt install python3-pip # this installs a shed-load of other stuff: binutils etc. -# sudo apt install python3.9 -# sudo apt install python3.9-venv -# sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 +# sudo apt install python3.10 +# sudo apt install python3.10-venv +# sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 # on WSL there is an oddity that creating a venv trips over a PowerShell permissions issue @@ -30,7 +32,7 @@ echo 'Run this in a terminal in the troggle directory: "bash venv-trog.sh"' # # copy this file and requirements.txt into the directory above where you want to install the VENV -VENAME=p310 # python3.10 and django 3.2 +VENAME=p10d3 # python3.10 and django 3.2 if [ -d requirements.txt ]; then echo "No requirements.txt found. Copy it from your most recent installation." From 364a636fa0315f717246a799929cf2a98f1f90e3 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sat, 25 Jun 2022 19:37:37 +0300 Subject: [PATCH 2/5] Updated pip packages --- requirements.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index d774a94..0e1c259 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ -asgiref==3.3.4 +asgiref==3.5 confusable-homoglyphs==3.2.0 -coverage==5.5 +coverage==6.4 Django==3.2 -docutils==0.14 +docutils==0.18 gunicorn==20.1.0 -Pillow==9.0.1 -pytz==2019.1 -reportlab==3.6.8 -sqlparse==0.2.4 -typing-extensions==3.7.4.3 -Unidecode==1.0.23 -mariadb +Pillow==9.1 +pytz==2022.1 +reportlab==3.6 +sqlparse==0.4 +typing-extensions==4.2 +Unidecode==1.3 +mariadb==1.0.11 From 859ae9d8259e8c15091ebe60879a12e2e0fd576a Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sat, 25 Jun 2022 20:00:19 +0300 Subject: [PATCH 3/5] update pip inside venv --- venv-trog.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/venv-trog.sh b/venv-trog.sh index 0439008..4fff9c3 100644 --- a/venv-trog.sh +++ b/venv-trog.sh @@ -62,6 +62,7 @@ echo "### Activated." #python -m pip install --upgrade pip PIP=pip +sudo python -m pip install --upgrade pip $PIP list > original-pip.list $PIP freeze >original.txt From 451326789b746936367eb3fdbdefe667425b27c5 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sat, 25 Jun 2022 20:01:43 +0300 Subject: [PATCH 4/5] updated packages and settings --- _deploy/wsl/localsettingsWSL.py | 34 +++++++++++++++++++++++++-------- deprecations.txt | 8 ++++++-- requirements-p10d3.txt | 18 ++++++++--------- requirements.txt | 14 +++++++------- 4 files changed, 48 insertions(+), 26 deletions(-) diff --git a/_deploy/wsl/localsettingsWSL.py b/_deploy/wsl/localsettingsWSL.py index 2673b99..c57a35b 100644 --- a/_deploy/wsl/localsettingsWSL.py +++ b/_deploy/wsl/localsettingsWSL.py @@ -33,7 +33,7 @@ EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py" EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py" -EXPOFILESREMOTE = True # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate. +EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate. #SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely SERVERPORT = '8000' # not needed @@ -56,7 +56,7 @@ JSLIB_ROOT = TROGGLE_PATH / 'media' / 'jslib' # used for CaveViewer JS ut #FILES = Path('/mnt/d/expofiles/') EXPOFILES = Path('/mnt/d/EXPO/expofiles/') -SURVEY_SCANS = EXPOFILES / 'surveyscans' +SCANS_ROOT = EXPOFILES / 'surveyscans' PHOTOS_ROOT = EXPOFILES / 'photos' PHOTOS_YEAR = "2022" @@ -68,8 +68,9 @@ DIR_ROOT = ''#this should end in / if a value is given URL_ROOT = '/' # URL_ROOT = 'http://localhost:'+ SERVERPORT +'/' +#Note that these constants are not actually used in urls.py, they should be.. MEDIA_URL = urllib.parse.urljoin(URL_ROOT , '/site_media/') -SURVEYS_URL = urllib.parse.urljoin(URL_ROOT , '/survey_scans/') +SCANS_URL = urllib.parse.urljoin(URL_ROOT , '/survey_scans/') PHOTOS_URL = urllib.parse.urljoin(URL_ROOT , '/photos/') SVX_URL = urllib.parse.urljoin(URL_ROOT , '/survex/') @@ -88,7 +89,7 @@ CACHEDPAGES = True # experimental page cache for a handful of page types CAVERN = 'cavern' # for parsing .svx files and producing .3d files SURVEXPORT = 'survexport' # for parsing .3d files and producing .pos files -DATABASES = { +DBSQLITE = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. 'NAME' : 'troggle.sqlite', @@ -99,7 +100,25 @@ DATABASES = { 'PORT' : '', # Set to empty string for default. Not used with sqlite3. } } -# add in 358 when they don't make it crash horribly +DBMARIADB = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + 'NAME' : 'troggle', # Or path to database file if using sqlite3. + 'USER' : 'expo', + 'PASSWORD' : 'my-secret-password-schwatzmooskogel', + 'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. + 'PORT' : '', # Set to empty string for default. Not used with sqlite3. + } +} + +# default database for me is squlite +DBSWITCH = "sqlite" + +if DBSWITCH == "sqlite": + DATABASES = DBSQLITE +if DBSWITCH == "mariadb": + DATABASES = DBMARIADB + NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"] PYTHON_PATH = REPOS_ROOT_PATH / 'troggle' @@ -137,7 +156,6 @@ TEMPLATES = [ }, ] -# Passwords are loaded from credentials.py by settings.py EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py" EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py" @@ -160,7 +178,7 @@ EXPOWEB = REPOS_ROOT_PATH / "expoweb" CAVEDESCRIPTIONS = EXPOWEB / "cave_data" ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data" EXPOWEB_URL = '' -SURVEYS_URL = '/survey_scans/' +SCANS_URL = '/survey_scans/' # Sanitise these to be strings as all other code is expecting strings # and we have not made the change to pathlib Path type in the other localsettings-* variants yet. @@ -175,4 +193,4 @@ REPOS_ROOT_PATH = os.fspath(REPOS_ROOT_PATH) TEMPLATE_PATH = os.fspath(TROGGLE_PATH) MEDIA_ROOT = os.fspath(MEDIA_ROOT) JSLIB_ROOT = os.fspath(JSLIB_ROOT) -SURVEY_SCANS = os.fspath(SURVEY_SCANS) +SCANS_ROOT = os.fspath(SCANS_ROOT) diff --git a/deprecations.txt b/deprecations.txt index a8f4ed8..d5d211f 100644 --- a/deprecations.txt +++ b/deprecations.txt @@ -1,2 +1,6 @@ -/mnt/d/EXPO/troggle/core/views/uploads.py:120: DeprecationWarning: invalid escape sequence \d - if not re.match('(19|20)\d\d[:#]\d\d', wallet): +/home/philip/p10d3/lib/python3.10/site-packages/django/utils/version.py:6: DeprecationWarning: The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives + from distutils.version import LooseVersion +/home/philip/p10d3/lib/python3.10/site-packages/django/utils/asyncio.py:19: DeprecationWarning: There is no current event loop + event_loop = asyncio.get_event_loop() +/home/philip/p10d3/lib/python3.10/site-packages/django/utils/asyncio.py:19: DeprecationWarning: There is no current event loop + event_loop = asyncio.get_event_loop() diff --git a/requirements-p10d3.txt b/requirements-p10d3.txt index 82024e9..f361372 100644 --- a/requirements-p10d3.txt +++ b/requirements-p10d3.txt @@ -1,13 +1,13 @@ -asgiref==3.3.4 +asgiref==3.5.0 confusable-homoglyphs==3.2.0 -coverage==5.5 +coverage==6.4 Django==3.2 -docutils==0.14 +docutils==0.18 gunicorn==20.1.0 mariadb==1.0.11 -Pillow==9.0.1 -pytz==2019.1 -reportlab==3.6.8 -sqlparse==0.2.4 -typing-extensions==3.7.4.3 -Unidecode==1.0.23 +Pillow==9.1.0 +pytz==2022.1 +reportlab==3.6.0 +sqlparse==0.4.0 +typing_extensions==4.2.0 +Unidecode==1.3.0 diff --git a/requirements.txt b/requirements.txt index 0e1c259..f361372 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ -asgiref==3.5 +asgiref==3.5.0 confusable-homoglyphs==3.2.0 coverage==6.4 Django==3.2 docutils==0.18 gunicorn==20.1.0 -Pillow==9.1 -pytz==2022.1 -reportlab==3.6 -sqlparse==0.4 -typing-extensions==4.2 -Unidecode==1.3 mariadb==1.0.11 +Pillow==9.1.0 +pytz==2022.1 +reportlab==3.6.0 +sqlparse==0.4.0 +typing_extensions==4.2.0 +Unidecode==1.3.0 From 2f1ba9cb54514368c1585af1fb540730e286b643 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sat, 25 Jun 2022 21:34:42 +0300 Subject: [PATCH 5/5] install updates --- README.txt | 6 ++++-- os-trog.sh | 6 ++++-- requirements-p10d3.txt | 3 +++ requirements.txt | 3 +++ venv-trog.sh | 9 ++++++--- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/README.txt b/README.txt index dff42e3..f3ef518 100644 --- a/README.txt +++ b/README.txt @@ -49,10 +49,10 @@ Follow the instructions contained in the file to fill out your settings. Python3, Django, and Database setup ----------------------------------- -We are now using Django 2.2.19 and will move to 3.2 shortly +We are now using Django 3.2 and will move to 4.2 in 2024 We are installing with python3.9 -Install Django using pip, not with apt, on your test system. +Install Django using pip, not with apt, on your test system in a venv. Conventionally on our main master expo server we install everything that we can as debian packages, not using pip. [installation instructions removed - now in http://expo.survex.com/handbook/troggle/troglaptop.html ] @@ -151,6 +151,8 @@ these permissions are set in a different 'info' database which usually is untouc PERMISSIONS https://linuxize.com/post/usermod-command-in-linux/ +THIS MAY BE OUT OF DATE - in 2022 we are running Apache as user 'expo' + sudo usermod -a expo www-data adds expo to the www-data group which is what the webserver uses, and thus so the user troggle is acting as when running live. diff --git a/os-trog.sh b/os-trog.sh index d145422..02749a4 100644 --- a/os-trog.sh +++ b/os-trog.sh @@ -11,7 +11,9 @@ sudo apt dist-upgrade sudo apt install sqlite3 sudo apt install python3-pip # this installs a shed-load of other stuff: binutils etc. - +sudo apt install survex-aven +sudo apt install openssh-client tunnelx therion +sudo apt install git sftp # On a clean debian 11 (bullseye) installation with Xfce & ssh, # as debian does not install everything that ubuntu does, you need: sudo usermod -a -G sudo expo # to put expo in sudoers group, re-login required @@ -20,7 +22,7 @@ sudo apt install python3.10-venv sudo apt install python3.10-dev sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 -sudo apt -y install mariadb-server +sudo apt install mariadb-server sudo apt install libmariadb-dev sudo python -m pip install --upgrade pip diff --git a/requirements-p10d3.txt b/requirements-p10d3.txt index f361372..b05fda2 100644 --- a/requirements-p10d3.txt +++ b/requirements-p10d3.txt @@ -5,7 +5,10 @@ Django==3.2 docutils==0.18 gunicorn==20.1.0 mariadb==1.0.11 +mysql-connector-python==8.0.29 +mysqlclient==2.1.0 Pillow==9.1.0 +protobuf==4.21.2 pytz==2022.1 reportlab==3.6.0 sqlparse==0.4.0 diff --git a/requirements.txt b/requirements.txt index f361372..b05fda2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,10 @@ Django==3.2 docutils==0.18 gunicorn==20.1.0 mariadb==1.0.11 +mysql-connector-python==8.0.29 +mysqlclient==2.1.0 Pillow==9.1.0 +protobuf==4.21.2 pytz==2022.1 reportlab==3.6.0 sqlparse==0.4.0 diff --git a/venv-trog.sh b/venv-trog.sh index 4fff9c3..722f65b 100644 --- a/venv-trog.sh +++ b/venv-trog.sh @@ -58,11 +58,10 @@ source bin/activate echo "### Activated." # update local version of pip, more recent than OS version # debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3 -#echo "### installing later version of pip inside $VENAME" -#python -m pip install --upgrade pip +echo "### installing later version of pip inside $VENAME" +python -m pip install --upgrade pip PIP=pip -sudo python -m pip install --upgrade pip $PIP list > original-pip.list $PIP freeze >original.txt @@ -119,7 +118,11 @@ python --version echo "Django version:" django-admin --version echo "### Now do + 'sudo service mysql start' + 'sudo service mariadb restart' + 'sudo mysql_secure_installation' 'cd ../$VENAME' 'source bin/activate' 'cd troggle' + 'python manage.py check' 'python manage.py test -v 3' " \ No newline at end of file