Merge branch 'python3-new' of ssh://expo.survex.com/home/expo/troggle into python3-new

README.txt

@@ -49,10 +49,10 @@ Follow the instructions contained in the file to fill out your settings.
 
 Python3, Django, and Database setup
 -----------------------------------
-We are now using Django 2.2.19 and will move to 3.2 shortly
+We are now using Django 3.2 and will move to 4.2 in 2024
 We are installing with python3.9
 
-Install Django using pip, not with apt, on your test system.
+Install Django using pip, not with apt, on your test system in a venv.
 Conventionally on our main master expo server we install everything that we can as debian packages, not using pip.
 
 [installation instructions removed - now in http://expo.survex.com/handbook/troggle/troglaptop.html ]
@@ -151,6 +151,8 @@ these permissions are set in a different 'info' database which usually is untouc
 PERMISSIONS
 https://linuxize.com/post/usermod-command-in-linux/
 
+THIS MAY BE OUT OF DATE - in 2022 we are running Apache as user 'expo'
+
 sudo usermod -a expo www-data
 adds expo to the www-data group which is what the webserver uses, and thus so the user troggle is acting as when running live.
 

_deploy/wsl/localsettingsWSL.py

@@ -33,7 +33,7 @@ EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py
 EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"
 EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"
 
-EXPOFILESREMOTE = True # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate.
+EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate.
 #SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely
 
 SERVERPORT = '8000' # not needed
@@ -56,7 +56,7 @@ JSLIB_ROOT        = TROGGLE_PATH / 'media' / 'jslib' # used for CaveViewer JS ut
 
 #FILES = Path('/mnt/d/expofiles/')
 EXPOFILES = Path('/mnt/d/EXPO/expofiles/')
-SURVEY_SCANS = EXPOFILES / 'surveyscans'
+SCANS_ROOT = EXPOFILES / 'surveyscans'
 PHOTOS_ROOT = EXPOFILES / 'photos'
 PHOTOS_YEAR = "2022"
 
@@ -68,8 +68,9 @@ DIR_ROOT = ''#this should end in / if a value is given
 URL_ROOT = '/'
 # URL_ROOT = 'http://localhost:'+ SERVERPORT +'/'
 
+#Note that these constants are not actually used in urls.py, they should be..
 MEDIA_URL   = urllib.parse.urljoin(URL_ROOT , '/site_media/')
-SURVEYS_URL = urllib.parse.urljoin(URL_ROOT , '/survey_scans/')
+SCANS_URL   = urllib.parse.urljoin(URL_ROOT , '/survey_scans/')
 PHOTOS_URL  = urllib.parse.urljoin(URL_ROOT , '/photos/')
 SVX_URL     = urllib.parse.urljoin(URL_ROOT , '/survex/')
 
@@ -88,7 +89,7 @@ CACHEDPAGES = True  # experimental page cache for a handful of page types
 CAVERN = 'cavern' # for parsing .svx files and producing .3d files
 SURVEXPORT = 'survexport' # for parsing .3d files and producing .pos files
 
-DATABASES = {
+DBSQLITE = {
     'default': { 
         'ENGINE': 'django.db.backends.sqlite3', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
         'NAME' : 'troggle.sqlite',
@@ -99,7 +100,25 @@ DATABASES = {
         'PORT' : '',            # Set to empty string for default. Not used with sqlite3.
     }
 }
-# add in 358 when they don't make it crash horribly
+DBMARIADB = {
+    'default': { 
+        'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
+        'NAME' : 'troggle',                   # Or path to database file if using sqlite3.
+        'USER' : 'expo',                      
+        'PASSWORD' : 'my-secret-password-schwatzmooskogel',       
+        'HOST' : '',                          # Set to empty string for localhost. Not used with sqlite3.
+        'PORT' : '',                          # Set to empty string for default. Not used with sqlite3.
+    }
+}
+
+# default database for me is squlite
+DBSWITCH = "sqlite"
+
+if DBSWITCH == "sqlite":
+    DATABASES = DBSQLITE
+if DBSWITCH == "mariadb":
+    DATABASES = DBMARIADB
+
 NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"]
 
 PYTHON_PATH = REPOS_ROOT_PATH / 'troggle'
@@ -137,7 +156,6 @@ TEMPLATES = [
     },
 ]
 
-# Passwords are loaded from credentials.py by settings.py
 EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py"
 EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"
 
@@ -160,7 +178,7 @@ EXPOWEB = REPOS_ROOT_PATH / "expoweb"
 CAVEDESCRIPTIONS        = EXPOWEB / "cave_data"
 ENTRANCEDESCRIPTIONS    = EXPOWEB / "entrance_data"
 EXPOWEB_URL = ''
-SURVEYS_URL = '/survey_scans/'
+SCANS_URL = '/survey_scans/'
 
 # Sanitise these to be strings as all other code is expecting strings
 # and we have not made the change to pathlib Path type in the other localsettings-* variants yet.
@@ -175,4 +193,4 @@ REPOS_ROOT_PATH = os.fspath(REPOS_ROOT_PATH)
 TEMPLATE_PATH   = os.fspath(TROGGLE_PATH)
 MEDIA_ROOT      = os.fspath(MEDIA_ROOT)
 JSLIB_ROOT      = os.fspath(JSLIB_ROOT)
-SURVEY_SCANS    = os.fspath(SURVEY_SCANS)
+SCANS_ROOT    = os.fspath(SCANS_ROOT)

deprecations.txt

@@ -1,2 +1,6 @@
-/mnt/d/EXPO/troggle/core/views/uploads.py:120: DeprecationWarning: invalid escape sequence \d
+/home/philip/p10d3/lib/python3.10/site-packages/django/utils/version.py:6: DeprecationWarning: The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives
-  if not re.match('(19|20)\d\d[:#]\d\d', wallet):
+  from distutils.version import LooseVersion
+/home/philip/p10d3/lib/python3.10/site-packages/django/utils/asyncio.py:19: DeprecationWarning: There is no current event loop
+  event_loop = asyncio.get_event_loop()
+/home/philip/p10d3/lib/python3.10/site-packages/django/utils/asyncio.py:19: DeprecationWarning: There is no current event loop
+  event_loop = asyncio.get_event_loop()

os-trog.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Run this in a terminal in the troggle directory: 'bash os-trog.sh'
+echo 'Run this in a terminal in the troggle directory: "bash venv-trog.sh"'
+
+# Expects an Ubuntu 22.04 relatively clean install.
+
+
+python --version : ensure python is an alias for python3 not python2.7
+sudo apt update
+sudo apt dist-upgrade
+sudo apt install sqlite3
+sudo apt install python3-pip  # this installs a shed-load of other stuff: binutils etc.
+
+sudo apt install survex-aven
+sudo apt install openssh-client tunnelx therion 
+sudo apt install git sftp
+# On a clean debian 11 (bullseye) installation with Xfce & ssh, 
+# as debian does not install everything that ubuntu does, you need:
+sudo usermod -a -G sudo expo # to put expo in sudoers group, re-login required
+sudo apt install python3.10
+sudo apt install python3.10-venv
+sudo apt install python3.10-dev
+sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1
+
+sudo apt install mariadb-server
+sudo apt install libmariadb-dev
+
+sudo python -m pip install --upgrade pip
+
+# Go to https://expo.survex.com/handbook/troggle/troglaptop.html#dbtools
+# sudo service mysql start

requirements-p10d3.txt

@@ -0,0 +1,16 @@
+asgiref==3.5.0
+confusable-homoglyphs==3.2.0
+coverage==6.4
+Django==3.2
+docutils==0.18
+gunicorn==20.1.0
+mariadb==1.0.11
+mysql-connector-python==8.0.29
+mysqlclient==2.1.0
+Pillow==9.1.0
+protobuf==4.21.2
+pytz==2022.1
+reportlab==3.6.0
+sqlparse==0.4.0
+typing_extensions==4.2.0
+Unidecode==1.3.0

requirements.txt

@@ -1,12 +1,16 @@
-asgiref==3.3.4
+asgiref==3.5.0
 confusable-homoglyphs==3.2.0
-coverage==5.5
+coverage==6.4
 Django==3.2
-docutils==0.14
+docutils==0.18
 gunicorn==20.1.0
-Pillow==9.0.1
+mariadb==1.0.11
-pytz==2019.1
+mysql-connector-python==8.0.29
-reportlab==3.6.8
+mysqlclient==2.1.0
-sqlparse==0.2.4
+Pillow==9.1.0
-typing-extensions==3.7.4.3
+protobuf==4.21.2
-Unidecode==1.0.23
+pytz==2022.1
+reportlab==3.6.0
+sqlparse==0.4.0
+typing_extensions==4.2.0
+Unidecode==1.3.0

venv-trog.sh

@@ -2,17 +2,19 @@
 # Run this in a terminal in the troggle directory: 'bash venv-trog.sh'
 echo 'Run this in a terminal in the troggle directory: "bash venv-trog.sh"'
 
-# Expects an Ubuntu 20.04 relatively clean install.
+# Expects an Ubuntu 22.04 relatively clean install.
-# Running using SQLite 3.31.1 2020-01-27 19:55:54
+# Running using SQLite version 3.37.2 2022-01-06 13:25:41
 # Many permissions conflicts under WSL2 mean more sudo needed
 
 # If you have not already installed these on your clean Ubuntu install DO THIS FIRST
-# /usr/bin/python --version : ensure python is an alias for python3 not python2.7
+# use the script os-trog.sh
+
+# python --version : ensure python is an alias for python3 not python2.7
 # sudo apt install sqlite3
 # sudo apt install python3-pip  # this installs a shed-load of other stuff: binutils etc.
-# sudo apt install python3.9
+# sudo apt install python3.10
-# sudo apt install python3.9-venv
+# sudo apt install python3.10-venv
-# sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1
+# sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1
 
 # on WSL there is an oddity that creating a venv trips over a PowerShell permissions issue
 
@@ -30,7 +32,7 @@ echo 'Run this in a terminal in the troggle directory: "bash venv-trog.sh"'
 #
 
 # copy this file and requirements.txt into the directory above where you want to install the VENV
-VENAME=p310 # python3.10 and django 3.2
+VENAME=p10d3 # python3.10 and django 3.2
 
 if [ -d requirements.txt ]; then
   echo "No requirements.txt found. Copy it from your most recent installation."
@@ -56,8 +58,8 @@ source bin/activate
 echo "### Activated."
   # update local version of pip, more recent than OS version
   # debian bullseye installs pip 20.3.4 which barfs, we want >22.0.3
-#echo "### installing later version of pip inside $VENAME"
+echo "### installing later version of pip inside $VENAME"
-#python -m pip install --upgrade pip
+python -m pip install --upgrade pip
 
 PIP=pip
 
@@ -116,7 +118,11 @@ python --version
 echo "Django version:"
 django-admin --version
 echo "### Now do 
+  'sudo service mysql start'
+  'sudo service mariadb restart'
+  'sudo mysql_secure_installation'
   'cd ../$VENAME'
   'source bin/activate'
   'cd troggle'
+  'python manage.py check'
   'python manage.py test -v 3' "