WORKING Dj2.2.24 & 2.2.25 py3.7

core/TESTS/test_imports.py

@@ -34,7 +34,7 @@ class SimpleTest(SimpleTestCase):
     def test_import_Cave(self):
         from troggle.core.models.caves import Cave
     def test_import_parsers_surveys(self):
-        from PIL import Image
+        #from PIL import Image
         from troggle.core.utils import save_carefully
         from functools import reduce
     def test_import_parsers_survex(self):

core/models/caves.py

@@ -56,7 +56,7 @@ def writetrogglefile(filepath, filecontent):
     #os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
     # should replace .call with .run and capture_output=True
     call([git, "add", filename], cwd=cwd)
-    call([git, "commit", "-m", 'Online cave or entrance edit'], cwd=cwd)
+    call([git, "commit", "-m", 'Troggle online cave or entrance edit'], cwd=cwd)
 
 
 class Area(TroggleModel):

core/views/caves.py

@@ -5,7 +5,6 @@ import settings
 import urllib.parse
 import subprocess
 from pathlib import Path
-#from PIL import Image, ImageDraw, ImageFont
 
 from django import forms
 from django.conf import settings

core/views/prospect.py

@@ -1,10 +1,8 @@
 import os
 import string
 import re
-#import settings
 import urllib.parse
 # from pathlib import Path
-from PIL import Image, ImageDraw, ImageFont
 
 from django.http import HttpResponse
 from django.shortcuts import render
@@ -18,7 +16,7 @@ from troggle.parsers.survex import MapLocations
 ''' Generates the prospecting guide document.
 
 Also produces the overlay of points on top of a prospecting_image map - to be deleted.
-I have tried to make this work with the version of PIL we have installed but something is missing still.
+Not working with recent PIL aka Pillow image package - removed.
 '''
 
 AREANAMES = [
@@ -143,7 +141,7 @@ for FONT in [
 TEXTSIZE = 16
 CIRCLESIZE =8
 LINEWIDTH = 2
-myFont = ImageFont.truetype(FONT, TEXTSIZE)
+#myFont = ImageFont.truetype(FONT, TEXTSIZE) # disabled as not importing PIL
 #print(f' - myFont {myFont} {FONT} {TEXTSIZE}')
 
 def mungecoord(x, y, mapcode, img):

parsers/drawings.py

@@ -7,7 +7,6 @@ import re
 import datetime
 from pathlib import Path
 
-from PIL import Image
 from functools import reduce
 
 import settings

parsers/logbooks.py

@@ -659,9 +659,10 @@ def LoadLogbooks():
     expos = Expedition.objects.all()
     if len(expos) <= 1:
         print(" ! No expeditions found. Load 'people' first.\n")
-    nologbook = ["1976", "1977", "1978", "1979", "1980", "1981", 
-        "1987", "1988", "1989", # breaks mysql with db constraint fail - debug locally first
-        "1986", "2020", "2021",] #no expo
+    noexpo =      ["1986", "2020", "2021",] #no expo
+    lostlogbook = ["1976", "1977", "1978", "1979", "1980", "1981"]
+    sqlfail =     ["1987", "1988", "1989"] # breaks mysql with db constraint fail - debug locally first]
+    nologbook = noexpo + lostlogbook + sqlfail
     entries = { "2019": 20, "2018": 74, "2017": 60, "2016": 81, "2015": 79, 
         "2014": 65, "2013": 51, "2012": 75, "2011": 68, "2010": 22, "2009": 52, 
         "2008": 49, "2007": 111, "2006": 60, "2005": 55, "2004": 76, "2003": 40, "2002": 31, 
@@ -678,6 +679,12 @@ def LoadLogbooks():
     with open("loadlogbk.log", "a") as log:
         for expo in expos:
             TROG['pagecache']['expedition'][expo.year] = None # clear cache
+            if expo.year in sqlfail:
+                print(" - Logbook for: " + expo.year + " NO parsing attempted - known sql failures")
+                message = f" ! - Not even attempting to parse logbook for {expo.year} until code fixed" 
+                DataIssue.objects.create(parser='logbooks', message=message)
+                logdataissues[f"sqlfail {expo.year}"]=message
+
             if expo.year not in nologbook:
                 print((" - Logbook for: " + expo.year))
                 if expo.year in entries:
@@ -686,7 +693,7 @@ def LoadLogbooks():
                     nlbe[expo.year]=numentries
                     expd[expo.year]= 0
                 else:
-                    print((" - No Logbook yet for: " + expo.year)) # catch case when preparing for next expo
+                    print(" - No Logbook yet for: " + expo.year) # catch case when preparing for next expo
     print("** total trips in ObjStore:", len(trips))
     #for i in logdataissues:
     #    print("{:15s}:   {}".format(i, logdataissues[i]))
@@ -700,11 +707,19 @@ def LoadLogbooks():
         yt += expd[y]
     print("total {} log entries in all expeditions".format(yt))
     
-    with shelve.open('logbktrips.shelve',writeback=True) as odb:
-        for lbe in trips:
-            odb[lbe]=trips[lbe]
-    odb.sync()
-    odb.close()
+    try:
+        shelvfilenm = 'logbktrips.shelve' # ".db" automatically apended after python 3.8
+        with shelve.open(shelvfilenm, writeback=True) as odb:
+            for lbe in trips:
+                odb[lbe]=trips[lbe]
+            odb.sync()
+            odb.close()
+    except:
+        message = f" ! - Failed store cached logbooks in '{shelvfilenm}.db' - Delete old file and try again" 
+        DataIssue.objects.create(parser='logbooks', message=message)
+        logdataissues["Shelve Fail"]=message
+        print(message)
+
 
 # dateRegex = re.compile(r'<span\s+class="date">(\d\d\d\d)-(\d\d)-(\d\d)</span>', re.S)
 # expeditionYearRegex = re.compile(r'<span\s+class="expeditionyear">(.*?)</span>', re.S)

parsers/scans.py

@@ -6,7 +6,6 @@ import csv
 import re
 import datetime
 
-from PIL import Image
 from functools import reduce
 
 import settings

requirements.txt

@@ -1,6 +1,7 @@
 asgiref==3.3.4
 confusable-homoglyphs==3.2.0
-Django==2.2.20
+coverage==5.5
+Django==2.2.25
 docutils==0.14
 gunicorn==20.1.0
 Pillow==5.4.1

settings.py

@@ -117,7 +117,8 @@ APPEND_SLASH = False # never relevant because we have urls that match unknown fi
 SMART_APPEND_SLASH = True #not eorking as middleware different after Dj2.0
 
 
-LOGIN_REDIRECT_URL = '/'
+LOGIN_REDIRECT_URL = '/' # does not seem to have any effect
+
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_BROWSER_XSS_FILTER = True
 # SESSION_COOKIE_SECURE = True # if enabled, cannot login to Django control panel, bug elsewhere?
@@ -128,7 +129,7 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' # from Django 3.2
 
 INSTALLED_APPS = (
     'django.contrib.admin',
-    'django.contrib.auth',
+    'django.contrib.auth', # includes the url redirections for login, logout
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.messages',

templates/login/logout.html

@@ -27,7 +27,7 @@ This is because Django is Opinionated and does lots of Invisible Defaults
 	<div class='align-center'>
 	<div class='space'></div>
 	<div class='align-center'>
-		<form action="/login/" method="post" accept-charset="utf-8">{% csrf_token %}
+		<form action="/accounts/login/" method="post" accept-charset="utf-8">{% csrf_token %}
 			<table class='form'>
 			{{form.as_table}}
 			</table>

templates/registration/login.html

@@ -0,0 +1,55 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<!-- To understand how this all works, with invisible default behaviour, see
+
+troggle/templates/login/logout.html
+troggle/templates/registration/ 
+
+because magic 
+This is because Django is Opinionated and does lots of Invisible Defaults
+
+This file is actually identical to troggle/templates/login/index.html
+-->
+<div class='middle'>
+<h2>Troggle user login</h2>
+</div>
+
+{% if message %}
+    <div style='width: 350px;'  class='middle3 login'>
+        {% if title %}
+        <div class='align-center'>
+        <h3>{{title}}</h3>
+        <div class='space'></div>
+        {% endif %}
+    <p>{{message }}</p>
+    </div>
+{% endif %}
+<h3>Troggle ordinary user login - no access to Django control panel</h3>
+<p>(This is using template registration/login.html)
+<div style='width: 250px;'  class='middle3 login'>
+<div class='align-center'>
+
+<div class='space'></div>
+{% if invalid %}
+	<p class='error'>The username and password you provided don't match. Please try again.</p>
+	<p>Have you <a href='/accounts/forgottenpassword/'>forgotten your password</a>?<br/>
+	   Or perhaps <a href='/accounts/forgottenusername/'>your username</a>?</p>
+       
+    <p>Neither of those links work yet, by the way, I'm only trying to *appear* helpful.
+
+	<div class='space'></div>
+{% endif %}
+<div class='align-center'>
+    <form action="" method="post" accept-charset="utf-8">{% csrf_token %}
+        <table class='form'>
+            <tr><th><label for="id_username">Username:</label></th><td><input id="id_username" type="text" name="username" maxlength="30" /></td></tr>
+            <tr><th><label for="id_password">Password:</label></th><td><input type="password" name="password" id="id_password" /></td></tr>
+        </table>
+        <div class='space'></div>
+    
+    <br/>
+    <p><input type="submit" value="Login &rarr;"></p>
+    </form>
+</div>
+{% endblock %}

urls.py

@@ -1,12 +1,11 @@
 from django.conf import settings
-from django.urls import path
 from django.conf.urls import url, include, re_path
 from django.views.generic.base import RedirectView 
 from django.views.generic.edit import UpdateView
 from django.views.generic.list import ListView
 from django.contrib import admin
 from django.contrib import auth
-from django.urls import reverse, resolve
+from django.urls import path, reverse, resolve
 
 from troggle.core.views import caves, statistics, survex
 from troggle.core.views.scans import scansingle, singlewallet, allwallets
@@ -59,7 +58,8 @@ else:
         path('/<path:filepath>', expofilessingle, name="single"), # local copy of EXPOFILES
         path('',                 expofilessingle, {'filepath': ""}, name="single"), 
         ]
-    
+
+# see https://docs.djangoproject.com/en/dev/topics/auth/default/
 # The URLs provided by include('django.contrib.auth.urls') are:
 
 # accounts/login/ [name='login']
@@ -71,6 +71,8 @@ else:
 # accounts/reset/<uidb64>/<token>/ [name='password_reset_confirm']
 # accounts/reset/done/ [name='password_reset_complete']
 
+# these worked in Django 2.2.24 but failed in .25 even though we include django.contrib.auth
+
 trogglepatterns = [
     path('expofiles/', include(expofilesurls)), # intercepted by Apache, if it is running.
     path('expofiles',  include(expofilesurls)), # curious interaction with the include() here, not just a slash problem.
@@ -90,10 +92,9 @@ trogglepatterns = [
     path('dwguploadnogit/<path:folder>', dwgupload,  {'gitdisable': 'yes'}, name='dwguploadnogit'), # used in testing
 
 # setting LOGIN_URL = '/accounts/login/' is default
-# url ENDS WITH this string
-    re_path(r'logout/$', expologout, name='expologout'), # higher precedence than /accounts/logout
-    re_path(r'login/$',  expologin,  name='expologin'),  # higher precedence than /accounts/login
-    #re_path(r'^accounts/', include('django.contrib.auth.urls')),  # from Dj3.0, see site-packages\registration\auth_urls_classes.py
+    path('accounts/logout/', expologout, name='expologout'), # same as in django.contrib.auth.urls
+    path('accounts/login/',  expologin,  name='expologin'),  # same as in django.contrib.auth.urls
+    #re_path(r'^accounts/', include('django.contrib.auth.urls')),  # see site-packages\registration\auth_urls_classes.py
 
 #   Persons - nasty surname recognition logic fails for 19 people!
     re_path(r'^person/(?P<first_name>[A-Z]*[a-z\-\'&;]*)[^a-zA-Z]*(?P<last_name>[a-z\-\']*[^a-zA-Z]*[\-]*[A-Z]*[a-zA-Z\-&;]*)/?', person, name="person"),