Fixup after rebase and pre- scripts fixed

core/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2020-06-20 14:27
+# Generated by Django 1.11.29 on 2020-06-20 16:03
 from __future__ import unicode_literals
 
 from django.conf import settings

deprecations.txt

@@ -4,8 +4,6 @@
   from collections import Counter, Iterator, Mapping, OrderedDict
 /mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/django/core/paginator.py:126: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
   class Page(collections.Sequence):
-/mnt/d/CUCC-Expo/t37/troggle/core/view_surveys.py:41: DeprecationWarning: invalid escape sequence \d
-  mname = re.match("(.*?)(?:-(\d+))?\.(png|jpg|jpeg)$(?i)", fname)
 /mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/registration/auth_urls_classes.py:19: DeprecationWarning: 
 include('registration.auth_urls') is deprecated and will be
 removed in django-registration 3.0. Use

diffsettings.txt

@@ -0,0 +1,43 @@
+ACCOUNT_ACTIVATION_DAYS = 3  ###
+AUTH_PROFILE_MODULE = 'core.person'  ###
+BASE_DIR = '/mnt/d/CUCC-Expo/t37'  ###
+CAVEDESCRIPTIONS = '/mnt/d/CUCC-Expo/t37/expoweb/cave_data'  ###
+CAVERN = 'cavern'  ###
+DEFAULT_LOGBOOK_FILE = 'logbook.html'  ###
+DEFAULT_LOGBOOK_PARSER = 'Parseloghtmltxt'  ###
+DIR_ROOT = ''  ###
+ENTRANCEDESCRIPTIONS = '/mnt/d/CUCC-Expo/t37/expoweb/entrance_data'  ###
+EXPOFILES = '/mnt/f/expofiles/'  ###
+EXPOUSER = 'expo'  ###
+EXPOUSERPASS = "nnn:gggggg"
+EXPOUSER_EMAIL = 'philip.sargent@gmail.com'  ###
+EXPOWEB = '/mnt/d/CUCC-Expo/t37/expoweb/'  ###
+EXPOWEB_URL = ''  ###
+FILES = '/mnt/f/expofiles/'  ###
+FIX_PERMISSIONS = []  ###
+JSLIB_URL = 'http://localhost:3777/javascript/'  ###
+LIBDIR = '/mnt/d/CUCC-Expo/t37/lib/python3.7/'  ###
+LOGBOOK_PARSER_SETTINGS = {'2010': ('2010/logbook.html', 'Parseloghtmltxt'), '2009': ('2009/2009logbook.txt', 'Parselogwikitxt'), '2008': ('2008/2008logbook.txt', 'Parselogwikitxt'), '2007': ('2007/logbook.html', 'Parseloghtmltxt'), '2006': ('2006/logbook/logbook_06.txt', 'Parselogwikitxt'), '2005': ('2005/logbook.html', 'Parseloghtmltxt'), '2004': ('2004/logbook.html', 'Parseloghtmltxt'), '2003': ('2003/logbook.html', 'Parseloghtml03'), '2002': ('2002/logbook.html', 'Parseloghtmltxt'), '2001': ('2001/log.htm', 'Parseloghtml01'), '2000': ('2000/log.htm', 'Parseloghtml01'), '1999': ('1999/log.htm', 'Parseloghtml01'), '1998': ('1998/log.htm', 'Parseloghtml01'), '1997': ('1997/log.htm', 'Parseloghtml01'), '1996': ('1996/log.htm', 'Parseloghtml01'), '1995': ('1995/log.htm', 'Parseloghtml01'), '1994': ('1994/log.htm', 'Parseloghtml01'), '1993': ('1993/log.htm', 'Parseloghtml01'), '1992': ('1992/log.htm', 'Parseloghtml01'), '1991': ('1991/log.htm', 'Parseloghtml01')}  ###
+LOGFILE = '/mnt/d/CUCC-Expo/t37/troggle/troggle.log'  ###
+MAX_LOGBOOK_ENTRY_TITLE_LENGTH = 200  ###
+NOTABLECAVESHREFS = ['264', '258', '204', '76', '107']  ###
+PHOTOS_URL = 'http://localhost:3777/photos/'  ###
+PUBLIC_SITE = False  ###
+PYTHON_PATH = '/mnt/d/CUCC-Expo/t37/troggle/'  ###
+QM_PATTERN = '\\[\\[\\s*[Qq][Mm]:([ABC]?)(\\d{4})-(\\d*)-(\\d*)\\]\\]'  ###
+REPOS_ROOT_PATH = '/mnt/d/CUCC-Expo/t37/'  ###
+ROOT_URLCONF = 'troggle.urls'  ###
+SERVERPORT = '3777'  ###
+SETTINGS_MODULE = 'settings'  ###
+SITE_ID = 1  ###
+SMART_APPEND_SLASH = True  ###
+SURVEX_DATA = '/mnt/d/CUCC-Expo/t37/loser/'  ###
+SURVEX_TOPNAME = '1623'  ###
+SURVEYS = '/mnt/d/CUCC-Expo/t37/'  ###
+SURVEYS_URL = 'http://localhost:3777/survey_scans/'  ###
+SURVEY_SCANS = '/mnt/f/expofiles/surveyscans/'  ###
+SVX_URL = 'http://localhost:3777/survex/'  ###
+THREEDCACHEDIR = '/mnt/d/CUCC-Expo/t37/expowebcache/3d/'  ###
+THREEDTOPOS = '3dtopos'  ###
+TUNNEL_DATA = '/mnt/d/CUCC-Expo/t37/drawings/'  ###
+URL_ROOT = 'http://localhost:3777/'  ###

django-patch/html_parser.py

@@ -1,132 +0,0 @@
-from django.utils.six.moves import html_parser as _html_parser
-import re
-import sys
-
-current_version = sys.version_info
-
-use_workaround = (
-    (current_version < (2, 7, 3)) or
-    (current_version >= (3, 0) and current_version < (3, 2, 3))
-)
-
-#HTMLParseError = _html_parser.HTMLParseError
-# see http://thefourtheye.in/2015/02/15/python-35-and-django-17s-htmlparseerror/
-# https://github.com/django/django/commit/b07aa52e8a8e4c7fdc7265f75ce2e7992e657ae9
-try:
-    HTMLParseError = _html_parser.HTMLParseError
-except AttributeError:
-    # create a dummy class for Python 3.5+ where it's been removed
-    class HTMLParseError(Exception):
-        pass
-        
-if not use_workaround:
-    if current_version >= (3, 4):
-        class HTMLParser(_html_parser.HTMLParser):
-            """Explicitly set convert_charrefs to be False.
-
-            This silences a deprecation warning on Python 3.4, but we can't do
-            it at call time because Python 2.7 does not have the keyword
-            argument.
-            """
-            def __init__(self, convert_charrefs=False, **kwargs):
-                _html_parser.HTMLParser.__init__(self, convert_charrefs=convert_charrefs, **kwargs)
-    else:
-        HTMLParser = _html_parser.HTMLParser
-else:
-    tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*')
-
-    class HTMLParser(_html_parser.HTMLParser):
-        """
-        Patched version of stdlib's HTMLParser with patch from:
-        http://bugs.python.org/issue670664
-        """
-        def __init__(self):
-            _html_parser.HTMLParser.__init__(self)
-            self.cdata_tag = None
-
-        def set_cdata_mode(self, tag):
-            try:
-                self.interesting = _html_parser.interesting_cdata
-            except AttributeError:
-                self.interesting = re.compile(r'</\s*%s\s*>' % tag.lower(), re.I)
-            self.cdata_tag = tag.lower()
-
-        def clear_cdata_mode(self):
-            self.interesting = _html_parser.interesting_normal
-            self.cdata_tag = None
-
-        # Internal -- handle starttag, return end or -1 if not terminated
-        def parse_starttag(self, i):
-            self.__starttag_text = None
-            endpos = self.check_for_whole_start_tag(i)
-            if endpos < 0:
-                return endpos
-            rawdata = self.rawdata
-            self.__starttag_text = rawdata[i:endpos]
-
-            # Now parse the data between i+1 and j into a tag and attrs
-            attrs = []
-            match = tagfind.match(rawdata, i + 1)
-            assert match, 'unexpected call to parse_starttag()'
-            k = match.end()
-            self.lasttag = tag = match.group(1).lower()
-
-            while k < endpos:
-                m = _html_parser.attrfind.match(rawdata, k)
-                if not m:
-                    break
-                attrname, rest, attrvalue = m.group(1, 2, 3)
-                if not rest:
-                    attrvalue = None
-                elif (attrvalue[:1] == '\'' == attrvalue[-1:] or
-                        attrvalue[:1] == '"' == attrvalue[-1:]):
-                    attrvalue = attrvalue[1:-1]
-                if attrvalue:
-                    attrvalue = self.unescape(attrvalue)
-                attrs.append((attrname.lower(), attrvalue))
-                k = m.end()
-
-            end = rawdata[k:endpos].strip()
-            if end not in (">", "/>"):
-                lineno, offset = self.getpos()
-                if "\n" in self.__starttag_text:
-                    lineno = lineno + self.__starttag_text.count("\n")
-                    offset = (len(self.__starttag_text)
-                             - self.__starttag_text.rfind("\n"))
-                else:
-                    offset = offset + len(self.__starttag_text)
-                self.error("junk characters in start tag: %r"
-                           % (rawdata[k:endpos][:20],))
-            if end.endswith('/>'):
-                # XHTML-style empty tag: <span attr="value" />
-                self.handle_startendtag(tag, attrs)
-            else:
-                self.handle_starttag(tag, attrs)
-                if tag in self.CDATA_CONTENT_ELEMENTS:
-                    self.set_cdata_mode(tag)  # <--------------------------- Changed
-            return endpos
-
-        # Internal -- parse endtag, return end or -1 if incomplete
-        def parse_endtag(self, i):
-            rawdata = self.rawdata
-            assert rawdata[i:i + 2] == "</", "unexpected call to parse_endtag"
-            match = _html_parser.endendtag.search(rawdata, i + 1)  # >
-            if not match:
-                return -1
-            j = match.end()
-            match = _html_parser.endtagfind.match(rawdata, i)  # </ + tag + >
-            if not match:
-                if self.cdata_tag is not None:  # *** add ***
-                    self.handle_data(rawdata[i:j])  # *** add ***
-                    return j  # *** add ***
-                self.error("bad end tag: %r" % (rawdata[i:j],))
-            # --- changed start ---------------------------------------------------
-            tag = match.group(1).strip()
-            if self.cdata_tag is not None:
-                if tag.lower() != self.cdata_tag:
-                    self.handle_data(rawdata[i:j])
-                    return j
-            # --- changed end -----------------------------------------------------
-            self.handle_endtag(tag.lower())
-            self.clear_cdata_mode()
-            return j

flatpages/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2020-06-20 14:27
+# Generated by Django 1.11.29 on 2020-06-20 16:03
 from __future__ import unicode_literals
 
 from django.db import migrations, models

lines-of-python.txt

@@ -31,7 +31,6 @@
 99 ./parsers/QMs.py
 102 ./parsers/people.py
 103 ./core/view_surveys.py
-103 ./django-patch/html_parser.py
 124 ./core/templatetags/wiki_markup.py
 135 ./utils.py
 156 ./flatpages/views.py
@@ -42,11 +41,10 @@
 217 ./core/views_logbooks.py
 256 ./core/views_survex.py
 276 ./profiles/views.py
-355 ./troggle-inspectdb.py
 359 ./databaseReset.py
+382 ./troggle-inspectdb.py
 387 ./core/views_caves.py
 431 ./parsers/survex.py
 462 ./core/migrations/0001_initial.py
 515 ./parsers/logbooks.py
-534 ./0001_initial.py
+5999
-6609

localsettingsWSL.py

@@ -0,0 +1,116 @@
+import sys
+import os
+import urllib.parse
+
+# link 'localsettings.py' to localsettingsWSL.py for use on a Windows 10 machine running WSL1
+print(" * importing troggle/localsettings.py")
+
+SERVERPORT = '3777'
+
+LIBDIR =   '/mnt/d/CUCC-Expo/t37/lib/python3.7/'
+#LIBDIR  = '/mnt/d/CUCC-Expo/deb37/lib/python3.7/'
+#LIBDIR  = '/mnt/d/CUCC-Expo/t38/lib/python3.8/'
+#LIBDIR  = '/usr/lib/python3.8/'
+
+REPOS_ROOT_PATH  = '/mnt/d/CUCC-Expo/t37/'
+#REPOS_ROOT_PATH = '/mnt/d/CUCC-Expo/t38/'
+#REPOS_ROOT_PATH = '/mnt/d/CUCC-Expo/deb37/'
+#REPOS_ROOT_PATH = '/mnt/d/CUCC-Expo/'
+
+MEDIA_ROOT = REPOS_ROOT_PATH + 'troggle/media/'
+#MEDIA_URL = urllib.parse.urljoin(URL_ROOT , '/site_media/')
+MEDIA_URL = '/site_media/'
+
+#STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views_surveys.py
+
+PUBLIC_SITE = False
+DEBUG = True
+
+# executables:
+CAVERN = 'cavern'
+THREEDTOPOS = '3dtopos'
+
+DATABASES = {
+    'default': { 
+        'ENGINE': 'django.db.backends.sqlite3', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
+        'NAME' : 'troggle.sqlite',
+ #       'NAME' : ':memory:',
+        'USER' : 'expo',        # Not used with sqlite3.
+        'PASSWORD' : 'sekrit',  # Not used with sqlite3.
+        'HOST' : '',            # Set to empty string for localhost. Not used with sqlite3.
+        'PORT' : '',            # Set to empty string for default. Not used with sqlite3.
+    }
+}
+# add in 290, 291, 358 when they don't make it crash horribly
+NOTABLECAVESHREFS = [ "264", "258", "204", "76", "107"]
+
+sys.path.append(REPOS_ROOT_PATH)
+sys.path.append(REPOS_ROOT_PATH + 'troggle')
+PYTHON_PATH = REPOS_ROOT_PATH + 'troggle/'
+
+LOGFILE = PYTHON_PATH + 'troggle.log'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [
+            PYTHON_PATH + "templates"
+        ],
+#        'APP_DIRS': True,
+        'OPTIONS': {
+            'debug': 'DEBUG',
+            'context_processors': [
+                'django.contrib.auth.context_processors.auth',
+                 'core.context.troggle_context', 
+                'django.template.context_processors.debug',
+                'django.template.context_processors.i18n',
+                'django.template.context_processors.media',
+                'django.template.context_processors.static',
+                'django.template.context_processors.tz',
+                'django.contrib.messages.context_processors.messages',
+            ],
+            'loaders': [
+                'django.template.loaders.filesystem.Loader',
+                'django.template.loaders.app_directories.Loader',
+                # insert your TEMPLATE_LOADERS here
+           ]
+       },
+    },
+]
+
+EXPOUSER = 'expo'
+EXPOUSERPASS = "nnn:gggggg"
+EXPOUSER_EMAIL = 'philip.sargent@gmail.com'
+
+EMAIL_HOST = "smtp.gmail.com"
+EMAIL_HOST_USER = "philip.sargent@gmail.com"
+EMAIL_HOST_PASSWORD = "insert-real-email-password-here"
+EMAIL_PORT=587
+EMAIL_USE_TLS = True
+
+SURVEX_DATA = REPOS_ROOT_PATH + 'loser/'
+TUNNEL_DATA = REPOS_ROOT_PATH + 'drawings/'
+THREEDCACHEDIR = REPOS_ROOT_PATH + 'expowebcache/3d/'
+
+EXPOWEB = REPOS_ROOT_PATH + 'expoweb/'
+SURVEYS = REPOS_ROOT_PATH
+SURVEY_SCANS = '/mnt/f/expofiles/surveyscans/'
+FILES = '/mnt/f/expofiles/'
+CAVEDESCRIPTIONS = os.path.join(EXPOWEB, "cave_data")
+ENTRANCEDESCRIPTIONS = os.path.join(EXPOWEB, "entrance_data")
+EXPOWEB_URL = ''
+SURVEYS_URL = '/survey_scans/'
+EXPOFILES ='/mnt/f/expofiles/'
+
+URL_ROOT = 'http://localhost:'+ SERVERPORT +'/'
+DIR_ROOT = ''#this should end in / if a value is given
+
+SURVEYS_URL = urllib.parse.urljoin(URL_ROOT , '/survey_scans/')
+PHOTOS_URL  = urllib.parse.urljoin(URL_ROOT , '/photos/')
+SVX_URL = urllib.parse.urljoin(URL_ROOT , '/survex/')
+
+
+STATIC_URL = urllib.parse.urljoin(URL_ROOT , '/static/') # used by Django admin pages. Do not delete.
+JSLIB_URL  = urllib.parse.urljoin(URL_ROOT , '/javascript/') # always fails, try to revive it ?
+
+print(" + finished importing troggle/localsettings.py")

pre-push.sh

@@ -8,6 +8,7 @@ python manage.py diffsettings | grep "###" > diffsettings.txt
 echo pip freeze.
 pip freeze > requirements.txt
 echo inspectdb.
+# this next line requires database setting to be troggle.sqlite:
 python manage.py inspectdb > troggle-inspectdb.py
 #egrep -in "unable|error" troggle-inspectdb.py
 echo remove passwords.

pre-run.sh

@@ -1,10 +1,6 @@
 #! /bin/sh
 # Do these before final testing, *not* just before pushing:
 # Philip Sargent 2020/06/20
-python reset-django.py
-echo After cleanup deletion, remake all migrations.
-python manage.py makemigrations >/dev/null
-python manage.py test
 echo Run inspectdb:
 python manage.py inspectdb > troggle-inspectdb.py
 # egrep -in "unable|error" troggle-inspectdb.py
@@ -15,3 +11,9 @@ find . -name \*.html -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v "0$" | a
 
 find . -name \*.py -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v "0$" | awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-python.txt
 echo `tail -1 lines-of-python.txt`   non-comment lines of python.
+
+# This deletes the database so must run after generating troggle-inspectdb.py
+python reset-django.py
+echo After cleanup deletion, remake all migrations.
+python manage.py makemigrations >/dev/null
+python manage.py test