From 3aca0d0c76364c8f9fa3fec43370f54f8270b43f Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 30 Jan 2023 23:04:11 +0000 Subject: [PATCH] ruf cleanup imports, bigly. --- _deploy/wsl/localsettingsWSL.py | 160 +++++++++++++-------------- core/TESTS/tests.py | 10 +- core/TESTS/tests_caves.py | 5 +- core/TESTS/tests_logins.py | 21 ++-- core/admin.py | 3 - core/forms.py | 5 - core/management/commands/dummycmd.py | 8 +- core/middleware.py | 4 +- core/models/caves.py | 24 +--- core/models/logbooks.py | 19 +--- core/models/survex.py | 6 - core/models/troggle.py | 14 --- core/models/wallets.py | 1 - core/utils.py | 20 +--- core/views/auth.py | 1 - core/views/caves.py | 19 +--- core/views/drawings.py | 5 - core/views/editor_helpers.py | 9 +- core/views/expo.py | 7 +- core/views/logbooks.py | 19 +--- core/views/other.py | 15 +-- core/views/prospect.py | 8 +- core/views/scans.py | 21 ++-- core/views/statistics.py | 11 +- core/views/survex.py | 18 +-- core/views/uploads.py | 45 +++----- parsers/locations.py | 3 +- parsers/survex.py | 3 +- requirements.txt | 6 + 29 files changed, 165 insertions(+), 325 deletions(-) diff --git a/_deploy/wsl/localsettingsWSL.py b/_deploy/wsl/localsettingsWSL.py index 79d9026..7998996 100644 --- a/_deploy/wsl/localsettingsWSL.py +++ b/_deploy/wsl/localsettingsWSL.py @@ -1,6 +1,4 @@ -import os import sys -import urllib.parse from pathlib import Path """Settings for a troggle installation which may vary among different @@ -22,11 +20,11 @@ Read https://adamj.eu/tech/2020/03/16/use-pathlib-in-your-django-project/ print(" * importing troggle/localsettings.py") -#----------------------------------------------------------------- +# ----------------------------------------------------------------- # THINK before you push this to a repo # - have you checked that credentials.py is in .gitignore ? # - we don't want to have to change the expo system password ! -#----------------------------------------------------------------- +# ----------------------------------------------------------------- # default values, real secrets imported from credentials.py SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py" @@ -34,10 +32,10 @@ EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py" EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py" -EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate. -#SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely +EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate. +# SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely -SERVERPORT = '8000' # not needed +SERVERPORT = "8000" # not needed PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) @@ -45,70 +43,70 @@ PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) # so we use pathlib which has been standard since python 3.4 # If pathlib is new to you, you will need to read https://realpython.com/python-pathlib/ -# --------------------- MEDIA redirections BEGIN --------------------- +# --------------------- MEDIA redirections BEGIN --------------------- REPOS_ROOT_PATH = Path(__file__).parent.parent -LIBDIR = REPOS_ROOT_PATH / 'lib' / PV -#LIBDIR = REPOS_ROOT_PATH / 'lib' / 'python3.9' +LIBDIR = REPOS_ROOT_PATH / "lib" / PV +# LIBDIR = REPOS_ROOT_PATH / 'lib' / 'python3.9' TROGGLE_PATH = Path(__file__).parent -TEMPLATE_PATH = TROGGLE_PATH / 'templates' -MEDIA_ROOT = TROGGLE_PATH / 'media' -JSLIB_ROOT = TROGGLE_PATH / 'media' / 'jslib' # used for CaveViewer JS utility +TEMPLATE_PATH = TROGGLE_PATH / "templates" +MEDIA_ROOT = TROGGLE_PATH / "media" +JSLIB_ROOT = TROGGLE_PATH / "media" / "jslib" # used for CaveViewer JS utility -#FILES = Path('/mnt/d/expofiles/') +# FILES = Path('/mnt/d/expofiles/') EXPOFILES = REPOS_ROOT_PATH / "expofiles" -SCANS_ROOT = EXPOFILES / 'surveyscans' -PHOTOS_ROOT = EXPOFILES / 'photos' +SCANS_ROOT = EXPOFILES / "surveyscans" +PHOTOS_ROOT = EXPOFILES / "photos" PHOTOS_YEAR = "2022" # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). -MEDIA_URL = '/site-media/' +MEDIA_URL = "/site-media/" -DIR_ROOT = Path('') #this should end in / if a value is given -URL_ROOT = '/' +DIR_ROOT = Path("") # this should end in / if a value is given +URL_ROOT = "/" # URL_ROOT = 'http://localhost:'+ SERVERPORT +'/' -#Note that these constants are not actually used in urls.py, they should be.. +# Note that these constants are not actually used in urls.py, they should be.. # and they all need to end with / so using 'Path' doesn't work.. -MEDIA_URL = Path(URL_ROOT , '/site_media/') -PHOTOS_URL = Path(URL_ROOT , '/photos/') +MEDIA_URL = Path(URL_ROOT, "/site_media/") +PHOTOS_URL = Path(URL_ROOT, "/photos/") -STATIC_URL = Path(URL_ROOT , '/static/') # used by Django admin pages. Do not delete. -JSLIB_URL = Path(URL_ROOT , '/javascript/') # used for CaveViewer JS utility +STATIC_URL = Path(URL_ROOT, "/static/") # used by Django admin pages. Do not delete. +JSLIB_URL = Path(URL_ROOT, "/javascript/") # used for CaveViewer JS utility -#STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py -# --------------------- MEDIA redirections END --------------------- +# STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py +# --------------------- MEDIA redirections END --------------------- PUBLIC_SITE = True -DEBUG = True # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES ! +DEBUG = True # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES ! CACHEDPAGES = True # experimental page cache for a handful of page types # executables: -CAVERN = 'cavern' # for parsing .svx files and producing .3d files -SURVEXPORT = 'survexport' # for parsing .3d files and producing .pos files +CAVERN = "cavern" # for parsing .svx files and producing .3d files +SURVEXPORT = "survexport" # for parsing .3d files and producing .pos files DBSQLITE = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - 'NAME' : '/home/philip/p11d5/troggle.sqlite', -# 'NAME' : ':memory:', - 'USER' : 'expo', # Not used with sqlite3. - 'PASSWORD' : 'sekrit', # Not used with sqlite3. - 'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT' : '', # Set to empty string for default. Not used with sqlite3. + "default": { + "ENGINE": "django.db.backends.sqlite3", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + "NAME": "/home/philip/p11d5/troggle.sqlite", + # 'NAME' : ':memory:', + "USER": "expo", # Not used with sqlite3. + "PASSWORD": "sekrit", # Not used with sqlite3. + "HOST": "", # Set to empty string for localhost. Not used with sqlite3. + "PORT": "", # Set to empty string for default. Not used with sqlite3. } } DBMARIADB = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - 'NAME' : 'troggle', # Or path to database file if using sqlite3. - 'USER' : 'expo', - 'PASSWORD' : 'my-secret-password-schwatzmooskogel', - 'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT' : '', # Set to empty string for default. Not used with sqlite3. + "default": { + "ENGINE": "django.db.backends.mysql", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + "NAME": "troggle", # Or path to database file if using sqlite3. + "USER": "expo", + "PASSWORD": "my-secret-password-schwatzmooskogel", + "HOST": "", # Set to empty string for localhost. Not used with sqlite3. + "PORT": "", # Set to empty string for default. Not used with sqlite3. } } @@ -120,59 +118,57 @@ if DBSWITCH == "sqlite": if DBSWITCH == "mariadb": DATABASES = DBMARIADB -NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"] +NOTABLECAVESHREFS = ["290", "291", "359", "264", "258", "204", "76", "107"] -PYTHON_PATH = REPOS_ROOT_PATH / 'troggle' +PYTHON_PATH = REPOS_ROOT_PATH / "troggle" -LOGFILE = PYTHON_PATH / 'troggle.log' +LOGFILE = PYTHON_PATH / "troggle.log" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - TEMPLATE_PATH - ], - 'OPTIONS': { - 'debug': 'DEBUG', - 'context_processors': [ + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [TEMPLATE_PATH], + "OPTIONS": { + "debug": "DEBUG", + "context_processors": [ # django.template.context_processors.csrf, # is always enabled and cannot be removed, sets csrf_token - 'django.contrib.auth.context_processors.auth', # knowledge of logged-on user & permissions - 'core.context.troggle_context', # in core/troggle.py - only used in expedition.html - 'django.template.context_processors.debug', - 'django.template.context_processors.i18n', - 'django.template.context_processors.media', # includes a variable MEDIA_URL - 'django.template.context_processors.static', # includes a variable STATIC_URL used by admin pages - 'django.template.context_processors.tz', - 'django.template.context_processors.request', # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar. - 'django.contrib.messages.context_processors.messages', + "django.contrib.auth.context_processors.auth", # knowledge of logged-on user & permissions + "core.context.troggle_context", # in core/troggle.py - only used in expedition.html + "django.template.context_processors.debug", + "django.template.context_processors.i18n", + "django.template.context_processors.media", # includes a variable MEDIA_URL + "django.template.context_processors.static", # includes a variable STATIC_URL used by admin pages + "django.template.context_processors.tz", + "django.template.context_processors.request", # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar. + "django.contrib.messages.context_processors.messages", ], - 'loaders': [ - 'django.template.loaders.filesystem.Loader', # default lcation is troggle/templates/ - 'django.template.loaders.app_directories.Loader', # needed for admin 'app' - ] - }, + "loaders": [ + "django.template.loaders.filesystem.Loader", # default lcation is troggle/templates/ + "django.template.loaders.app_directories.Loader", # needed for admin 'app' + ], + }, }, ] -EXPOUSER = 'expo' -EXPOUSER_EMAIL = 'philip.sargent@gmail.com' -EXPOADMINUSER = 'expoadmin' -EXPOADMINUSER_EMAIL = 'philip.sargent@gmail.com' +EXPOUSER = "expo" +EXPOUSER_EMAIL = "philip.sargent@gmail.com" +EXPOADMINUSER = "expoadmin" +EXPOADMINUSER_EMAIL = "philip.sargent@gmail.com" EMAIL_HOST = "smtp-auth.mythic-beasts.com" -EMAIL_HOST_USER = "django-test@klebos.net" # Philip Sargent really -EMAIL_PORT=587 +EMAIL_HOST_USER = "django-test@klebos.net" # Philip Sargent really +EMAIL_PORT = 587 EMAIL_USE_TLS = True -DEFAULT_FROM_EMAIL = 'django-test@klebos.net' +DEFAULT_FROM_EMAIL = "django-test@klebos.net" SURVEX_DATA = REPOS_ROOT_PATH / "loser" DRAWINGS_DATA = REPOS_ROOT_PATH / "drawings" -EXPOWEB = REPOS_ROOT_PATH / "expoweb" +EXPOWEB = REPOS_ROOT_PATH / "expoweb" -CAVEDESCRIPTIONS = EXPOWEB / "cave_data" -ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data" -EXPOWEB_URL = '' +CAVEDESCRIPTIONS = EXPOWEB / "cave_data" +ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data" +EXPOWEB_URL = "" # SCANS_URL = '/survey_scans/' # defunct, removed. # Sanitise these to be strings as all other code is expecting strings @@ -189,9 +185,9 @@ EXPOWEB_URL = '' # SCANS_ROOT = str(SCANS_ROOT) # EXPOFILES = str(EXPOFILES) # PHOTOS_ROOT = str(PHOTOS_ROOT) -STATIC_URL = str(STATIC_URL) +"/" -MEDIA_URL = str(MEDIA_URL) +"/" +STATIC_URL = str(STATIC_URL) + "/" +MEDIA_URL = str(MEDIA_URL) + "/" # PYTHON_PATH = str(PYTHON_PATH) # REPOS_ROOT_PATH = str(REPOS_ROOT_PATH) sys.path.append(str(REPOS_ROOT_PATH)) -sys.path.append(str(PYTHON_PATH)) \ No newline at end of file +sys.path.append(str(PYTHON_PATH)) diff --git a/core/TESTS/tests.py b/core/TESTS/tests.py index 5176859..3f60394 100644 --- a/core/TESTS/tests.py +++ b/core/TESTS/tests.py @@ -26,10 +26,8 @@ todo = """ADD TESTS when we are redirecting /expofiles/ to a remote file-deliver """ import re -import unittest -from http import HTTPStatus -from django.test import Client, SimpleTestCase, TestCase +from django.test import Client, TestCase # class SimplePageTest(unittest.TestCase): @@ -76,12 +74,12 @@ class PageTests(TestCase): def test_expoweb_dir(self): response = self.client.get("/handbook") - content = response.content.decode() + response.content.decode() self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm def test_expoweb_dirslash(self): response = self.client.get("/handbook/") - content = response.content.decode() + response.content.decode() self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm def test_expoweb_dir_no_index(self): @@ -211,7 +209,7 @@ class PageTests(TestCase): def test_page_admindocs_exped(self): # Get redirected to login page response = self.client.get("/admin/doc/models/core.expedition/") - content = response.content.decode() + response.content.decode() self.assertEqual(response.status_code, 302) def test_page_expofiles_root_dir(self): diff --git a/core/TESTS/tests_caves.py b/core/TESTS/tests_caves.py index 3d9270d..529c712 100644 --- a/core/TESTS/tests_caves.py +++ b/core/TESTS/tests_caves.py @@ -3,12 +3,11 @@ Modified for Expo April 2021. """ import re -import unittest -from django.test import Client, SimpleTestCase, TestCase +from django.test import Client, TestCase from troggle.core.models.caves import Area, Cave -from troggle.core.models.troggle import Expedition, Person, PersonExpedition +from troggle.core.models.troggle import Person, PersonExpedition class FixtureTests(TestCase): diff --git a/core/TESTS/tests_logins.py b/core/TESTS/tests_logins.py index 365ec04..0dae63e 100644 --- a/core/TESTS/tests_logins.py +++ b/core/TESTS/tests_logins.py @@ -7,14 +7,12 @@ Modified for Expo April 2021. import pathlib import re -import subprocess -import unittest from http import HTTPStatus -from django.test import Client, SimpleTestCase, TestCase, TransactionTestCase +from django.test import Client, TestCase import troggle.settings as settings -from troggle.core.models.survex import Wallet +from troggle.core.models.wallets import Wallet from troggle.core.models.troggle import Expedition @@ -112,14 +110,13 @@ class PostTests(TestCase): identified in the wallet """ c = self.client - w = self.wallet from django.contrib.auth.models import User u = User.objects.get(username="expotest") testyear = self.testyear self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") - logged_in = c.login(username=u.username, password="secretword") + c.login(username=u.username, password="secretword") with open("core/fixtures/test_upload_file.txt", "r") as testf: response = self.client.post( @@ -162,7 +159,7 @@ class PostTests(TestCase): u = User.objects.get(username="expotest") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") - logged_in = c.login(username=u.username, password="secretword") + c.login(username=u.username, password="secretword") with open("core/fixtures/test_upload_file.txt", "r") as testf: response = self.client.post( @@ -199,7 +196,7 @@ class PostTests(TestCase): u = User.objects.get(username="expotest") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") - logged_in = c.login(username=u.username, password="secretword") + c.login(username=u.username, password="secretword") rename = "RENAMED-FILE.JPG" with open("core/fixtures/test_upload_file.txt", "r") as testf: @@ -231,7 +228,7 @@ class PostTests(TestCase): u = User.objects.get(username="expotest") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") - logged_in = c.login(username=u.username, password="secretword") + c.login(username=u.username, password="secretword") response = self.client.post("/photoupload/", data={"photographer": "GussieFinkNottle"}) content = response.content.decode() @@ -257,7 +254,7 @@ class PostTests(TestCase): u = User.objects.get(username="expotest") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") - logged_in = c.login(username=u.username, password="secretword") + c.login(username=u.username, password="secretword") with open("core/fixtures/test_upload_file.pdf", "r") as testf: response = self.client.post( @@ -279,7 +276,7 @@ class PostTests(TestCase): u = User.objects.get(username="expotest") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") - logged_in = c.login(username=u.username, password="secretword") + c.login(username=u.username, password="secretword") with open("core/fixtures/test_upload_nosuffix", "r") as testf: response = self.client.post( @@ -383,7 +380,6 @@ class ComplexLoginTests(TestCase): self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get the Troggle Admin page") def test_noinfo_login(self): - from django.contrib.auth.models import User c = self.client # inherited from TestCase u = self.user @@ -401,7 +397,6 @@ class ComplexLoginTests(TestCase): self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get /noinfo/ content") def test_user_force(self): - from django.conf import settings c = self.client u = self.user diff --git a/core/admin.py b/core/admin.py index e60583e..bfbd727 100644 --- a/core/admin.py +++ b/core/admin.py @@ -1,7 +1,5 @@ -import django.forms as forms from django.contrib import admin from django.core import serializers -from django.forms import ModelForm from django.http import HttpResponse from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance @@ -17,7 +15,6 @@ from troggle.core.models.survex import ( ) from troggle.core.models.wallets import Wallet from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition -from troggle.core.views.other import exportlogbook """This code significantly adds to the capabilities of the Django Management control panel for Troggle data. In particular, it enables JSON export of any data with 'export_as_json' diff --git a/core/forms.py b/core/forms.py index 709ed62..b20354f 100644 --- a/core/forms.py +++ b/core/forms.py @@ -1,14 +1,9 @@ -import string -from datetime import date import django.forms as forms -from django.contrib.admin.widgets import AdminDateWidget from django.forms import ModelForm from django.forms.models import modelformset_factory from troggle.core.models.caves import Cave, CaveAndEntrance, Entrance -from troggle.core.models.logbooks import QM, LogbookEntry -from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.views.editor_helpers import HTMLarea # from tinymce.widgets import TinyMCE diff --git a/core/management/commands/dummycmd.py b/core/management/commands/dummycmd.py index b682c0f..1340beb 100644 --- a/core/management/commands/dummycmd.py +++ b/core/management/commands/dummycmd.py @@ -1,12 +1,6 @@ -import os -from optparse import make_option -from django.contrib.auth.models import User -from django.core import management -from django.core.management.base import BaseCommand, CommandError -from django.db import connection +from django.core.management.base import BaseCommand -import settings """this is now replaced by databaseRest.py diff --git a/core/middleware.py b/core/middleware.py index 54cdb11..686d7d5 100644 --- a/core/middleware.py +++ b/core/middleware.py @@ -1,6 +1,6 @@ from django import http from django.conf import settings -from django.urls import Resolver404, resolve, reverse +from django.urls import Resolver404, resolve """Non-standard django middleware is loaded from this file. @@ -66,7 +66,7 @@ class SmartAppendSlashMiddleware(object): def _resolves(url): try: # If the URL does not resolve, the function raises a Resolver404 exception (a subclass of Http404) - match = resolve(url) + resolve(url) # this will ALWAYS be resolved by expopages because it will produce pagenotfound if not the thing asked for # so handle this in expopages, not in middleware return True diff --git a/core/models/caves.py b/core/models/caves.py index 6123210..a18a996 100644 --- a/core/models/caves.py +++ b/core/models/caves.py @@ -1,29 +1,16 @@ -import datetime -import json -import operator import os import re -import string -import subprocess from collections import defaultdict from datetime import datetime, timezone from pathlib import Path -from urllib.parse import urljoin -from django.conf import settings -from django.contrib.auth.models import User -from django.contrib.contenttypes.models import ContentType -from django.core.files.storage import FileSystemStorage from django.db import models -from django.db.models import Max, Min -from django.shortcuts import render -from django.template import Context, loader -from django.urls import reverse +from django.template import loader import settings from troggle.core.models.logbooks import QM from troggle.core.models.survex import SurvexStation -from troggle.core.models.troggle import DataIssue, Expedition, Person, TroggleModel +from troggle.core.models.troggle import DataIssue, TroggleModel from troggle.core.utils import TROG, writetrogglefile # Use the TROG global object to cache the cave lookup list. No good for multi-user.. @@ -149,11 +136,11 @@ class Cave(TroggleModel): def get_absolute_url(self): if self.kataster_number: - href = self.kataster_number + pass elif self.unofficial_number: - href = self.unofficial_number + pass else: - href = self.official_name.lower() + self.official_name.lower() # return settings.URL_ROOT + '/cave/' + href + '/' # return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) # WRONG. This produces /cave/161 and should be /1623/161 return Path(settings.URL_ROOT) / self.url # not good Django style.. NEEDS actual URL @@ -392,7 +379,6 @@ class Entrance(TroggleModel): def cavelist(self): rs = [] - res = "" for e in CaveAndEntrance.objects.filter(entrance=self): if e.cave: rs.append(e.cave) diff --git a/core/models/logbooks.py b/core/models/logbooks.py index ecb7946..eac552a 100644 --- a/core/models/logbooks.py +++ b/core/models/logbooks.py @@ -1,28 +1,11 @@ -import datetime -import json -import operator -import os -import re -import string -import subprocess -from collections import defaultdict -from datetime import datetime, timezone from pathlib import Path from urllib.parse import urljoin -from django.conf import settings -from django.contrib.auth.models import User -from django.contrib.contenttypes.models import ContentType -from django.core.files.storage import FileSystemStorage from django.db import models -from django.db.models import Max, Min -from django.shortcuts import render -from django.template import Context, loader from django.urls import reverse import settings -from troggle.core.models.survex import SurvexStation -from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition, TroggleModel +from troggle.core.models.troggle import Expedition, TroggleModel """The model declarations LogBookEntry, PersonLogEntry, QM """ diff --git a/core/models/survex.py b/core/models/survex.py index 11ede65..338a235 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -1,17 +1,11 @@ -import datetime -import json -import operator import os import re -from functools import reduce -from pathlib import Path from urllib.parse import urljoin from django.conf import settings from django.db import models from django.urls import reverse -from troggle.core.models.wallets import Wallet # from troggle.core.models.troggle import DataIssue # circular import. Hmm diff --git a/core/models/troggle.py b/core/models/troggle.py index 8fe01f7..ee66b64 100644 --- a/core/models/troggle.py +++ b/core/models/troggle.py @@ -1,26 +1,12 @@ -import datetime -import os -import re -import resource -import string from decimal import Decimal, getcontext -from subprocess import call from urllib.parse import urljoin getcontext().prec = 2 # use 2 significant figures for decimal calculations -from django.conf import settings -from django.contrib import admin -from django.contrib.auth.models import User -from django.contrib.contenttypes.models import ContentType -from django.core.files.storage import FileSystemStorage from django.db import models -from django.template import Context, loader from django.urls import reverse import settings -import troggle.core.models.survex -from troggle.core.utils import get_process_memory """This file declares TroggleModel which inherits from django.db.models.Model All TroggleModel and models.Model subclasses inherit persistence in the django relational database. This is known as diff --git a/core/models/wallets.py b/core/models/wallets.py index 7b691d5..eb93d10 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -1,7 +1,6 @@ import datetime import json import operator -import os import re from functools import reduce from pathlib import Path diff --git a/core/utils.py b/core/utils.py index 6d0dced..2c0a3a2 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1,24 +1,12 @@ -import datetime import logging -import os import random -import re import resource -import string import subprocess -from decimal import Decimal, getcontext +from decimal import getcontext from pathlib import Path -from urllib.parse import urljoin getcontext().prec = 2 # use 2 significant figures for decimal calculations -from django.conf import settings -from django.contrib import admin -from django.contrib.auth.models import User -from django.contrib.contenttypes.models import ContentType -from django.db import models -from django.template import Context, loader -from django.urls import reverse import settings @@ -156,7 +144,7 @@ def write_and_commit(files, message): ) else: print(f"No change {filepah}") - cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True) + subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True) cp_status = subprocess.run([git, "status"], cwd=cwd, capture_output=True, text=True) # This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb if cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean": @@ -265,9 +253,9 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}): print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}") raise try: - msg = str(instance) + str(instance) except: - msg = f"FAULT getting __str__ for instance with lookupattribs: {lookupAttribs}:" + pass if created: logging.info(str(instance) + " was just added to the database for the first time. \n") diff --git a/core/views/auth.py b/core/views/auth.py index 9b303c9..2257dd0 100644 --- a/core/views/auth.py +++ b/core/views/auth.py @@ -1,4 +1,3 @@ -from builtins import str from django.conf import settings from django.contrib.auth import authenticate diff --git a/core/views/caves.py b/core/views/caves.py index 3ffbbbb..168e076 100644 --- a/core/views/caves.py +++ b/core/views/caves.py @@ -1,24 +1,18 @@ import os import re -import string import subprocess -import urllib.parse from pathlib import Path -from django import forms -from django.conf import settings from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.http import HttpResponse, HttpResponseNotFound, HttpResponseRedirect -from django.shortcuts import get_object_or_404, render -from django.urls import NoReverseMatch, reverse +from django.shortcuts import render +from django.urls import NoReverseMatch -import settings import troggle.settings as settings from troggle.core.forms import CaveAndEntranceFormSet, CaveForm, EntranceForm, EntranceLetterForm -from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance, EntranceSlug, GetCaveLookup, SurvexStation +from troggle.core.models.caves import Cave, CaveAndEntrance, Entrance, EntranceSlug, GetCaveLookup from troggle.core.models.logbooks import CaveSlug, QM -from troggle.core.models.troggle import DataIssue, Expedition -from troggle.core.utils import write_and_commit, writetrogglefile +from troggle.core.utils import write_and_commit from troggle.core.views import expo from .auth import login_required_if_public @@ -107,13 +101,13 @@ def getnotablecaves(): caves = Cave.objects.all().filter(kataster_number=kataster_number) for c in caves: # print(c.kataster_number, c.slug()) - if c.slug() != None: + if c.slug() is not None: notablecaves.append(c) return notablecaves def caveindex(request): - caves = Cave.objects.all() + Cave.objects.all() caves1623 = list(Cave.objects.filter(area__short_name="1623")) caves1626 = list(Cave.objects.filter(area__short_name="1626")) caves1623.sort(key=caveKey) @@ -201,7 +195,6 @@ def file3d(request, cave, cave_id): survexpath = Path(settings.SURVEX_DATA, cave.survex_file) threedname = Path(survexname).with_suffix(".3d") # removes .svx, replaces with .3d threedpath = Path(settings.SURVEX_DATA, threedname) - threedcachedir = Path(settings.SURVEX_DATA) # These if statements need refactoring more cleanly if cave.survex_file: diff --git a/core/views/drawings.py b/core/views/drawings.py index d6565a5..9fdaf52 100644 --- a/core/views/drawings.py +++ b/core/views/drawings.py @@ -1,9 +1,4 @@ -import os -import re -import stat from pathlib import Path -from urllib.parse import unquote as urlunquote -from urllib.parse import urljoin from django.conf import settings from django.http import HttpResponse diff --git a/core/views/editor_helpers.py b/core/views/editor_helpers.py index a49c571..cd1638b 100644 --- a/core/views/editor_helpers.py +++ b/core/views/editor_helpers.py @@ -1,12 +1,11 @@ import io -import re from pathlib import Path import django.forms as forms -from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse -from django.shortcuts import redirect, render -from django.template import Context, loader -from django.urls import resolve, reverse +from django.http import JsonResponse +from django.shortcuts import render +from django.template import loader +from django.urls import reverse from django.views.decorators.csrf import ensure_csrf_cookie from PIL import Image diff --git a/core/views/expo.py b/core/views/expo.py index ffd62a1..0ee78ca 100644 --- a/core/views/expo.py +++ b/core/views/expo.py @@ -4,14 +4,11 @@ from pathlib import Path from sys import getfilesystemencoding as sys_getfilesystemencoding from urllib.parse import unquote as urlunquote from urllib.parse import urljoin -from urllib.request import urlopen import django.forms as forms -from django.contrib import admin -from django.http import Http404, HttpResponse, HttpResponseRedirect +from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import redirect, render -from django.template import Context, loader -from django.urls import resolve, reverse +from django.urls import reverse from django.views.decorators.csrf import ensure_csrf_cookie import troggle.core.views.caves diff --git a/core/views/logbooks.py b/core/views/logbooks.py index 54c48c4..37d6aa9 100644 --- a/core/views/logbooks.py +++ b/core/views/logbooks.py @@ -1,28 +1,15 @@ -import datetime -import os.path -import re -import time -import django.db.models -from django.db.models import Max, Min -from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render -from django.template import Context, loader -from django.template.defaultfilters import slugify -from django.urls import reverse -from django.utils import timezone from django.views.generic.list import ListView import troggle.settings as settings from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry from troggle.core.models.survex import SurvexBlock -from troggle.core.models.troggle import Expedition, Person, PersonExpedition +from troggle.core.models.troggle import Expedition, Person from troggle.core.models.wallets import Wallet from troggle.core.utils import TROG from troggle.parsers.imports import import_logbook -from troggle.parsers.people import GetPersonExpeditionNameLookup -from .auth import login_required_if_public """These views are for logbook items when they appear in an 'expedition' page and for persons: their individual pages and their perseonexpedition pages. @@ -78,7 +65,7 @@ def expedition(request, expeditionname): expo = Expedition.objects.get(year=int(expeditionname)) except: message = ( - f"Expedition not found - database apparently empty, you probably need to do a full re-import of all data." + "Expedition not found - database apparently empty, you probably need to do a full re-import of all data." ) return render(request, "errors/generic.html", {"message": message}) @@ -92,7 +79,7 @@ def expedition(request, expeditionname): ts = TROG["pagecache"]["expedition"] # not much use unless single user! if settings.CACHEDPAGES: - nexpos = len(TROG["pagecache"]["expedition"]) + len(TROG["pagecache"]["expedition"]) # print(f'! - expo {expeditionname} CACHEDPAGES {nexpos} expo pages in cache.') if expeditionname in ts: # print('! - expo {expeditionanme} using cached page') diff --git a/core/views/other.py b/core/views/other.py index bad5f75..5552f71 100644 --- a/core/views/other.py +++ b/core/views/other.py @@ -1,23 +1,16 @@ -import os import re -import subprocess from pathlib import Path -from django import forms from django.conf import settings -from django.core.files.storage import FileSystemStorage, default_storage -from django.db.models import Q -from django.http import HttpResponse, HttpResponseRedirect +from django.http import HttpResponse from django.shortcuts import render -from django.template import Context, loader -from django.urls import reverse +from django.template import loader from troggle.core.models.caves import Cave -from troggle.core.models.logbooks import QM, LogbookEntry # , PersonLogEntry -from troggle.core.models.survex import DrawingFile +from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* -from troggle.core.models.troggle import Expedition, Person, PersonExpedition +from troggle.core.models.troggle import Expedition from troggle.parsers.imports import ( import_caves, import_drawingsfiles, diff --git a/core/views/prospect.py b/core/views/prospect.py index e20281a..a538ab6 100644 --- a/core/views/prospect.py +++ b/core/views/prospect.py @@ -1,15 +1,11 @@ import os -import re -import string -import urllib.parse from django.http import HttpResponse from django.shortcuts import render import troggle.settings as settings -from troggle.core.models.caves import Area, Cave, Entrance, SurvexStation +from troggle.core.models.caves import Area, Cave, Entrance from troggle.core.views.caves import caveKey -from troggle.parsers.locations import MapLocations # from pathlib import Path @@ -65,7 +61,7 @@ def prospecting(request): has been disabled. """ message = ( - f"This prospecting guide text report contains many broken URLs because of a major redesign\n" + "This prospecting guide text report contains many broken URLs because of a major redesign\n" + " to put caves into 1623/ and 1624/ folders in 2017. It was mostly useless because recent QM info was not in it anyway.\n\n" + 'It is disabled in the python code in "prospecting(request):" in troggle/core/views/prospect.py' ) diff --git a/core/views/scans.py b/core/views/scans.py index e1db7e6..39aebbc 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -1,19 +1,14 @@ import datetime -import os -import re -import stat from pathlib import Path from urllib.parse import unquote as urlunquote -from urllib.parse import urljoin -from urllib.request import urlopen from django.conf import settings -from django.db import transaction from django.http import HttpResponse from django.shortcuts import render from troggle.core.models.caves import GetCaveLookup -from troggle.core.models.survex import SingleScan, SurvexBlock, Wallet +from troggle.core.models.survex import SingleScan, SurvexBlock +from troggle.core.models.wallets import Wallet from troggle.core.models.troggle import DataIssue, Expedition, Person from troggle.core.views.expo import getmimetype @@ -158,7 +153,7 @@ def walletslistperson(request, first_name, last_name): fixsurvextick(w, w.ticks) return manywallets - print(f"-walletslistperson") + print("-walletslistperson") try: if last_name: @@ -176,7 +171,7 @@ def walletslistperson(request, first_name, last_name): manywallets = tickspersonwallet(p) expeditions = Expedition.objects.all() - print(f"--") + print("--") return render( request, "personwallets.html", @@ -209,7 +204,7 @@ def walletslistyear(request, year): return manywallets - print(f"-walletslistyear") + print("-walletslistyear") if year < 1976 or year > 2050: return render(request, "errors/generic.html", {"message": "Year out of range. Must be between 1976 and 2050"}) @@ -219,7 +214,7 @@ def walletslistyear(request, year): manywallets = ticksyearwallet(year) expeditions = Expedition.objects.all() expedition = expeditions.filter(year=year) - print(f"--") + print("--") return render( request, "yearwallets.html", @@ -235,7 +230,7 @@ def walletslistyear(request, year): def cavewallets(request, caveid): """Returns all the wallets for just one cave""" - print(f"-cavewalletsl") + print("-cavewalletsl") Gcavelookup = GetCaveLookup() if caveid in Gcavelookup: @@ -270,7 +265,7 @@ def cavewallets(request, caveid): w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc fixsurvextick(w, w.ticks) expeditions = Expedition.objects.all() - print(f"--") + print("--") return render( request, "cavewallets.html", diff --git a/core/views/statistics.py b/core/views/statistics.py index 1f39d5b..b7af823 100644 --- a/core/views/statistics.py +++ b/core/views/statistics.py @@ -1,21 +1,12 @@ -import datetime -import operator -import os.path -import re from collections import OrderedDict from pathlib import Path -import django.db.models -from django.db.models import Max, Min from django.shortcuts import render -from django.template import Context, loader -from django.template.defaultfilters import slugify -from django.utils import timezone import troggle.settings as settings from troggle.core.models.caves import Cave, Entrance from troggle.core.models.logbooks import LogbookEntry -from troggle.core.models.survex import SurvexBlock, SurvexStation +from troggle.core.models.survex import SurvexStation from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition from troggle.parsers.people import GetPersonExpeditionNameLookup, foreign_friends diff --git a/core/views/survex.py b/core/views/survex.py index 14b904e..3575bc9 100644 --- a/core/views/survex.py +++ b/core/views/survex.py @@ -7,19 +7,13 @@ from pathlib import Path from django import forms from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist -from django.http import Http404, HttpResponse, HttpResponseRedirect +from django.http import HttpResponse from django.shortcuts import render -from django.template.context_processors import csrf from django.views.decorators.csrf import ensure_csrf_cookie -import parsers.survex import troggle.settings as settings from troggle.core.models.caves import Cave -from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry -from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole -from troggle.core.models.troggle import Expedition, Person, PersonExpedition -from troggle.core.utils import WriteAndCommitError, only_commit -from troggle.parsers.people import GetPersonExpeditionNameLookup +from troggle.core.utils import only_commit """Everything that views survexfiles but also displays data on a cave or caves when there is ambiguity @@ -165,8 +159,8 @@ class SvxForm(forms.Form): ) # javascript seems to insert CRLF on WSL1 whatever you say. So fix that: - res = fout.write(rcode.replace("\r", "")) - res = fout.write("\n") + fout.write(rcode.replace("\r", "")) + fout.write("\n") fout.close() if socket.gethostname() == "expo": @@ -323,7 +317,7 @@ def process(survex_file): def threed(request, survex_file): filepath3d = survexdatasetpath / str(survex_file + ".3d") - filepathlog = survexdatasetpath / str(survex_file + ".log") + survexdatasetpath / str(survex_file + ".log") if filepath3d.is_file(): threed = open(filepath3d, "rb") return HttpResponse(threed, content_type="application/x-aven") @@ -447,7 +441,7 @@ def survexcaveslist(request): if os.path.isdir(gcavedir) and cavedir[0] != ".": subdirs, subsvx = identifycavedircontents(gcavedir) - caveid = check_cave_registered( + check_cave_registered( area, cavedir ) # should do this only once per database load or it will be slow survdirobj = [] diff --git a/core/views/uploads.py b/core/views/uploads.py index ebd326f..6763d8c 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -1,45 +1,30 @@ import datetime import json -import operator import os import re import socket import subprocess import urllib -from functools import reduce from pathlib import Path -from urllib.parse import unquote from django import forms -from django.conf import settings from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist -from django.core.files.storage import FileSystemStorage, default_storage -from django.db.models import Q -from django.http import HttpResponse, HttpResponseRedirect +from django.core.files.storage import FileSystemStorage +from django.http import HttpResponseRedirect from django.shortcuts import render -from django.template import Context, loader -from django.urls import reverse import settings from troggle.core.models.caves import Cave -from troggle.core.models.logbooks import QM, LogbookEntry # , PersonLogEntry -from troggle.core.models.survex import DrawingFile, SurvexBlock, SurvexFile, SurvexPersonRole, Wallet +from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry +from troggle.core.models.survex import DrawingFile, SurvexBlock, SurvexFile, SurvexPersonRole +from troggle.core.models.wallets import Wallet # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* -from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition +from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.views.caves import getCave from troggle.core.views.scans import caveifywallet, oldwallet # from troggle import settings -from troggle.parsers.imports import ( - import_caves, - import_drawingsfiles, - import_logbooks, - import_people, - import_QMs, - import_survex, - import_surveyscans, -) from troggle.parsers.scans import contentsjson from .auth import login_required_if_public @@ -194,13 +179,13 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): else: try: sxpath = str(Path(sx).with_suffix("")) - svxfile = SurvexFile.objects.get(path=sxpath) + SurvexFile.objects.get(path=sxpath) except MultipleObjectsReturned: # can happen if connecting a wallet to a survex file.. i think.. QSsvxfiles = SurvexFile.objects.filter(path=sxpath) for s in QSsvxfiles: print(s.path, s.cave, s.survexdirectory) - svxfile = QSsvxfiles[0] # dont' know how this happened, fix later.. + # QSsvxfiles[0] # dont' know how this happened, fix later.. except: file_complaint = ( f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.." @@ -286,7 +271,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): caveobject = None else: complaints.append( - f'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"' + 'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"' ) caveobject = None @@ -323,7 +308,6 @@ def scanupload(request, path=None): Should sanitise to ensure no spurious backslashes e.g. in windows style paths""" waldata = {} if contents_path.is_file(): - create = False # yes wallet exists because JSON exists, even if no files in the surveyscans folder, or even if that folder does not exist with open(contents_path) as json_file: try: waldata = json.load(json_file) @@ -333,7 +317,7 @@ def scanupload(request, path=None): DataIssue.objects.create(parser="scans", message=message, url=wurl) # set URL to this wallet folder raise else: # no JSON file exists - print(f"--- No JSON exists, so creating blank copy") + print("--- No JSON exists, so creating blank copy") waldata = WALLET_BLANK_JSON.copy() if not waldata["survex file"]: try: @@ -347,10 +331,9 @@ def scanupload(request, path=None): return waldata def save_json(jsondict): - newfolder = contents_path.parent # print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}') if not os.path.exists(contents_path.parent): - print(f"--- No wallet directory in :drawings: repo, so creating it") + print("--- No wallet directory in :drawings: repo, so creating it") os.makedirs(contents_path.parent) with open(contents_path, "w") as jfile: @@ -521,7 +504,7 @@ def scanupload(request, path=None): commit_json(wd) else: - print(f"--- INVALID JSON Update form submitted") + print("--- INVALID JSON Update form submitted") print(formj.errors) return render(request, "errors/generic.html", {"message": formj.errors}) @@ -671,7 +654,7 @@ def scanupload(request, path=None): # else: # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}') except: - message = f"Specified survex file not found - database may be empty, or this survex file is not *included anywhere." + message = "Specified survex file not found - database may be empty, or this survex file is not *included anywhere." # return render(request, 'errors/generic.html', {'message': message}) pass @@ -704,7 +687,7 @@ def scanupload(request, path=None): print(f" - Setting wallet name to {names[0]}") elif len(names) == 0: waldata["name"] = "" - print(f" - Setting wallet name blank") + print(" - Setting wallet name blank") else: waldata["name"] = f"several, please edit: {names}" print( diff --git a/parsers/locations.py b/parsers/locations.py index 15fc841..fbed49b 100644 --- a/parsers/locations.py +++ b/parsers/locations.py @@ -11,7 +11,8 @@ from pathlib import Path import troggle.settings as settings from troggle.core.models.caves import Cave, Entrance from troggle.core.models.logbooks import QM -from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation, Wallet +from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation +from troggle.core.models.wallets import Wallet from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.utils import chaosmonkey, get_process_memory from troggle.parsers.logbooks import GetCaveLookup diff --git a/parsers/survex.py b/parsers/survex.py index 6c06307..b55485a 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -11,7 +11,8 @@ from pathlib import Path import troggle.settings as settings from troggle.core.models.caves import Cave, Entrance from troggle.core.models.logbooks import QM -from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation, Wallet +from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation +from troggle.core.models.wallets import Wallet from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.utils import chaosmonkey, get_process_memory from troggle.parsers.logbooks import GetCaveLookup diff --git a/requirements.txt b/requirements.txt index 2ee6766..e0762b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,20 @@ asgiref==3.5.2 attrs==22.2.0 +black==22.12.0 click==8.1.3 colorama==0.4.6 coverage==6.5.0 Django==3.2 docutils==0.19 interrogate==1.5.0 +isort==5.11.4 +mypy-extensions==0.4.3 +pathspec==0.10.3 Pillow==9.3.0 +platformdirs==2.6.2 py==1.11.0 pytz==2022.6 +ruff==0.0.226 sqlparse==0.4.3 tabulate==0.9.0 toml==0.10.2