expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-22 07:11:52 +00:00
Code Issues 19 Projects 3 Activity

ruf cleanup imports, bigly.

Browse Source
This commit is contained in:
Philip Sargent 2023-01-30 23:04:11 +00:00
parent e35fccea5d
commit 3aca0d0c76
29 changed files with 165 additions and 325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

152
_deploy/wsl/localsettingsWSL.py
View File

@ -1,6 +1,4 @@
import os
import sys import sys
import urllib.parse
from pathlib import Path from pathlib import Path
"""Settings for a troggle installation which may vary among different """Settings for a troggle installation which may vary among different
@ -22,11 +20,11 @@ Read https://adamj.eu/tech/2020/03/16/use-pathlib-in-your-django-project/
print(" * importing troggle/localsettings.py") print(" * importing troggle/localsettings.py")
#----------------------------------------------------------------- # -----------------------------------------------------------------
# THINK before you push this to a repo # THINK before you push this to a repo
# - have you checked that credentials.py is in .gitignore ? # - have you checked that credentials.py is in .gitignore ?
# - we don't want to have to change the expo system password ! # - we don't want to have to change the expo system password !
#----------------------------------------------------------------- # -----------------------------------------------------------------
# default values, real secrets imported from credentials.py # default values, real secrets imported from credentials.py
SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py" SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py"
@ -34,10 +32,10 @@ EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py
EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py" EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"
EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py" EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"
EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate. EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate.
#SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely # SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely
SERVERPORT = '8000' # not needed SERVERPORT = "8000" # not needed
PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor)
@ -47,68 +45,68 @@ PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor)
# --------------------- MEDIA redirections BEGIN --------------------- # --------------------- MEDIA redirections BEGIN ---------------------
REPOS_ROOT_PATH = Path(__file__).parent.parent REPOS_ROOT_PATH = Path(__file__).parent.parent
LIBDIR = REPOS_ROOT_PATH / 'lib' / PV LIBDIR = REPOS_ROOT_PATH / "lib" / PV
#LIBDIR = REPOS_ROOT_PATH / 'lib' / 'python3.9' # LIBDIR = REPOS_ROOT_PATH / 'lib' / 'python3.9'
TROGGLE_PATH = Path(__file__).parent TROGGLE_PATH = Path(__file__).parent
TEMPLATE_PATH = TROGGLE_PATH / 'templates' TEMPLATE_PATH = TROGGLE_PATH / "templates"
MEDIA_ROOT = TROGGLE_PATH / 'media' MEDIA_ROOT = TROGGLE_PATH / "media"
JSLIB_ROOT = TROGGLE_PATH / 'media' / 'jslib' # used for CaveViewer JS utility JSLIB_ROOT = TROGGLE_PATH / "media" / "jslib" # used for CaveViewer JS utility
#FILES = Path('/mnt/d/expofiles/') # FILES = Path('/mnt/d/expofiles/')
EXPOFILES = REPOS_ROOT_PATH / "expofiles" EXPOFILES = REPOS_ROOT_PATH / "expofiles"
SCANS_ROOT = EXPOFILES / 'surveyscans' SCANS_ROOT = EXPOFILES / "surveyscans"
PHOTOS_ROOT = EXPOFILES / 'photos' PHOTOS_ROOT = EXPOFILES / "photos"
PHOTOS_YEAR = "2022" PHOTOS_YEAR = "2022"
# URL that handles the media served from MEDIA_ROOT. Make sure to use a # URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases). # trailing slash if there is a path component (optional in other cases).
MEDIA_URL = '/site-media/' MEDIA_URL = "/site-media/"
DIR_ROOT = Path('') #this should end in / if a value is given DIR_ROOT = Path("") # this should end in / if a value is given
URL_ROOT = '/' URL_ROOT = "/"
# URL_ROOT = 'http://localhost:'+ SERVERPORT +'/' # URL_ROOT = 'http://localhost:'+ SERVERPORT +'/'
#Note that these constants are not actually used in urls.py, they should be.. # Note that these constants are not actually used in urls.py, they should be..
# and they all need to end with / so using 'Path' doesn't work.. # and they all need to end with / so using 'Path' doesn't work..
MEDIA_URL = Path(URL_ROOT , '/site_media/') MEDIA_URL = Path(URL_ROOT, "/site_media/")
PHOTOS_URL = Path(URL_ROOT , '/photos/') PHOTOS_URL = Path(URL_ROOT, "/photos/")
STATIC_URL = Path(URL_ROOT , '/static/') # used by Django admin pages. Do not delete. STATIC_URL = Path(URL_ROOT, "/static/") # used by Django admin pages. Do not delete.
JSLIB_URL = Path(URL_ROOT , '/javascript/') # used for CaveViewer JS utility JSLIB_URL = Path(URL_ROOT, "/javascript/") # used for CaveViewer JS utility
#STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py # STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py
# --------------------- MEDIA redirections END --------------------- # --------------------- MEDIA redirections END ---------------------
PUBLIC_SITE = True PUBLIC_SITE = True
DEBUG = True # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES ! DEBUG = True # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES !
CACHEDPAGES = True # experimental page cache for a handful of page types CACHEDPAGES = True # experimental page cache for a handful of page types
# executables: # executables:
CAVERN = 'cavern' # for parsing .svx files and producing .3d files CAVERN = "cavern" # for parsing .svx files and producing .3d files
SURVEXPORT = 'survexport' # for parsing .3d files and producing .pos files SURVEXPORT = "survexport" # for parsing .3d files and producing .pos files
DBSQLITE = { DBSQLITE = {
'default': { "default": {
'ENGINE': 'django.db.backends.sqlite3', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. "ENGINE": "django.db.backends.sqlite3", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME' : '/home/philip/p11d5/troggle.sqlite', "NAME": "/home/philip/p11d5/troggle.sqlite",
# 'NAME' : ':memory:', # 'NAME' : ':memory:',
'USER' : 'expo', # Not used with sqlite3. "USER": "expo", # Not used with sqlite3.
'PASSWORD' : 'sekrit', # Not used with sqlite3. "PASSWORD": "sekrit", # Not used with sqlite3.
'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. "HOST": "", # Set to empty string for localhost. Not used with sqlite3.
'PORT' : '', # Set to empty string for default. Not used with sqlite3. "PORT": "", # Set to empty string for default. Not used with sqlite3.
} }
} }
DBMARIADB = { DBMARIADB = {
'default': { "default": {
'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. "ENGINE": "django.db.backends.mysql", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME' : 'troggle', # Or path to database file if using sqlite3. "NAME": "troggle", # Or path to database file if using sqlite3.
'USER' : 'expo', "USER": "expo",
'PASSWORD' : 'my-secret-password-schwatzmooskogel', "PASSWORD": "my-secret-password-schwatzmooskogel",
'HOST' : '', # Set to empty string for localhost. Not used with sqlite3. "HOST": "", # Set to empty string for localhost. Not used with sqlite3.
'PORT' : '', # Set to empty string for default. Not used with sqlite3. "PORT": "", # Set to empty string for default. Not used with sqlite3.
} }
} }
@ -120,59 +118,57 @@ if DBSWITCH == "sqlite":
if DBSWITCH == "mariadb": if DBSWITCH == "mariadb":
DATABASES = DBMARIADB DATABASES = DBMARIADB
NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"] NOTABLECAVESHREFS = ["290", "291", "359", "264", "258", "204", "76", "107"]
PYTHON_PATH = REPOS_ROOT_PATH / 'troggle' PYTHON_PATH = REPOS_ROOT_PATH / "troggle"
LOGFILE = PYTHON_PATH / 'troggle.log' LOGFILE = PYTHON_PATH / "troggle.log"
TEMPLATES = [ TEMPLATES = [
{ {
'BACKEND': 'django.template.backends.django.DjangoTemplates', "BACKEND": "django.template.backends.django.DjangoTemplates",
'DIRS': [ "DIRS": [TEMPLATE_PATH],
TEMPLATE_PATH "OPTIONS": {
], "debug": "DEBUG",
'OPTIONS': { "context_processors": [
'debug': 'DEBUG',
'context_processors': [
# django.template.context_processors.csrf, # is always enabled and cannot be removed, sets csrf_token # django.template.context_processors.csrf, # is always enabled and cannot be removed, sets csrf_token
'django.contrib.auth.context_processors.auth', # knowledge of logged-on user & permissions "django.contrib.auth.context_processors.auth", # knowledge of logged-on user & permissions
'core.context.troggle_context', # in core/troggle.py - only used in expedition.html "core.context.troggle_context", # in core/troggle.py - only used in expedition.html
'django.template.context_processors.debug', "django.template.context_processors.debug",
'django.template.context_processors.i18n', "django.template.context_processors.i18n",
'django.template.context_processors.media', # includes a variable MEDIA_URL "django.template.context_processors.media", # includes a variable MEDIA_URL
'django.template.context_processors.static', # includes a variable STATIC_URL used by admin pages "django.template.context_processors.static", # includes a variable STATIC_URL used by admin pages
'django.template.context_processors.tz', "django.template.context_processors.tz",
'django.template.context_processors.request', # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar. "django.template.context_processors.request", # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar.
'django.contrib.messages.context_processors.messages', "django.contrib.messages.context_processors.messages",
], ],
'loaders': [ "loaders": [
'django.template.loaders.filesystem.Loader', # default lcation is troggle/templates/ "django.template.loaders.filesystem.Loader", # default lcation is troggle/templates/
'django.template.loaders.app_directories.Loader', # needed for admin 'app' "django.template.loaders.app_directories.Loader", # needed for admin 'app'
] ],
}, },
}, },
] ]
EXPOUSER = 'expo' EXPOUSER = "expo"
EXPOUSER_EMAIL = 'philip.sargent@gmail.com' EXPOUSER_EMAIL = "philip.sargent@gmail.com"
EXPOADMINUSER = 'expoadmin' EXPOADMINUSER = "expoadmin"
EXPOADMINUSER_EMAIL = 'philip.sargent@gmail.com' EXPOADMINUSER_EMAIL = "philip.sargent@gmail.com"
EMAIL_HOST = "smtp-auth.mythic-beasts.com" EMAIL_HOST = "smtp-auth.mythic-beasts.com"
EMAIL_HOST_USER = "django-test@klebos.net" # Philip Sargent really EMAIL_HOST_USER = "django-test@klebos.net" # Philip Sargent really
EMAIL_PORT=587 EMAIL_PORT = 587
EMAIL_USE_TLS = True EMAIL_USE_TLS = True
DEFAULT_FROM_EMAIL = 'django-test@klebos.net' DEFAULT_FROM_EMAIL = "django-test@klebos.net"
SURVEX_DATA = REPOS_ROOT_PATH / "loser" SURVEX_DATA = REPOS_ROOT_PATH / "loser"
DRAWINGS_DATA = REPOS_ROOT_PATH / "drawings" DRAWINGS_DATA = REPOS_ROOT_PATH / "drawings"
EXPOWEB = REPOS_ROOT_PATH / "expoweb" EXPOWEB = REPOS_ROOT_PATH / "expoweb"
CAVEDESCRIPTIONS = EXPOWEB / "cave_data" CAVEDESCRIPTIONS = EXPOWEB / "cave_data"
ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data" ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data"
EXPOWEB_URL = '' EXPOWEB_URL = ""
# SCANS_URL = '/survey_scans/' # defunct, removed. # SCANS_URL = '/survey_scans/' # defunct, removed.
# Sanitise these to be strings as all other code is expecting strings # Sanitise these to be strings as all other code is expecting strings
@ -189,8 +185,8 @@ EXPOWEB_URL = ''
# SCANS_ROOT = str(SCANS_ROOT) # SCANS_ROOT = str(SCANS_ROOT)
# EXPOFILES = str(EXPOFILES) # EXPOFILES = str(EXPOFILES)
# PHOTOS_ROOT = str(PHOTOS_ROOT) # PHOTOS_ROOT = str(PHOTOS_ROOT)
STATIC_URL = str(STATIC_URL) +"/" STATIC_URL = str(STATIC_URL) + "/"
MEDIA_URL = str(MEDIA_URL) +"/" MEDIA_URL = str(MEDIA_URL) + "/"
# PYTHON_PATH = str(PYTHON_PATH) # PYTHON_PATH = str(PYTHON_PATH)
# REPOS_ROOT_PATH = str(REPOS_ROOT_PATH) # REPOS_ROOT_PATH = str(REPOS_ROOT_PATH)
sys.path.append(str(REPOS_ROOT_PATH)) sys.path.append(str(REPOS_ROOT_PATH))

10
core/TESTS/tests.py
View File

@ -26,10 +26,8 @@ todo = """ADD TESTS when we are redirecting /expofiles/ to a remote file-deliver
""" """
import re import re
import unittest
from http import HTTPStatus
from django.test import Client, SimpleTestCase, TestCase from django.test import Client, TestCase
# class SimplePageTest(unittest.TestCase): # class SimplePageTest(unittest.TestCase):
@ -76,12 +74,12 @@ class PageTests(TestCase):
def test_expoweb_dir(self): def test_expoweb_dir(self):
response = self.client.get("/handbook") response = self.client.get("/handbook")
content = response.content.decode() response.content.decode()
self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm
def test_expoweb_dirslash(self): def test_expoweb_dirslash(self):
response = self.client.get("/handbook/") response = self.client.get("/handbook/")
content = response.content.decode() response.content.decode()
self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm
def test_expoweb_dir_no_index(self): def test_expoweb_dir_no_index(self):
@ -211,7 +209,7 @@ class PageTests(TestCase):
def test_page_admindocs_exped(self): def test_page_admindocs_exped(self):
# Get redirected to login page # Get redirected to login page
response = self.client.get("/admin/doc/models/core.expedition/") response = self.client.get("/admin/doc/models/core.expedition/")
content = response.content.decode() response.content.decode()
self.assertEqual(response.status_code, 302) self.assertEqual(response.status_code, 302)
def test_page_expofiles_root_dir(self): def test_page_expofiles_root_dir(self):

5
core/TESTS/tests_caves.py
View File

@ -3,12 +3,11 @@ Modified for Expo April 2021.
""" """
import re import re
import unittest
from django.test import Client, SimpleTestCase, TestCase from django.test import Client, TestCase
from troggle.core.models.caves import Area, Cave from troggle.core.models.caves import Area, Cave
from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.models.troggle import Person, PersonExpedition
class FixtureTests(TestCase): class FixtureTests(TestCase):

21
core/TESTS/tests_logins.py
View File

@ -7,14 +7,12 @@ Modified for Expo April 2021.
import pathlib import pathlib
import re import re
import subprocess
import unittest
from http import HTTPStatus from http import HTTPStatus
from django.test import Client, SimpleTestCase, TestCase, TransactionTestCase from django.test import Client, TestCase
import troggle.settings as settings import troggle.settings as settings
from troggle.core.models.survex import Wallet from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import Expedition from troggle.core.models.troggle import Expedition
@ -112,14 +110,13 @@ class PostTests(TestCase):
identified in the wallet identified in the wallet
""" """
c = self.client c = self.client
w = self.wallet
from django.contrib.auth.models import User from django.contrib.auth.models import User
u = User.objects.get(username="expotest") u = User.objects.get(username="expotest")
testyear = self.testyear testyear = self.testyear
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword") c.login(username=u.username, password="secretword")
with open("core/fixtures/test_upload_file.txt", "r") as testf: with open("core/fixtures/test_upload_file.txt", "r") as testf:
response = self.client.post( response = self.client.post(
@ -162,7 +159,7 @@ class PostTests(TestCase):
u = User.objects.get(username="expotest") u = User.objects.get(username="expotest")
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword") c.login(username=u.username, password="secretword")
with open("core/fixtures/test_upload_file.txt", "r") as testf: with open("core/fixtures/test_upload_file.txt", "r") as testf:
response = self.client.post( response = self.client.post(
@ -199,7 +196,7 @@ class PostTests(TestCase):
u = User.objects.get(username="expotest") u = User.objects.get(username="expotest")
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword") c.login(username=u.username, password="secretword")
rename = "RENAMED-FILE.JPG" rename = "RENAMED-FILE.JPG"
with open("core/fixtures/test_upload_file.txt", "r") as testf: with open("core/fixtures/test_upload_file.txt", "r") as testf:
@ -231,7 +228,7 @@ class PostTests(TestCase):
u = User.objects.get(username="expotest") u = User.objects.get(username="expotest")
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword") c.login(username=u.username, password="secretword")
response = self.client.post("/photoupload/", data={"photographer": "GussieFinkNottle"}) response = self.client.post("/photoupload/", data={"photographer": "GussieFinkNottle"})
content = response.content.decode() content = response.content.decode()
@ -257,7 +254,7 @@ class PostTests(TestCase):
u = User.objects.get(username="expotest") u = User.objects.get(username="expotest")
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword") c.login(username=u.username, password="secretword")
with open("core/fixtures/test_upload_file.pdf", "r") as testf: with open("core/fixtures/test_upload_file.pdf", "r") as testf:
response = self.client.post( response = self.client.post(
@ -279,7 +276,7 @@ class PostTests(TestCase):
u = User.objects.get(username="expotest") u = User.objects.get(username="expotest")
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE") self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword") c.login(username=u.username, password="secretword")
with open("core/fixtures/test_upload_nosuffix", "r") as testf: with open("core/fixtures/test_upload_nosuffix", "r") as testf:
response = self.client.post( response = self.client.post(
@ -383,7 +380,6 @@ class ComplexLoginTests(TestCase):
self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get the Troggle Admin page") self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get the Troggle Admin page")
def test_noinfo_login(self): def test_noinfo_login(self):
from django.contrib.auth.models import User
c = self.client # inherited from TestCase c = self.client # inherited from TestCase
u = self.user u = self.user
@ -401,7 +397,6 @@ class ComplexLoginTests(TestCase):
self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get /noinfo/ content") self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get /noinfo/ content")
def test_user_force(self): def test_user_force(self):
from django.conf import settings
c = self.client c = self.client
u = self.user u = self.user

3
core/admin.py
View File

@ -1,7 +1,5 @@
import django.forms as forms
from django.contrib import admin from django.contrib import admin
from django.core import serializers from django.core import serializers
from django.forms import ModelForm
from django.http import HttpResponse from django.http import HttpResponse
from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance
@ -17,7 +15,6 @@ from troggle.core.models.survex import (
) )
from troggle.core.models.wallets import Wallet from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition
from troggle.core.views.other import exportlogbook
"""This code significantly adds to the capabilities of the Django Management control panel for Troggle data. """This code significantly adds to the capabilities of the Django Management control panel for Troggle data.
In particular, it enables JSON export of any data with 'export_as_json' In particular, it enables JSON export of any data with 'export_as_json'

5
core/forms.py
View File

@ -1,14 +1,9 @@
import string
from datetime import date
import django.forms as forms import django.forms as forms
from django.contrib.admin.widgets import AdminDateWidget
from django.forms import ModelForm from django.forms import ModelForm
from django.forms.models import modelformset_factory from django.forms.models import modelformset_factory
from troggle.core.models.caves import Cave, CaveAndEntrance, Entrance from troggle.core.models.caves import Cave, CaveAndEntrance, Entrance
from troggle.core.models.logbooks import QM, LogbookEntry
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.views.editor_helpers import HTMLarea from troggle.core.views.editor_helpers import HTMLarea
# from tinymce.widgets import TinyMCE # from tinymce.widgets import TinyMCE

8
core/management/commands/dummycmd.py
View File

@ -1,12 +1,6 @@
import os
from optparse import make_option
from django.contrib.auth.models import User from django.core.management.base import BaseCommand
from django.core import management
from django.core.management.base import BaseCommand, CommandError
from django.db import connection
import settings
"""this is now replaced by databaseRest.py """this is now replaced by databaseRest.py

4
core/middleware.py
View File

@ -1,6 +1,6 @@
from django import http from django import http
from django.conf import settings from django.conf import settings
from django.urls import Resolver404, resolve, reverse from django.urls import Resolver404, resolve
"""Non-standard django middleware is loaded from this file. """Non-standard django middleware is loaded from this file.
@ -66,7 +66,7 @@ class SmartAppendSlashMiddleware(object):
def _resolves(url): def _resolves(url):
try: try:
# If the URL does not resolve, the function raises a Resolver404 exception (a subclass of Http404) # If the URL does not resolve, the function raises a Resolver404 exception (a subclass of Http404)
match = resolve(url) resolve(url)
# this will ALWAYS be resolved by expopages because it will produce pagenotfound if not the thing asked for # this will ALWAYS be resolved by expopages because it will produce pagenotfound if not the thing asked for
# so handle this in expopages, not in middleware # so handle this in expopages, not in middleware
return True return True

24
core/models/caves.py
View File

@ -1,29 +1,16 @@
import datetime
import json
import operator
import os import os
import re import re
import string
import subprocess
from collections import defaultdict from collections import defaultdict
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from urllib.parse import urljoin
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.files.storage import FileSystemStorage
from django.db import models from django.db import models
from django.db.models import Max, Min from django.template import loader
from django.shortcuts import render
from django.template import Context, loader
from django.urls import reverse
import settings import settings
from troggle.core.models.logbooks import QM from troggle.core.models.logbooks import QM
from troggle.core.models.survex import SurvexStation from troggle.core.models.survex import SurvexStation
from troggle.core.models.troggle import DataIssue, Expedition, Person, TroggleModel from troggle.core.models.troggle import DataIssue, TroggleModel
from troggle.core.utils import TROG, writetrogglefile from troggle.core.utils import TROG, writetrogglefile
# Use the TROG global object to cache the cave lookup list. No good for multi-user.. # Use the TROG global object to cache the cave lookup list. No good for multi-user..
@ -149,11 +136,11 @@ class Cave(TroggleModel):
def get_absolute_url(self): def get_absolute_url(self):
if self.kataster_number: if self.kataster_number:
href = self.kataster_number pass
elif self.unofficial_number: elif self.unofficial_number:
href = self.unofficial_number pass
else: else:
href = self.official_name.lower() self.official_name.lower()
# return settings.URL_ROOT + '/cave/' + href + '/' # return settings.URL_ROOT + '/cave/' + href + '/'
# return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) # WRONG. This produces /cave/161 and should be /1623/161 # return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) # WRONG. This produces /cave/161 and should be /1623/161
return Path(settings.URL_ROOT) / self.url # not good Django style.. NEEDS actual URL return Path(settings.URL_ROOT) / self.url # not good Django style.. NEEDS actual URL
@ -392,7 +379,6 @@ class Entrance(TroggleModel):
def cavelist(self): def cavelist(self):
rs = [] rs = []
res = ""
for e in CaveAndEntrance.objects.filter(entrance=self): for e in CaveAndEntrance.objects.filter(entrance=self):
if e.cave: if e.cave:
rs.append(e.cave) rs.append(e.cave)

19
core/models/logbooks.py
View File

@ -1,28 +1,11 @@
import datetime
import json
import operator
import os
import re
import string
import subprocess
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from urllib.parse import urljoin from urllib.parse import urljoin
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.files.storage import FileSystemStorage
from django.db import models from django.db import models
from django.db.models import Max, Min
from django.shortcuts import render
from django.template import Context, loader
from django.urls import reverse from django.urls import reverse
import settings import settings
from troggle.core.models.survex import SurvexStation from troggle.core.models.troggle import Expedition, TroggleModel
from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition, TroggleModel
"""The model declarations LogBookEntry, PersonLogEntry, QM """The model declarations LogBookEntry, PersonLogEntry, QM
""" """

6
core/models/survex.py
View File

@ -1,17 +1,11 @@
import datetime
import json
import operator
import os import os
import re import re
from functools import reduce
from pathlib import Path
from urllib.parse import urljoin from urllib.parse import urljoin
from django.conf import settings from django.conf import settings
from django.db import models from django.db import models
from django.urls import reverse from django.urls import reverse
from troggle.core.models.wallets import Wallet
# from troggle.core.models.troggle import DataIssue # circular import. Hmm # from troggle.core.models.troggle import DataIssue # circular import. Hmm

14
core/models/troggle.py
View File

@ -1,26 +1,12 @@
import datetime
import os
import re
import resource
import string
from decimal import Decimal, getcontext from decimal import Decimal, getcontext
from subprocess import call
from urllib.parse import urljoin from urllib.parse import urljoin
getcontext().prec = 2 # use 2 significant figures for decimal calculations getcontext().prec = 2 # use 2 significant figures for decimal calculations
from django.conf import settings
from django.contrib import admin
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.files.storage import FileSystemStorage
from django.db import models from django.db import models
from django.template import Context, loader
from django.urls import reverse from django.urls import reverse
import settings import settings
import troggle.core.models.survex
from troggle.core.utils import get_process_memory
"""This file declares TroggleModel which inherits from django.db.models.Model """This file declares TroggleModel which inherits from django.db.models.Model
All TroggleModel and models.Model subclasses inherit persistence in the django relational database. This is known as All TroggleModel and models.Model subclasses inherit persistence in the django relational database. This is known as

1
core/models/wallets.py
View File

@ -1,7 +1,6 @@
import datetime import datetime
import json import json
import operator import operator
import os
import re import re
from functools import reduce from functools import reduce
from pathlib import Path from pathlib import Path

20
core/utils.py
View File

@ -1,24 +1,12 @@
import datetime
import logging import logging
import os
import random import random
import re
import resource import resource
import string
import subprocess import subprocess
from decimal import Decimal, getcontext from decimal import getcontext
from pathlib import Path from pathlib import Path
from urllib.parse import urljoin
getcontext().prec = 2 # use 2 significant figures for decimal calculations getcontext().prec = 2 # use 2 significant figures for decimal calculations
from django.conf import settings
from django.contrib import admin
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.template import Context, loader
from django.urls import reverse
import settings import settings
@ -156,7 +144,7 @@ def write_and_commit(files, message):
) )
else: else:
print(f"No change {filepah}") print(f"No change {filepah}")
cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True) subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True)
cp_status = subprocess.run([git, "status"], cwd=cwd, capture_output=True, text=True) cp_status = subprocess.run([git, "status"], cwd=cwd, capture_output=True, text=True)
# This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb # This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb
if cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean": if cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean":
@ -265,9 +253,9 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}") print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
raise raise
try: try:
msg = str(instance) str(instance)
except: except:
msg = f"FAULT getting __str__ for instance with lookupattribs: {lookupAttribs}:" pass
if created: if created:
logging.info(str(instance) + " was just added to the database for the first time. \n") logging.info(str(instance) + " was just added to the database for the first time. \n")

1
core/views/auth.py
View File

@ -1,4 +1,3 @@
from builtins import str
from django.conf import settings from django.conf import settings
from django.contrib.auth import authenticate from django.contrib.auth import authenticate

19
core/views/caves.py
View File

@ -1,24 +1,18 @@
import os import os
import re import re
import string
import subprocess import subprocess
import urllib.parse
from pathlib import Path from pathlib import Path
from django import forms
from django.conf import settings
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.http import HttpResponse, HttpResponseNotFound, HttpResponseRedirect from django.http import HttpResponse, HttpResponseNotFound, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render from django.shortcuts import render
from django.urls import NoReverseMatch, reverse from django.urls import NoReverseMatch
import settings
import troggle.settings as settings import troggle.settings as settings
from troggle.core.forms import CaveAndEntranceFormSet, CaveForm, EntranceForm, EntranceLetterForm from troggle.core.forms import CaveAndEntranceFormSet, CaveForm, EntranceForm, EntranceLetterForm
from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance, EntranceSlug, GetCaveLookup, SurvexStation from troggle.core.models.caves import Cave, CaveAndEntrance, Entrance, EntranceSlug, GetCaveLookup
from troggle.core.models.logbooks import CaveSlug, QM from troggle.core.models.logbooks import CaveSlug, QM
from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.utils import write_and_commit
from troggle.core.utils import write_and_commit, writetrogglefile
from troggle.core.views import expo from troggle.core.views import expo
from .auth import login_required_if_public from .auth import login_required_if_public
@ -107,13 +101,13 @@ def getnotablecaves():
caves = Cave.objects.all().filter(kataster_number=kataster_number) caves = Cave.objects.all().filter(kataster_number=kataster_number)
for c in caves: for c in caves:
# print(c.kataster_number, c.slug()) # print(c.kataster_number, c.slug())
if c.slug() != None: if c.slug() is not None:
notablecaves.append(c) notablecaves.append(c)
return notablecaves return notablecaves
def caveindex(request): def caveindex(request):
caves = Cave.objects.all() Cave.objects.all()
caves1623 = list(Cave.objects.filter(area__short_name="1623")) caves1623 = list(Cave.objects.filter(area__short_name="1623"))
caves1626 = list(Cave.objects.filter(area__short_name="1626")) caves1626 = list(Cave.objects.filter(area__short_name="1626"))
caves1623.sort(key=caveKey) caves1623.sort(key=caveKey)
@ -201,7 +195,6 @@ def file3d(request, cave, cave_id):
survexpath = Path(settings.SURVEX_DATA, cave.survex_file) survexpath = Path(settings.SURVEX_DATA, cave.survex_file)
threedname = Path(survexname).with_suffix(".3d") # removes .svx, replaces with .3d threedname = Path(survexname).with_suffix(".3d") # removes .svx, replaces with .3d
threedpath = Path(settings.SURVEX_DATA, threedname) threedpath = Path(settings.SURVEX_DATA, threedname)
threedcachedir = Path(settings.SURVEX_DATA)
# These if statements need refactoring more cleanly # These if statements need refactoring more cleanly
if cave.survex_file: if cave.survex_file:

5
core/views/drawings.py
View File

@ -1,9 +1,4 @@
import os
import re
import stat
from pathlib import Path from pathlib import Path
from urllib.parse import unquote as urlunquote
from urllib.parse import urljoin
from django.conf import settings from django.conf import settings
from django.http import HttpResponse from django.http import HttpResponse

9
core/views/editor_helpers.py
View File

@ -1,12 +1,11 @@
import io import io
import re
from pathlib import Path from pathlib import Path
import django.forms as forms import django.forms as forms
from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse from django.http import JsonResponse
from django.shortcuts import redirect, render from django.shortcuts import render
from django.template import Context, loader from django.template import loader
from django.urls import resolve, reverse from django.urls import reverse
from django.views.decorators.csrf import ensure_csrf_cookie from django.views.decorators.csrf import ensure_csrf_cookie
from PIL import Image from PIL import Image

7
core/views/expo.py
View File

@ -4,14 +4,11 @@ from pathlib import Path
from sys import getfilesystemencoding as sys_getfilesystemencoding from sys import getfilesystemencoding as sys_getfilesystemencoding
from urllib.parse import unquote as urlunquote from urllib.parse import unquote as urlunquote
from urllib.parse import urljoin from urllib.parse import urljoin
from urllib.request import urlopen
import django.forms as forms import django.forms as forms
from django.contrib import admin from django.http import HttpResponse, HttpResponseRedirect
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.shortcuts import redirect, render from django.shortcuts import redirect, render
from django.template import Context, loader from django.urls import reverse
from django.urls import resolve, reverse
from django.views.decorators.csrf import ensure_csrf_cookie from django.views.decorators.csrf import ensure_csrf_cookie
import troggle.core.views.caves import troggle.core.views.caves

19
core/views/logbooks.py
View File

@ -1,28 +1,15 @@
import datetime
import os.path
import re
import time
import django.db.models
from django.db.models import Max, Min
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render from django.shortcuts import render
from django.template import Context, loader
from django.template.defaultfilters import slugify
from django.urls import reverse
from django.utils import timezone
from django.views.generic.list import ListView from django.views.generic.list import ListView
import troggle.settings as settings import troggle.settings as settings
from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry
from troggle.core.models.survex import SurvexBlock from troggle.core.models.survex import SurvexBlock
from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.models.troggle import Expedition, Person
from troggle.core.models.wallets import Wallet from troggle.core.models.wallets import Wallet
from troggle.core.utils import TROG from troggle.core.utils import TROG
from troggle.parsers.imports import import_logbook from troggle.parsers.imports import import_logbook
from troggle.parsers.people import GetPersonExpeditionNameLookup
from .auth import login_required_if_public
"""These views are for logbook items when they appear in an 'expedition' page """These views are for logbook items when they appear in an 'expedition' page
and for persons: their individual pages and their perseonexpedition pages. and for persons: their individual pages and their perseonexpedition pages.
@ -78,7 +65,7 @@ def expedition(request, expeditionname):
expo = Expedition.objects.get(year=int(expeditionname)) expo = Expedition.objects.get(year=int(expeditionname))
except: except:
message = ( message = (
f"Expedition not found - database apparently empty, you probably need to do a full re-import of all data." "Expedition not found - database apparently empty, you probably need to do a full re-import of all data."
) )
return render(request, "errors/generic.html", {"message": message}) return render(request, "errors/generic.html", {"message": message})
@ -92,7 +79,7 @@ def expedition(request, expeditionname):
ts = TROG["pagecache"]["expedition"] # not much use unless single user! ts = TROG["pagecache"]["expedition"] # not much use unless single user!
if settings.CACHEDPAGES: if settings.CACHEDPAGES:
nexpos = len(TROG["pagecache"]["expedition"]) len(TROG["pagecache"]["expedition"])
# print(f'! - expo {expeditionname} CACHEDPAGES {nexpos} expo pages in cache.') # print(f'! - expo {expeditionname} CACHEDPAGES {nexpos} expo pages in cache.')
if expeditionname in ts: if expeditionname in ts:
# print('! - expo {expeditionanme} using cached page') # print('! - expo {expeditionanme} using cached page')

15
core/views/other.py
View File

@ -1,23 +1,16 @@
import os
import re import re
import subprocess
from pathlib import Path from pathlib import Path
from django import forms
from django.conf import settings from django.conf import settings
from django.core.files.storage import FileSystemStorage, default_storage from django.http import HttpResponse
from django.db.models import Q
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render from django.shortcuts import render
from django.template import Context, loader from django.template import loader
from django.urls import reverse
from troggle.core.models.caves import Cave from troggle.core.models.caves import Cave
from troggle.core.models.logbooks import QM, LogbookEntry # , PersonLogEntry from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
from troggle.core.models.survex import DrawingFile
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.models.troggle import Expedition
from troggle.parsers.imports import ( from troggle.parsers.imports import (
import_caves, import_caves,
import_drawingsfiles, import_drawingsfiles,

8
core/views/prospect.py
View File

@ -1,15 +1,11 @@
import os import os
import re
import string
import urllib.parse
from django.http import HttpResponse from django.http import HttpResponse
from django.shortcuts import render from django.shortcuts import render
import troggle.settings as settings import troggle.settings as settings
from troggle.core.models.caves import Area, Cave, Entrance, SurvexStation from troggle.core.models.caves import Area, Cave, Entrance
from troggle.core.views.caves import caveKey from troggle.core.views.caves import caveKey
from troggle.parsers.locations import MapLocations
# from pathlib import Path # from pathlib import Path
@ -65,7 +61,7 @@ def prospecting(request):
has been disabled. has been disabled.
""" """
message = ( message = (
f"This prospecting guide text report contains many broken URLs because of a major redesign\n" "This prospecting guide text report contains many broken URLs because of a major redesign\n"
+ " to put caves into 1623/ and 1624/ folders in 2017. It was mostly useless because recent QM info was not in it anyway.\n\n" + " to put caves into 1623/ and 1624/ folders in 2017. It was mostly useless because recent QM info was not in it anyway.\n\n"
+ 'It is disabled in the python code in "prospecting(request):" in troggle/core/views/prospect.py' + 'It is disabled in the python code in "prospecting(request):" in troggle/core/views/prospect.py'
) )

21
core/views/scans.py
View File

@ -1,19 +1,14 @@
import datetime import datetime
import os
import re
import stat
from pathlib import Path from pathlib import Path
from urllib.parse import unquote as urlunquote from urllib.parse import unquote as urlunquote
from urllib.parse import urljoin
from urllib.request import urlopen
from django.conf import settings from django.conf import settings
from django.db import transaction
from django.http import HttpResponse from django.http import HttpResponse
from django.shortcuts import render from django.shortcuts import render
from troggle.core.models.caves import GetCaveLookup from troggle.core.models.caves import GetCaveLookup
from troggle.core.models.survex import SingleScan, SurvexBlock, Wallet from troggle.core.models.survex import SingleScan, SurvexBlock
from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import DataIssue, Expedition, Person from troggle.core.models.troggle import DataIssue, Expedition, Person
from troggle.core.views.expo import getmimetype from troggle.core.views.expo import getmimetype
@ -158,7 +153,7 @@ def walletslistperson(request, first_name, last_name):
fixsurvextick(w, w.ticks) fixsurvextick(w, w.ticks)
return manywallets return manywallets
print(f"-walletslistperson") print("-walletslistperson")
try: try:
if last_name: if last_name:
@ -176,7 +171,7 @@ def walletslistperson(request, first_name, last_name):
manywallets = tickspersonwallet(p) manywallets = tickspersonwallet(p)
expeditions = Expedition.objects.all() expeditions = Expedition.objects.all()
print(f"--") print("--")
return render( return render(
request, request,
"personwallets.html", "personwallets.html",
@ -209,7 +204,7 @@ def walletslistyear(request, year):
return manywallets return manywallets
print(f"-walletslistyear") print("-walletslistyear")
if year < 1976 or year > 2050: if year < 1976 or year > 2050:
return render(request, "errors/generic.html", {"message": "Year out of range. Must be between 1976 and 2050"}) return render(request, "errors/generic.html", {"message": "Year out of range. Must be between 1976 and 2050"})
@ -219,7 +214,7 @@ def walletslistyear(request, year):
manywallets = ticksyearwallet(year) manywallets = ticksyearwallet(year)
expeditions = Expedition.objects.all() expeditions = Expedition.objects.all()
expedition = expeditions.filter(year=year) expedition = expeditions.filter(year=year)
print(f"--") print("--")
return render( return render(
request, request,
"yearwallets.html", "yearwallets.html",
@ -235,7 +230,7 @@ def walletslistyear(request, year):
def cavewallets(request, caveid): def cavewallets(request, caveid):
"""Returns all the wallets for just one cave""" """Returns all the wallets for just one cave"""
print(f"-cavewalletsl") print("-cavewalletsl")
Gcavelookup = GetCaveLookup() Gcavelookup = GetCaveLookup()
if caveid in Gcavelookup: if caveid in Gcavelookup:
@ -270,7 +265,7 @@ def cavewallets(request, caveid):
w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
fixsurvextick(w, w.ticks) fixsurvextick(w, w.ticks)
expeditions = Expedition.objects.all() expeditions = Expedition.objects.all()
print(f"--") print("--")
return render( return render(
request, request,
"cavewallets.html", "cavewallets.html",

11
core/views/statistics.py
View File

@ -1,21 +1,12 @@
import datetime
import operator
import os.path
import re
from collections import OrderedDict from collections import OrderedDict
from pathlib import Path from pathlib import Path
import django.db.models
from django.db.models import Max, Min
from django.shortcuts import render from django.shortcuts import render
from django.template import Context, loader
from django.template.defaultfilters import slugify
from django.utils import timezone
import troggle.settings as settings import troggle.settings as settings
from troggle.core.models.caves import Cave, Entrance from troggle.core.models.caves import Cave, Entrance
from troggle.core.models.logbooks import LogbookEntry from troggle.core.models.logbooks import LogbookEntry
from troggle.core.models.survex import SurvexBlock, SurvexStation from troggle.core.models.survex import SurvexStation
from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition
from troggle.parsers.people import GetPersonExpeditionNameLookup, foreign_friends from troggle.parsers.people import GetPersonExpeditionNameLookup, foreign_friends

18
core/views/survex.py
View File

@ -7,19 +7,13 @@ from pathlib import Path
from django import forms from django import forms
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.http import Http404, HttpResponse, HttpResponseRedirect from django.http import HttpResponse
from django.shortcuts import render from django.shortcuts import render
from django.template.context_processors import csrf
from django.views.decorators.csrf import ensure_csrf_cookie from django.views.decorators.csrf import ensure_csrf_cookie
import parsers.survex
import troggle.settings as settings import troggle.settings as settings
from troggle.core.models.caves import Cave from troggle.core.models.caves import Cave
from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry from troggle.core.utils import only_commit
from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.utils import WriteAndCommitError, only_commit
from troggle.parsers.people import GetPersonExpeditionNameLookup
"""Everything that views survexfiles """Everything that views survexfiles
but also displays data on a cave or caves when there is ambiguity but also displays data on a cave or caves when there is ambiguity
@ -165,8 +159,8 @@ class SvxForm(forms.Form):
) )
# javascript seems to insert CRLF on WSL1 whatever you say. So fix that: # javascript seems to insert CRLF on WSL1 whatever you say. So fix that:
res = fout.write(rcode.replace("\r", "")) fout.write(rcode.replace("\r", ""))
res = fout.write("\n") fout.write("\n")
fout.close() fout.close()
if socket.gethostname() == "expo": if socket.gethostname() == "expo":
@ -323,7 +317,7 @@ def process(survex_file):
def threed(request, survex_file): def threed(request, survex_file):
filepath3d = survexdatasetpath / str(survex_file + ".3d") filepath3d = survexdatasetpath / str(survex_file + ".3d")
filepathlog = survexdatasetpath / str(survex_file + ".log") survexdatasetpath / str(survex_file + ".log")
if filepath3d.is_file(): if filepath3d.is_file():
threed = open(filepath3d, "rb") threed = open(filepath3d, "rb")
return HttpResponse(threed, content_type="application/x-aven") return HttpResponse(threed, content_type="application/x-aven")
@ -447,7 +441,7 @@ def survexcaveslist(request):
if os.path.isdir(gcavedir) and cavedir[0] != ".": if os.path.isdir(gcavedir) and cavedir[0] != ".":
subdirs, subsvx = identifycavedircontents(gcavedir) subdirs, subsvx = identifycavedircontents(gcavedir)
caveid = check_cave_registered( check_cave_registered(
area, cavedir area, cavedir
) # should do this only once per database load or it will be slow ) # should do this only once per database load or it will be slow
survdirobj = [] survdirobj = []

45
core/views/uploads.py
View File

@ -1,45 +1,30 @@
import datetime import datetime
import json import json
import operator
import os import os
import re import re
import socket import socket
import subprocess import subprocess
import urllib import urllib
from functools import reduce
from pathlib import Path from pathlib import Path
from urllib.parse import unquote
from django import forms from django import forms
from django.conf import settings
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.core.files.storage import FileSystemStorage, default_storage from django.core.files.storage import FileSystemStorage
from django.db.models import Q from django.http import HttpResponseRedirect
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render from django.shortcuts import render
from django.template import Context, loader
from django.urls import reverse
import settings import settings
from troggle.core.models.caves import Cave from troggle.core.models.caves import Cave
from troggle.core.models.logbooks import QM, LogbookEntry # , PersonLogEntry from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
from troggle.core.models.survex import DrawingFile, SurvexBlock, SurvexFile, SurvexPersonRole, Wallet from troggle.core.models.survex import DrawingFile, SurvexBlock, SurvexFile, SurvexPersonRole
from troggle.core.models.wallets import Wallet
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.views.caves import getCave from troggle.core.views.caves import getCave
from troggle.core.views.scans import caveifywallet, oldwallet from troggle.core.views.scans import caveifywallet, oldwallet
# from troggle import settings # from troggle import settings
from troggle.parsers.imports import (
import_caves,
import_drawingsfiles,
import_logbooks,
import_people,
import_QMs,
import_survex,
import_surveyscans,
)
from troggle.parsers.scans import contentsjson from troggle.parsers.scans import contentsjson
from .auth import login_required_if_public from .auth import login_required_if_public
@ -194,13 +179,13 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
else: else:
try: try:
sxpath = str(Path(sx).with_suffix("")) sxpath = str(Path(sx).with_suffix(""))
svxfile = SurvexFile.objects.get(path=sxpath) SurvexFile.objects.get(path=sxpath)
except MultipleObjectsReturned: except MultipleObjectsReturned:
# can happen if connecting a wallet to a survex file.. i think.. # can happen if connecting a wallet to a survex file.. i think..
QSsvxfiles = SurvexFile.objects.filter(path=sxpath) QSsvxfiles = SurvexFile.objects.filter(path=sxpath)
for s in QSsvxfiles: for s in QSsvxfiles:
print(s.path, s.cave, s.survexdirectory) print(s.path, s.cave, s.survexdirectory)
svxfile = QSsvxfiles[0] # dont' know how this happened, fix later.. # QSsvxfiles[0] # dont' know how this happened, fix later..
except: except:
file_complaint = ( file_complaint = (
f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.." f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.."
@ -286,7 +271,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
caveobject = None caveobject = None
else: else:
complaints.append( complaints.append(
f'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"' 'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"'
) )
caveobject = None caveobject = None
@ -323,7 +308,6 @@ def scanupload(request, path=None):
Should sanitise to ensure no spurious backslashes e.g. in windows style paths""" Should sanitise to ensure no spurious backslashes e.g. in windows style paths"""
waldata = {} waldata = {}
if contents_path.is_file(): if contents_path.is_file():
create = False # yes wallet exists because JSON exists, even if no files in the surveyscans folder, or even if that folder does not exist
with open(contents_path) as json_file: with open(contents_path) as json_file:
try: try:
waldata = json.load(json_file) waldata = json.load(json_file)
@ -333,7 +317,7 @@ def scanupload(request, path=None):
DataIssue.objects.create(parser="scans", message=message, url=wurl) # set URL to this wallet folder DataIssue.objects.create(parser="scans", message=message, url=wurl) # set URL to this wallet folder
raise raise
else: # no JSON file exists else: # no JSON file exists
print(f"--- No JSON exists, so creating blank copy") print("--- No JSON exists, so creating blank copy")
waldata = WALLET_BLANK_JSON.copy() waldata = WALLET_BLANK_JSON.copy()
if not waldata["survex file"]: if not waldata["survex file"]:
try: try:
@ -347,10 +331,9 @@ def scanupload(request, path=None):
return waldata return waldata
def save_json(jsondict): def save_json(jsondict):
newfolder = contents_path.parent
# print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}') # print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}')
if not os.path.exists(contents_path.parent): if not os.path.exists(contents_path.parent):
print(f"--- No wallet directory in :drawings: repo, so creating it") print("--- No wallet directory in :drawings: repo, so creating it")
os.makedirs(contents_path.parent) os.makedirs(contents_path.parent)
with open(contents_path, "w") as jfile: with open(contents_path, "w") as jfile:
@ -521,7 +504,7 @@ def scanupload(request, path=None):
commit_json(wd) commit_json(wd)
else: else:
print(f"--- INVALID JSON Update form submitted") print("--- INVALID JSON Update form submitted")
print(formj.errors) print(formj.errors)
return render(request, "errors/generic.html", {"message": formj.errors}) return render(request, "errors/generic.html", {"message": formj.errors})
@ -671,7 +654,7 @@ def scanupload(request, path=None):
# else: # else:
# print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}') # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}')
except: except:
message = f"Specified survex file not found - database may be empty, or this survex file is not *included anywhere." message = "Specified survex file not found - database may be empty, or this survex file is not *included anywhere."
# return render(request, 'errors/generic.html', {'message': message}) # return render(request, 'errors/generic.html', {'message': message})
pass pass
@ -704,7 +687,7 @@ def scanupload(request, path=None):
print(f" - Setting wallet name to {names[0]}") print(f" - Setting wallet name to {names[0]}")
elif len(names) == 0: elif len(names) == 0:
waldata["name"] = "" waldata["name"] = ""
print(f" - Setting wallet name blank") print(" - Setting wallet name blank")
else: else:
waldata["name"] = f"several, please edit: {names}" waldata["name"] = f"several, please edit: {names}"
print( print(

3
parsers/locations.py
View File

@ -11,7 +11,8 @@ from pathlib import Path
import troggle.settings as settings import troggle.settings as settings
from troggle.core.models.caves import Cave, Entrance from troggle.core.models.caves import Cave, Entrance
from troggle.core.models.logbooks import QM from troggle.core.models.logbooks import QM
from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation, Wallet from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation
from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.utils import chaosmonkey, get_process_memory from troggle.core.utils import chaosmonkey, get_process_memory
from troggle.parsers.logbooks import GetCaveLookup from troggle.parsers.logbooks import GetCaveLookup

3
parsers/survex.py
View File

@ -11,7 +11,8 @@ from pathlib import Path
import troggle.settings as settings import troggle.settings as settings
from troggle.core.models.caves import Cave, Entrance from troggle.core.models.caves import Cave, Entrance
from troggle.core.models.logbooks import QM from troggle.core.models.logbooks import QM
from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation, Wallet from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation
from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.utils import chaosmonkey, get_process_memory from troggle.core.utils import chaosmonkey, get_process_memory
from troggle.parsers.logbooks import GetCaveLookup from troggle.parsers.logbooks import GetCaveLookup

6
requirements.txt
View File

@ -1,14 +1,20 @@
asgiref==3.5.2 asgiref==3.5.2
attrs==22.2.0 attrs==22.2.0
black==22.12.0
click==8.1.3 click==8.1.3
colorama==0.4.6 colorama==0.4.6
coverage==6.5.0 coverage==6.5.0
Django==3.2 Django==3.2
docutils==0.19 docutils==0.19
interrogate==1.5.0 interrogate==1.5.0
isort==5.11.4
mypy-extensions==0.4.3
pathspec==0.10.3
Pillow==9.3.0 Pillow==9.3.0
platformdirs==2.6.2
py==1.11.0 py==1.11.0
pytz==2022.6 pytz==2022.6
ruff==0.0.226
sqlparse==0.4.3 sqlparse==0.4.3
tabulate==0.9.0 tabulate==0.9.0
toml==0.10.2 toml==0.10.2