expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-21 14:51:51 +00:00
Code Issues 19 Projects 3 Activity

ruf cleanup imports, bigly.

Browse Source
This commit is contained in:
Philip Sargent 2023-01-30 23:04:11 +00:00
parent e35fccea5d
commit 3aca0d0c76
29 changed files with 165 additions and 325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

160
_deploy/wsl/localsettingsWSL.py
View File

@ -1,6 +1,4 @@
import os
import sys
import urllib.parse
from pathlib import Path
"""Settings for a troggle installation which may vary among different
@ -22,11 +20,11 @@ Read https://adamj.eu/tech/2020/03/16/use-pathlib-in-your-django-project/
print(" * importing troggle/localsettings.py")
#-----------------------------------------------------------------
# -----------------------------------------------------------------
# THINK before you push this to a repo
# - have you checked that credentials.py is in .gitignore ?
# - we don't want to have to change the expo system password !
#-----------------------------------------------------------------
# -----------------------------------------------------------------
# default values, real secrets imported from credentials.py
SECRET_KEY = "real-SECRET_KEY--imported-from-localsettings.py"
@ -34,10 +32,10 @@ EXPOUSERPASS = "nnn:gggggg - real-expo-password---imported-from-localsettings.py
EXPOADMINUSERPASS = "gggggg:nnn - real-expo-password---imported-from-localsettings.py"
EMAIL_HOST_PASSWORD = "real-email-password---imported-from-localsettings.py"
EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate.
#SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely
EXPOFILESREMOTE = False # if True, then re-routes urls in expofiles to remote sever. Tests are then less accurate.
# SECURE_SSL_REDIRECT = True # breaks 7 tests in test suite 301 not 200 (or 302) and runserver fails completely
SERVERPORT = '8000' # not needed
SERVERPORT = "8000" # not needed
PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor)
@ -45,70 +43,70 @@ PV = "python" + str(sys.version_info.major) + "." + str(sys.version_info.minor)
# so we use pathlib which has been standard since python 3.4
# If pathlib is new to you, you will need to read https://realpython.com/python-pathlib/
# --------------------- MEDIA redirections BEGIN ---------------------
# --------------------- MEDIA redirections BEGIN ---------------------
REPOS_ROOT_PATH = Path(__file__).parent.parent
LIBDIR = REPOS_ROOT_PATH / 'lib' / PV
#LIBDIR = REPOS_ROOT_PATH / 'lib' / 'python3.9'
LIBDIR = REPOS_ROOT_PATH / "lib" / PV
# LIBDIR = REPOS_ROOT_PATH / 'lib' / 'python3.9'
TROGGLE_PATH = Path(__file__).parent
TEMPLATE_PATH = TROGGLE_PATH / 'templates'
MEDIA_ROOT = TROGGLE_PATH / 'media'
JSLIB_ROOT = TROGGLE_PATH / 'media' / 'jslib' # used for CaveViewer JS utility
TEMPLATE_PATH = TROGGLE_PATH / "templates"
MEDIA_ROOT = TROGGLE_PATH / "media"
JSLIB_ROOT = TROGGLE_PATH / "media" / "jslib" # used for CaveViewer JS utility
#FILES = Path('/mnt/d/expofiles/')
# FILES = Path('/mnt/d/expofiles/')
EXPOFILES = REPOS_ROOT_PATH / "expofiles"
SCANS_ROOT = EXPOFILES / 'surveyscans'
PHOTOS_ROOT = EXPOFILES / 'photos'
SCANS_ROOT = EXPOFILES / "surveyscans"
PHOTOS_ROOT = EXPOFILES / "photos"
PHOTOS_YEAR = "2022"
# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
MEDIA_URL = '/site-media/'
MEDIA_URL = "/site-media/"
DIR_ROOT = Path('') #this should end in / if a value is given
URL_ROOT = '/'
DIR_ROOT = Path("") # this should end in / if a value is given
URL_ROOT = "/"
# URL_ROOT = 'http://localhost:'+ SERVERPORT +'/'
#Note that these constants are not actually used in urls.py, they should be..
# Note that these constants are not actually used in urls.py, they should be..
# and they all need to end with / so using 'Path' doesn't work..
MEDIA_URL = Path(URL_ROOT , '/site_media/')
PHOTOS_URL = Path(URL_ROOT , '/photos/')
MEDIA_URL = Path(URL_ROOT, "/site_media/")
PHOTOS_URL = Path(URL_ROOT, "/photos/")
STATIC_URL = Path(URL_ROOT , '/static/') # used by Django admin pages. Do not delete.
JSLIB_URL = Path(URL_ROOT , '/javascript/') # used for CaveViewer JS utility
STATIC_URL = Path(URL_ROOT, "/static/") # used by Django admin pages. Do not delete.
JSLIB_URL = Path(URL_ROOT, "/javascript/") # used for CaveViewer JS utility
#STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py
# --------------------- MEDIA redirections END ---------------------
# STATIC_ROOT removed after merging content into MEDIA_ROOT. See urls.py & core/views/surveys.py
# --------------------- MEDIA redirections END ---------------------
PUBLIC_SITE = True
DEBUG = True # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES !
DEBUG = True # Always keep this True, even when on public server. Otherwise NO USEFUL ERROR MESSAGES !
CACHEDPAGES = True # experimental page cache for a handful of page types
# executables:
CAVERN = 'cavern' # for parsing .svx files and producing .3d files
SURVEXPORT = 'survexport' # for parsing .3d files and producing .pos files
CAVERN = "cavern" # for parsing .svx files and producing .3d files
SURVEXPORT = "survexport" # for parsing .3d files and producing .pos files
DBSQLITE = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME' : '/home/philip/p11d5/troggle.sqlite',
# 'NAME' : ':memory:',
'USER' : 'expo', # Not used with sqlite3.
'PASSWORD' : 'sekrit', # Not used with sqlite3.
'HOST' : '', # Set to empty string for localhost. Not used with sqlite3.
'PORT' : '', # Set to empty string for default. Not used with sqlite3.
"default": {
"ENGINE": "django.db.backends.sqlite3", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
"NAME": "/home/philip/p11d5/troggle.sqlite",
# 'NAME' : ':memory:',
"USER": "expo", # Not used with sqlite3.
"PASSWORD": "sekrit", # Not used with sqlite3.
"HOST": "", # Set to empty string for localhost. Not used with sqlite3.
"PORT": "", # Set to empty string for default. Not used with sqlite3.
}
}
DBMARIADB = {
'default': {
'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME' : 'troggle', # Or path to database file if using sqlite3.
'USER' : 'expo',
'PASSWORD' : 'my-secret-password-schwatzmooskogel',
'HOST' : '', # Set to empty string for localhost. Not used with sqlite3.
'PORT' : '', # Set to empty string for default. Not used with sqlite3.
"default": {
"ENGINE": "django.db.backends.mysql", # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
"NAME": "troggle", # Or path to database file if using sqlite3.
"USER": "expo",
"PASSWORD": "my-secret-password-schwatzmooskogel",
"HOST": "", # Set to empty string for localhost. Not used with sqlite3.
"PORT": "", # Set to empty string for default. Not used with sqlite3.
}
}
@ -120,59 +118,57 @@ if DBSWITCH == "sqlite":
if DBSWITCH == "mariadb":
DATABASES = DBMARIADB
NOTABLECAVESHREFS = [ "290", "291", "359", "264", "258", "204", "76", "107"]
NOTABLECAVESHREFS = ["290", "291", "359", "264", "258", "204", "76", "107"]
PYTHON_PATH = REPOS_ROOT_PATH / 'troggle'
PYTHON_PATH = REPOS_ROOT_PATH / "troggle"
LOGFILE = PYTHON_PATH / 'troggle.log'
LOGFILE = PYTHON_PATH / "troggle.log"
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [
TEMPLATE_PATH
],
'OPTIONS': {
'debug': 'DEBUG',
'context_processors': [
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [TEMPLATE_PATH],
"OPTIONS": {
"debug": "DEBUG",
"context_processors": [
# django.template.context_processors.csrf, # is always enabled and cannot be removed, sets csrf_token
'django.contrib.auth.context_processors.auth', # knowledge of logged-on user & permissions
'core.context.troggle_context', # in core/troggle.py - only used in expedition.html
'django.template.context_processors.debug',
'django.template.context_processors.i18n',
'django.template.context_processors.media', # includes a variable MEDIA_URL
'django.template.context_processors.static', # includes a variable STATIC_URL used by admin pages
'django.template.context_processors.tz',
'django.template.context_processors.request', # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar.
'django.contrib.messages.context_processors.messages',
"django.contrib.auth.context_processors.auth", # knowledge of logged-on user & permissions
"core.context.troggle_context", # in core/troggle.py - only used in expedition.html
"django.template.context_processors.debug",
"django.template.context_processors.i18n",
"django.template.context_processors.media", # includes a variable MEDIA_URL
"django.template.context_processors.static", # includes a variable STATIC_URL used by admin pages
"django.template.context_processors.tz",
"django.template.context_processors.request", # must be enabled in DjangoTemplates (TEMPLATES) in order to use the admin navigation sidebar.
"django.contrib.messages.context_processors.messages",
],
'loaders': [
'django.template.loaders.filesystem.Loader', # default lcation is troggle/templates/
'django.template.loaders.app_directories.Loader', # needed for admin 'app'
]
},
"loaders": [
"django.template.loaders.filesystem.Loader", # default lcation is troggle/templates/
"django.template.loaders.app_directories.Loader", # needed for admin 'app'
],
},
},
]
EXPOUSER = 'expo'
EXPOUSER_EMAIL = 'philip.sargent@gmail.com'
EXPOADMINUSER = 'expoadmin'
EXPOADMINUSER_EMAIL = 'philip.sargent@gmail.com'
EXPOUSER = "expo"
EXPOUSER_EMAIL = "philip.sargent@gmail.com"
EXPOADMINUSER = "expoadmin"
EXPOADMINUSER_EMAIL = "philip.sargent@gmail.com"
EMAIL_HOST = "smtp-auth.mythic-beasts.com"
EMAIL_HOST_USER = "django-test@klebos.net" # Philip Sargent really
EMAIL_PORT=587
EMAIL_HOST_USER = "django-test@klebos.net" # Philip Sargent really
EMAIL_PORT = 587
EMAIL_USE_TLS = True
DEFAULT_FROM_EMAIL = 'django-test@klebos.net'
DEFAULT_FROM_EMAIL = "django-test@klebos.net"
SURVEX_DATA = REPOS_ROOT_PATH / "loser"
DRAWINGS_DATA = REPOS_ROOT_PATH / "drawings"
EXPOWEB = REPOS_ROOT_PATH / "expoweb"
EXPOWEB = REPOS_ROOT_PATH / "expoweb"
CAVEDESCRIPTIONS = EXPOWEB / "cave_data"
ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data"
EXPOWEB_URL = ''
CAVEDESCRIPTIONS = EXPOWEB / "cave_data"
ENTRANCEDESCRIPTIONS = EXPOWEB / "entrance_data"
EXPOWEB_URL = ""
# SCANS_URL = '/survey_scans/' # defunct, removed.
# Sanitise these to be strings as all other code is expecting strings
@ -189,9 +185,9 @@ EXPOWEB_URL = ''
# SCANS_ROOT = str(SCANS_ROOT)
# EXPOFILES = str(EXPOFILES)
# PHOTOS_ROOT = str(PHOTOS_ROOT)
STATIC_URL = str(STATIC_URL) +"/"
MEDIA_URL = str(MEDIA_URL) +"/"
STATIC_URL = str(STATIC_URL) + "/"
MEDIA_URL = str(MEDIA_URL) + "/"
# PYTHON_PATH = str(PYTHON_PATH)
# REPOS_ROOT_PATH = str(REPOS_ROOT_PATH)
sys.path.append(str(REPOS_ROOT_PATH))
sys.path.append(str(PYTHON_PATH))
sys.path.append(str(PYTHON_PATH))

10
core/TESTS/tests.py
View File

@ -26,10 +26,8 @@ todo = """ADD TESTS when we are redirecting /expofiles/ to a remote file-deliver
"""
import re
import unittest
from http import HTTPStatus
from django.test import Client, SimpleTestCase, TestCase
from django.test import Client, TestCase
# class SimplePageTest(unittest.TestCase):
@ -76,12 +74,12 @@ class PageTests(TestCase):
def test_expoweb_dir(self):
response = self.client.get("/handbook")
content = response.content.decode()
response.content.decode()
self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm
def test_expoweb_dirslash(self):
response = self.client.get("/handbook/")
content = response.content.decode()
response.content.decode()
self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm
def test_expoweb_dir_no_index(self):
@ -211,7 +209,7 @@ class PageTests(TestCase):
def test_page_admindocs_exped(self):
# Get redirected to login page
response = self.client.get("/admin/doc/models/core.expedition/")
content = response.content.decode()
response.content.decode()
self.assertEqual(response.status_code, 302)
def test_page_expofiles_root_dir(self):

5
core/TESTS/tests_caves.py
View File

@ -3,12 +3,11 @@ Modified for Expo April 2021.
"""
import re
import unittest
from django.test import Client, SimpleTestCase, TestCase
from django.test import Client, TestCase
from troggle.core.models.caves import Area, Cave
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.models.troggle import Person, PersonExpedition
class FixtureTests(TestCase):

21
core/TESTS/tests_logins.py
View File

@ -7,14 +7,12 @@ Modified for Expo April 2021.
import pathlib
import re
import subprocess
import unittest
from http import HTTPStatus
from django.test import Client, SimpleTestCase, TestCase, TransactionTestCase
from django.test import Client, TestCase
import troggle.settings as settings
from troggle.core.models.survex import Wallet
from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import Expedition
@ -112,14 +110,13 @@ class PostTests(TestCase):
identified in the wallet
"""
c = self.client
w = self.wallet
from django.contrib.auth.models import User
u = User.objects.get(username="expotest")
testyear = self.testyear
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword")
c.login(username=u.username, password="secretword")
with open("core/fixtures/test_upload_file.txt", "r") as testf:
response = self.client.post(
@ -162,7 +159,7 @@ class PostTests(TestCase):
u = User.objects.get(username="expotest")
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword")
c.login(username=u.username, password="secretword")
with open("core/fixtures/test_upload_file.txt", "r") as testf:
response = self.client.post(
@ -199,7 +196,7 @@ class PostTests(TestCase):
u = User.objects.get(username="expotest")
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword")
c.login(username=u.username, password="secretword")
rename = "RENAMED-FILE.JPG"
with open("core/fixtures/test_upload_file.txt", "r") as testf:
@ -231,7 +228,7 @@ class PostTests(TestCase):
u = User.objects.get(username="expotest")
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword")
c.login(username=u.username, password="secretword")
response = self.client.post("/photoupload/", data={"photographer": "GussieFinkNottle"})
content = response.content.decode()
@ -257,7 +254,7 @@ class PostTests(TestCase):
u = User.objects.get(username="expotest")
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword")
c.login(username=u.username, password="secretword")
with open("core/fixtures/test_upload_file.pdf", "r") as testf:
response = self.client.post(
@ -279,7 +276,7 @@ class PostTests(TestCase):
u = User.objects.get(username="expotest")
self.assertTrue(u.is_active, "User '" + u.username + "' is INACTIVE")
logged_in = c.login(username=u.username, password="secretword")
c.login(username=u.username, password="secretword")
with open("core/fixtures/test_upload_nosuffix", "r") as testf:
response = self.client.post(
@ -383,7 +380,6 @@ class ComplexLoginTests(TestCase):
self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get the Troggle Admin page")
def test_noinfo_login(self):
from django.contrib.auth.models import User
c = self.client # inherited from TestCase
u = self.user
@ -401,7 +397,6 @@ class ComplexLoginTests(TestCase):
self.assertIsNotNone(t, "Logged in as '" + u.username + "' but failed to get /noinfo/ content")
def test_user_force(self):
from django.conf import settings
c = self.client
u = self.user

3
core/admin.py
View File

@ -1,7 +1,5 @@
import django.forms as forms
from django.contrib import admin
from django.core import serializers
from django.forms import ModelForm
from django.http import HttpResponse
from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance
@ -17,7 +15,6 @@ from troggle.core.models.survex import (
)
from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition
from troggle.core.views.other import exportlogbook
"""This code significantly adds to the capabilities of the Django Management control panel for Troggle data.
In particular, it enables JSON export of any data with 'export_as_json'

5
core/forms.py
View File

@ -1,14 +1,9 @@
import string
from datetime import date
import django.forms as forms
from django.contrib.admin.widgets import AdminDateWidget
from django.forms import ModelForm
from django.forms.models import modelformset_factory
from troggle.core.models.caves import Cave, CaveAndEntrance, Entrance
from troggle.core.models.logbooks import QM, LogbookEntry
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.views.editor_helpers import HTMLarea
# from tinymce.widgets import TinyMCE

8
core/management/commands/dummycmd.py
View File

@ -1,12 +1,6 @@
import os
from optparse import make_option
from django.contrib.auth.models import User
from django.core import management
from django.core.management.base import BaseCommand, CommandError
from django.db import connection
from django.core.management.base import BaseCommand
import settings
"""this is now replaced by databaseRest.py

4
core/middleware.py
View File

@ -1,6 +1,6 @@
from django import http
from django.conf import settings
from django.urls import Resolver404, resolve, reverse
from django.urls import Resolver404, resolve
"""Non-standard django middleware is loaded from this file.
@ -66,7 +66,7 @@ class SmartAppendSlashMiddleware(object):
def _resolves(url):
try:
# If the URL does not resolve, the function raises a Resolver404 exception (a subclass of Http404)
match = resolve(url)
resolve(url)
# this will ALWAYS be resolved by expopages because it will produce pagenotfound if not the thing asked for
# so handle this in expopages, not in middleware
return True

24
core/models/caves.py
View File

@ -1,29 +1,16 @@
import datetime
import json
import operator
import os
import re
import string
import subprocess
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urljoin
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.files.storage import FileSystemStorage
from django.db import models
from django.db.models import Max, Min
from django.shortcuts import render
from django.template import Context, loader
from django.urls import reverse
from django.template import loader
import settings
from troggle.core.models.logbooks import QM
from troggle.core.models.survex import SurvexStation
from troggle.core.models.troggle import DataIssue, Expedition, Person, TroggleModel
from troggle.core.models.troggle import DataIssue, TroggleModel
from troggle.core.utils import TROG, writetrogglefile
# Use the TROG global object to cache the cave lookup list. No good for multi-user..
@ -149,11 +136,11 @@ class Cave(TroggleModel):
def get_absolute_url(self):
if self.kataster_number:
href = self.kataster_number
pass
elif self.unofficial_number:
href = self.unofficial_number
pass
else:
href = self.official_name.lower()
self.official_name.lower()
# return settings.URL_ROOT + '/cave/' + href + '/'
# return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) # WRONG. This produces /cave/161 and should be /1623/161
return Path(settings.URL_ROOT) / self.url # not good Django style.. NEEDS actual URL
@ -392,7 +379,6 @@ class Entrance(TroggleModel):
def cavelist(self):
rs = []
res = ""
for e in CaveAndEntrance.objects.filter(entrance=self):
if e.cave:
rs.append(e.cave)

19
core/models/logbooks.py
View File

@ -1,28 +1,11 @@
import datetime
import json
import operator
import os
import re
import string
import subprocess
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urljoin
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.files.storage import FileSystemStorage
from django.db import models
from django.db.models import Max, Min
from django.shortcuts import render
from django.template import Context, loader
from django.urls import reverse
import settings
from troggle.core.models.survex import SurvexStation
from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition, TroggleModel
from troggle.core.models.troggle import Expedition, TroggleModel
"""The model declarations LogBookEntry, PersonLogEntry, QM
"""

6
core/models/survex.py
View File

@ -1,17 +1,11 @@
import datetime
import json
import operator
import os
import re
from functools import reduce
from pathlib import Path
from urllib.parse import urljoin
from django.conf import settings
from django.db import models
from django.urls import reverse
from troggle.core.models.wallets import Wallet
# from troggle.core.models.troggle import DataIssue # circular import. Hmm

14
core/models/troggle.py
View File

@ -1,26 +1,12 @@
import datetime
import os
import re
import resource
import string
from decimal import Decimal, getcontext
from subprocess import call
from urllib.parse import urljoin
getcontext().prec = 2 # use 2 significant figures for decimal calculations
from django.conf import settings
from django.contrib import admin
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.files.storage import FileSystemStorage
from django.db import models
from django.template import Context, loader
from django.urls import reverse
import settings
import troggle.core.models.survex
from troggle.core.utils import get_process_memory
"""This file declares TroggleModel which inherits from django.db.models.Model
All TroggleModel and models.Model subclasses inherit persistence in the django relational database. This is known as

1
core/models/wallets.py
View File

@ -1,7 +1,6 @@
import datetime
import json
import operator
import os
import re
from functools import reduce
from pathlib import Path

20
core/utils.py
View File

@ -1,24 +1,12 @@
import datetime
import logging
import os
import random
import re
import resource
import string
import subprocess
from decimal import Decimal, getcontext
from decimal import getcontext
from pathlib import Path
from urllib.parse import urljoin
getcontext().prec = 2 # use 2 significant figures for decimal calculations
from django.conf import settings
from django.contrib import admin
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.template import Context, loader
from django.urls import reverse
import settings
@ -156,7 +144,7 @@ def write_and_commit(files, message):
)
else:
print(f"No change {filepah}")
cp_commit = subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True)
subprocess.run([git, "commit", "-m", message], cwd=cwd, capture_output=True, text=True)
cp_status = subprocess.run([git, "status"], cwd=cwd, capture_output=True, text=True)
# This produces return code = 1 if it commits OK, but when the repo still needs to be pushed to origin/expoweb
if cp_status.stdout.split("\n")[-2] != "nothing to commit, working tree clean":
@ -265,9 +253,9 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
print(f" !! - lookupAttribs:{lookupAttribs}\n !! - nonLookupAttribs:{nonLookupAttribs}")
raise
try:
msg = str(instance)
str(instance)
except:
msg = f"FAULT getting __str__ for instance with lookupattribs: {lookupAttribs}:"
pass
if created:
logging.info(str(instance) + " was just added to the database for the first time. \n")

1
core/views/auth.py
View File

@ -1,4 +1,3 @@
from builtins import str
from django.conf import settings
from django.contrib.auth import authenticate

19
core/views/caves.py
View File

@ -1,24 +1,18 @@
import os
import re
import string
import subprocess
import urllib.parse
from pathlib import Path
from django import forms
from django.conf import settings
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.http import HttpResponse, HttpResponseNotFound, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
from django.urls import NoReverseMatch, reverse
from django.shortcuts import render
from django.urls import NoReverseMatch
import settings
import troggle.settings as settings
from troggle.core.forms import CaveAndEntranceFormSet, CaveForm, EntranceForm, EntranceLetterForm
from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance, EntranceSlug, GetCaveLookup, SurvexStation
from troggle.core.models.caves import Cave, CaveAndEntrance, Entrance, EntranceSlug, GetCaveLookup
from troggle.core.models.logbooks import CaveSlug, QM
from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.utils import write_and_commit, writetrogglefile
from troggle.core.utils import write_and_commit
from troggle.core.views import expo
from .auth import login_required_if_public
@ -107,13 +101,13 @@ def getnotablecaves():
caves = Cave.objects.all().filter(kataster_number=kataster_number)
for c in caves:
# print(c.kataster_number, c.slug())
if c.slug() != None:
if c.slug() is not None:
notablecaves.append(c)
return notablecaves
def caveindex(request):
caves = Cave.objects.all()
Cave.objects.all()
caves1623 = list(Cave.objects.filter(area__short_name="1623"))
caves1626 = list(Cave.objects.filter(area__short_name="1626"))
caves1623.sort(key=caveKey)
@ -201,7 +195,6 @@ def file3d(request, cave, cave_id):
survexpath = Path(settings.SURVEX_DATA, cave.survex_file)
threedname = Path(survexname).with_suffix(".3d") # removes .svx, replaces with .3d
threedpath = Path(settings.SURVEX_DATA, threedname)
threedcachedir = Path(settings.SURVEX_DATA)
# These if statements need refactoring more cleanly
if cave.survex_file:

5
core/views/drawings.py
View File

@ -1,9 +1,4 @@
import os
import re
import stat
from pathlib import Path
from urllib.parse import unquote as urlunquote
from urllib.parse import urljoin
from django.conf import settings
from django.http import HttpResponse

9
core/views/editor_helpers.py
View File

@ -1,12 +1,11 @@
import io
import re
from pathlib import Path
import django.forms as forms
from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse
from django.shortcuts import redirect, render
from django.template import Context, loader
from django.urls import resolve, reverse
from django.http import JsonResponse
from django.shortcuts import render
from django.template import loader
from django.urls import reverse
from django.views.decorators.csrf import ensure_csrf_cookie
from PIL import Image

7
core/views/expo.py
View File

@ -4,14 +4,11 @@ from pathlib import Path
from sys import getfilesystemencoding as sys_getfilesystemencoding
from urllib.parse import unquote as urlunquote
from urllib.parse import urljoin
from urllib.request import urlopen
import django.forms as forms
from django.contrib import admin
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import redirect, render
from django.template import Context, loader
from django.urls import resolve, reverse
from django.urls import reverse
from django.views.decorators.csrf import ensure_csrf_cookie
import troggle.core.views.caves

19
core/views/logbooks.py
View File

@ -1,28 +1,15 @@
import datetime
import os.path
import re
import time
import django.db.models
from django.db.models import Max, Min
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.template import Context, loader
from django.template.defaultfilters import slugify
from django.urls import reverse
from django.utils import timezone
from django.views.generic.list import ListView
import troggle.settings as settings
from troggle.core.models.logbooks import LogbookEntry, PersonLogEntry
from troggle.core.models.survex import SurvexBlock
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.models.troggle import Expedition, Person
from troggle.core.models.wallets import Wallet
from troggle.core.utils import TROG
from troggle.parsers.imports import import_logbook
from troggle.parsers.people import GetPersonExpeditionNameLookup
from .auth import login_required_if_public
"""These views are for logbook items when they appear in an 'expedition' page
and for persons: their individual pages and their perseonexpedition pages.
@ -78,7 +65,7 @@ def expedition(request, expeditionname):
expo = Expedition.objects.get(year=int(expeditionname))
except:
message = (
f"Expedition not found - database apparently empty, you probably need to do a full re-import of all data."
"Expedition not found - database apparently empty, you probably need to do a full re-import of all data."
)
return render(request, "errors/generic.html", {"message": message})
@ -92,7 +79,7 @@ def expedition(request, expeditionname):
ts = TROG["pagecache"]["expedition"] # not much use unless single user!
if settings.CACHEDPAGES:
nexpos = len(TROG["pagecache"]["expedition"])
len(TROG["pagecache"]["expedition"])
# print(f'! - expo {expeditionname} CACHEDPAGES {nexpos} expo pages in cache.')
if expeditionname in ts:
# print('! - expo {expeditionanme} using cached page')

15
core/views/other.py
View File

@ -1,23 +1,16 @@
import os
import re
import subprocess
from pathlib import Path
from django import forms
from django.conf import settings
from django.core.files.storage import FileSystemStorage, default_storage
from django.db.models import Q
from django.http import HttpResponse, HttpResponseRedirect
from django.http import HttpResponse
from django.shortcuts import render
from django.template import Context, loader
from django.urls import reverse
from django.template import loader
from troggle.core.models.caves import Cave
from troggle.core.models.logbooks import QM, LogbookEntry # , PersonLogEntry
from troggle.core.models.survex import DrawingFile
from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.models.troggle import Expedition
from troggle.parsers.imports import (
import_caves,
import_drawingsfiles,

8
core/views/prospect.py
View File

@ -1,15 +1,11 @@
import os
import re
import string
import urllib.parse
from django.http import HttpResponse
from django.shortcuts import render
import troggle.settings as settings
from troggle.core.models.caves import Area, Cave, Entrance, SurvexStation
from troggle.core.models.caves import Area, Cave, Entrance
from troggle.core.views.caves import caveKey
from troggle.parsers.locations import MapLocations
# from pathlib import Path
@ -65,7 +61,7 @@ def prospecting(request):
has been disabled.
"""
message = (
f"This prospecting guide text report contains many broken URLs because of a major redesign\n"
"This prospecting guide text report contains many broken URLs because of a major redesign\n"
+ " to put caves into 1623/ and 1624/ folders in 2017. It was mostly useless because recent QM info was not in it anyway.\n\n"
+ 'It is disabled in the python code in "prospecting(request):" in troggle/core/views/prospect.py'
)

21
core/views/scans.py
View File

@ -1,19 +1,14 @@
import datetime
import os
import re
import stat
from pathlib import Path
from urllib.parse import unquote as urlunquote
from urllib.parse import urljoin
from urllib.request import urlopen
from django.conf import settings
from django.db import transaction
from django.http import HttpResponse
from django.shortcuts import render
from troggle.core.models.caves import GetCaveLookup
from troggle.core.models.survex import SingleScan, SurvexBlock, Wallet
from troggle.core.models.survex import SingleScan, SurvexBlock
from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import DataIssue, Expedition, Person
from troggle.core.views.expo import getmimetype
@ -158,7 +153,7 @@ def walletslistperson(request, first_name, last_name):
fixsurvextick(w, w.ticks)
return manywallets
print(f"-walletslistperson")
print("-walletslistperson")
try:
if last_name:
@ -176,7 +171,7 @@ def walletslistperson(request, first_name, last_name):
manywallets = tickspersonwallet(p)
expeditions = Expedition.objects.all()
print(f"--")
print("--")
return render(
request,
"personwallets.html",
@ -209,7 +204,7 @@ def walletslistyear(request, year):
return manywallets
print(f"-walletslistyear")
print("-walletslistyear")
if year < 1976 or year > 2050:
return render(request, "errors/generic.html", {"message": "Year out of range. Must be between 1976 and 2050"})
@ -219,7 +214,7 @@ def walletslistyear(request, year):
manywallets = ticksyearwallet(year)
expeditions = Expedition.objects.all()
expedition = expeditions.filter(year=year)
print(f"--")
print("--")
return render(
request,
"yearwallets.html",
@ -235,7 +230,7 @@ def walletslistyear(request, year):
def cavewallets(request, caveid):
"""Returns all the wallets for just one cave"""
print(f"-cavewalletsl")
print("-cavewalletsl")
Gcavelookup = GetCaveLookup()
if caveid in Gcavelookup:
@ -270,7 +265,7 @@ def cavewallets(request, caveid):
w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc
fixsurvextick(w, w.ticks)
expeditions = Expedition.objects.all()
print(f"--")
print("--")
return render(
request,
"cavewallets.html",

11
core/views/statistics.py
View File

@ -1,21 +1,12 @@
import datetime
import operator
import os.path
import re
from collections import OrderedDict
from pathlib import Path
import django.db.models
from django.db.models import Max, Min
from django.shortcuts import render
from django.template import Context, loader
from django.template.defaultfilters import slugify
from django.utils import timezone
import troggle.settings as settings
from troggle.core.models.caves import Cave, Entrance
from troggle.core.models.logbooks import LogbookEntry
from troggle.core.models.survex import SurvexBlock, SurvexStation
from troggle.core.models.survex import SurvexStation
from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition
from troggle.parsers.people import GetPersonExpeditionNameLookup, foreign_friends

18
core/views/survex.py
View File

@ -7,19 +7,13 @@ from pathlib import Path
from django import forms
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.http import HttpResponse
from django.shortcuts import render
from django.template.context_processors import csrf
from django.views.decorators.csrf import ensure_csrf_cookie
import parsers.survex
import troggle.settings as settings
from troggle.core.models.caves import Cave
from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.utils import WriteAndCommitError, only_commit
from troggle.parsers.people import GetPersonExpeditionNameLookup
from troggle.core.utils import only_commit
"""Everything that views survexfiles
but also displays data on a cave or caves when there is ambiguity
@ -165,8 +159,8 @@ class SvxForm(forms.Form):
)
# javascript seems to insert CRLF on WSL1 whatever you say. So fix that:
res = fout.write(rcode.replace("\r", ""))
res = fout.write("\n")
fout.write(rcode.replace("\r", ""))
fout.write("\n")
fout.close()
if socket.gethostname() == "expo":
@ -323,7 +317,7 @@ def process(survex_file):
def threed(request, survex_file):
filepath3d = survexdatasetpath / str(survex_file + ".3d")
filepathlog = survexdatasetpath / str(survex_file + ".log")
survexdatasetpath / str(survex_file + ".log")
if filepath3d.is_file():
threed = open(filepath3d, "rb")
return HttpResponse(threed, content_type="application/x-aven")
@ -447,7 +441,7 @@ def survexcaveslist(request):
if os.path.isdir(gcavedir) and cavedir[0] != ".":
subdirs, subsvx = identifycavedircontents(gcavedir)
caveid = check_cave_registered(
check_cave_registered(
area, cavedir
) # should do this only once per database load or it will be slow
survdirobj = []

45
core/views/uploads.py
View File

@ -1,45 +1,30 @@
import datetime
import json
import operator
import os
import re
import socket
import subprocess
import urllib
from functools import reduce
from pathlib import Path
from urllib.parse import unquote
from django import forms
from django.conf import settings
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
from django.core.files.storage import FileSystemStorage, default_storage
from django.db.models import Q
from django.http import HttpResponse, HttpResponseRedirect
from django.core.files.storage import FileSystemStorage
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.template import Context, loader
from django.urls import reverse
import settings
from troggle.core.models.caves import Cave
from troggle.core.models.logbooks import QM, LogbookEntry # , PersonLogEntry
from troggle.core.models.survex import DrawingFile, SurvexBlock, SurvexFile, SurvexPersonRole, Wallet
from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
from troggle.core.models.survex import DrawingFile, SurvexBlock, SurvexFile, SurvexPersonRole
from troggle.core.models.wallets import Wallet
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition
from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.views.caves import getCave
from troggle.core.views.scans import caveifywallet, oldwallet
# from troggle import settings
from troggle.parsers.imports import (
import_caves,
import_drawingsfiles,
import_logbooks,
import_people,
import_QMs,
import_survex,
import_surveyscans,
)
from troggle.parsers.scans import contentsjson
from .auth import login_required_if_public
@ -194,13 +179,13 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
else:
try:
sxpath = str(Path(sx).with_suffix(""))
svxfile = SurvexFile.objects.get(path=sxpath)
SurvexFile.objects.get(path=sxpath)
except MultipleObjectsReturned:
# can happen if connecting a wallet to a survex file.. i think..
QSsvxfiles = SurvexFile.objects.filter(path=sxpath)
for s in QSsvxfiles:
print(s.path, s.cave, s.survexdirectory)
svxfile = QSsvxfiles[0] # dont' know how this happened, fix later..
# QSsvxfiles[0] # dont' know how this happened, fix later..
except:
file_complaint = (
f"{wallet} Survex file {sx} exists, but is not registered in the database {sxpath}. How?.."
@ -286,7 +271,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
caveobject = None
else:
complaints.append(
f'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"'
'No cave ID is given. If there is no survex file, please give something, even if it is just "1623-000", "surface survey" or "scraps found in hut"'
)
caveobject = None
@ -323,7 +308,6 @@ def scanupload(request, path=None):
Should sanitise to ensure no spurious backslashes e.g. in windows style paths"""
waldata = {}
if contents_path.is_file():
create = False # yes wallet exists because JSON exists, even if no files in the surveyscans folder, or even if that folder does not exist
with open(contents_path) as json_file:
try:
waldata = json.load(json_file)
@ -333,7 +317,7 @@ def scanupload(request, path=None):
DataIssue.objects.create(parser="scans", message=message, url=wurl) # set URL to this wallet folder
raise
else: # no JSON file exists
print(f"--- No JSON exists, so creating blank copy")
print("--- No JSON exists, so creating blank copy")
waldata = WALLET_BLANK_JSON.copy()
if not waldata["survex file"]:
try:
@ -347,10 +331,9 @@ def scanupload(request, path=None):
return waldata
def save_json(jsondict):
newfolder = contents_path.parent
# print(f'--- Wallet directory in :drawings: repo {newfolder=} {jsondict}')
if not os.path.exists(contents_path.parent):
print(f"--- No wallet directory in :drawings: repo, so creating it")
print("--- No wallet directory in :drawings: repo, so creating it")
os.makedirs(contents_path.parent)
with open(contents_path, "w") as jfile:
@ -521,7 +504,7 @@ def scanupload(request, path=None):
commit_json(wd)
else:
print(f"--- INVALID JSON Update form submitted")
print("--- INVALID JSON Update form submitted")
print(formj.errors)
return render(request, "errors/generic.html", {"message": formj.errors})
@ -671,7 +654,7 @@ def scanupload(request, path=None):
# else:
# print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}')
except:
message = f"Specified survex file not found - database may be empty, or this survex file is not *included anywhere."
message = "Specified survex file not found - database may be empty, or this survex file is not *included anywhere."
# return render(request, 'errors/generic.html', {'message': message})
pass
@ -704,7 +687,7 @@ def scanupload(request, path=None):
print(f" - Setting wallet name to {names[0]}")
elif len(names) == 0:
waldata["name"] = ""
print(f" - Setting wallet name blank")
print(" - Setting wallet name blank")
else:
waldata["name"] = f"several, please edit: {names}"
print(

3
parsers/locations.py
View File

@ -11,7 +11,8 @@ from pathlib import Path
import troggle.settings as settings
from troggle.core.models.caves import Cave, Entrance
from troggle.core.models.logbooks import QM
from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation, Wallet
from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation
from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.utils import chaosmonkey, get_process_memory
from troggle.parsers.logbooks import GetCaveLookup

3
parsers/survex.py
View File

@ -11,7 +11,8 @@ from pathlib import Path
import troggle.settings as settings
from troggle.core.models.caves import Cave, Entrance
from troggle.core.models.logbooks import QM
from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation, Wallet
from troggle.core.models.survex import SurvexBlock, SurvexDirectory, SurvexFile, SurvexPersonRole, SurvexStation
from troggle.core.models.wallets import Wallet
from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.utils import chaosmonkey, get_process_memory
from troggle.parsers.logbooks import GetCaveLookup

6
requirements.txt
View File

@ -1,14 +1,20 @@
asgiref==3.5.2
attrs==22.2.0
black==22.12.0
click==8.1.3
colorama==0.4.6
coverage==6.5.0
Django==3.2
docutils==0.19
interrogate==1.5.0
isort==5.11.4
mypy-extensions==0.4.3
pathspec==0.10.3
Pillow==9.3.0
platformdirs==2.6.2
py==1.11.0
pytz==2022.6
ruff==0.0.226
sqlparse==0.4.3
tabulate==0.9.0
toml==0.10.2