From e94a24bbd4c58748e44c2b0461d9a46acccdecd9 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Thu, 2 Feb 2023 21:50:40 +0000 Subject: [PATCH] to-do lists updates --- core/views/caves.py | 11 +++++++++-- core/views/other.py | 18 ++++++------------ core/views/uploads.py | 3 +-- core/views/wallets_edit.py | 6 +++++- parsers/caves.py | 12 ++++++++---- parsers/drawings.py | 9 +++++---- parsers/logbooks.py | 9 +++------ parsers/survex.py | 17 +++++++++++------ urls.py | 14 +++++++++----- 9 files changed, 57 insertions(+), 42 deletions(-) diff --git a/core/views/caves.py b/core/views/caves.py index c970f52..fa5e7ef 100644 --- a/core/views/caves.py +++ b/core/views/caves.py @@ -21,8 +21,15 @@ from .auth import login_required_if_public Manages the use of cavern to parse survex files to produce 3d and pos files """ -todo = """- Fix rendercave() so that CaveView works -- in getCaves() search GCavelookup first, which should raise a MultpleObjectsReturned exception if no duplicates +todo = """ +- Fix rendercave() so that CaveView works + +- in getCaves() search GCavelookup first, which should raise a MultpleObjectsReturned + exception if no duplicates + +- Learn to use Django .select_related() and .prefetch_related() to speed things up + especially on the big report pages + https://zerotobyte.com/how-to-use-django-select-related-and-prefetch-related/ """ diff --git a/core/views/other.py b/core/views/other.py index df347ea..8d40079 100644 --- a/core/views/other.py +++ b/core/views/other.py @@ -28,12 +28,8 @@ progress and task list (deprecated as we do not have individual user login). """ todo = """ - -- Use logbookdownloader to convert all older logbooks into the 2005-variant of HTML then we can - get rid of the parsers for older formats. - When we have done all the old logbooks, delete this function and the two templates. - - OR invent a new format, e.g. using
and
?, which is better! +- [Low priority] Fix Login page so that it produces the frontpage or + redirects to the page which produced the login prompt requirement. """ @@ -175,18 +171,16 @@ def controlpanel(request): def exportlogbook(request, year=None, extension=None): """Constructs, from the database, a complete HTML formatted logbook - for the current year. Formats available are HTML2005 (others old & broken or not written yet) + for the current year. Formats available are HTML2005. Other formats + have been retired. - There are no images stored in the database, so this is only a tool for a first pass, to be followed by - hand-editing. However links to images work in the HTML text of a logbook entry - - NEED TO ADD IN THE MATERIAL WHICH IS NOT IN ANY LBE ! e.g. front matter. + There are no images stored in the database. However links to images work in the HTML text of a logbook entry. This function is the recipient of the POST action os the export form in the control panel """ def lbeKey(lbe): - """This function goes into a lexicogrpahic sort function""" + """This function goes into a lexicographic sort function""" return str(lbe.date) if not request.method == "POST": diff --git a/core/views/uploads.py b/core/views/uploads.py index 21a6844..d6aaf8d 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -20,7 +20,7 @@ from .auth import login_required_if_public """ todo = """ -- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import +- Register freshly uploaded filenames in the Django db without needing to wait for a reset & bulk file import - Need to validate uploaded file as being a valid file type, not a dubious script or hack @@ -33,7 +33,6 @@ todo = """ https://stackoverflow.com/questions/889333/how-to-check-if-a-file-is-a-valid-image-file - Enable folder creation in dwguploads or as a separate form - """ class FilesForm(forms.Form): # not a model-form, just a form-form diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py index c601667..6083dab 100644 --- a/core/views/wallets_edit.py +++ b/core/views/wallets_edit.py @@ -31,9 +31,13 @@ from troggle.parsers.scans import contentsjson """ todo = """ +- Nasty bug in navingating to 'previous wallet' when we have a 2-year gap in expos + The xxxx#00 wallet is not getting edited correctly. Something is off by one somewhere.. + - Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import -- Refactor walletedit() as it contains all the wallets 'complaints' code from the pre-2022 script wallets.py +- Refactor walletedit() as it contains all the wallets 'complaints' code from the pre-2022 + script 'wallets.py' - Need to validate uploaded file as being a valid image file, not a dubious script or hack """ diff --git a/parsers/caves.py b/parsers/caves.py index b2a6e54..8e967b1 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -19,12 +19,16 @@ So is the first thing that creates tables. """ todo = """ - - Cannot use Edit This Page for pendingcaves.txt_edit as Edit This Page is expecting an html file. +- Cannot use Edit This Page for pendingcaves.txt_edit as Edit This Page is expecting an html file. So we will need a separate file-editing capability just for this configuration file ?! -- crashes on MariaDB in databasereset.py on server when deleting Caves and complains Area needs a non null parent, But this is not true. - The only solution we have found is to let it crash, then stop and restart MariaDB (requires a logon able to sudo) - and then restart the databasereset.py again. (status as of July 2022) +- Semi-automagically import all the 1627- pending caves and create HTML files for them to be + edited individually. (These are caves we only know about because we have German survex files.) + +- crashes on MariaDB in databasereset.py on server when deleting Caves and complains Area needs a + non null parent, But this is not true. The only solution we have found is to let it crash, then + stop and restart MariaDB (requires a logon able to sudo) and then restart the databasereset.py + again. (status as of July 2022) """ entrances_xslug = {} caves_xslug = {} diff --git a/parsers/drawings.py b/parsers/drawings.py index 0273210..a814ce2 100644 --- a/parsers/drawings.py +++ b/parsers/drawings.py @@ -12,14 +12,15 @@ from troggle.core.models.wallets import Wallet for tunnel and therion files """ -todo = """- Rename functions more consistently between tunnel and therion variants +todo = """ +- Rename functions more consistently between tunnel and therion variants -- Recode to use pathlib instead of whacky resetting of loop variable inside loop -to scan sub-folders. +- Refactor to use pathlib instead of whacky resetting of loop variable inside loop + to scan sub-folders. - Recode rx_valid_ext to use pathlib suffix() function -- Recode load_drawings_files() to use a list of suffices not huge if-else monstrosity +- Recode load_drawings_files() to use a list of suffices - not the huge if-else monstrosity """ rx_valid_ext = re.compile(r"(?i)\.(?:png|jpg|pdf|jpeg|gif|txt)$") diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 1ae7fb9..6afd620 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -26,24 +26,21 @@ todo = """ - this is a slow and uncertain function too: cave = getCaveByReference(caveRef) -- pre-compile all the heavily used regular expressions ! - - profile the code to find bad repetitive things, of which there are many. - attach or link a DataIssue to an individual expo (logbook) so that it can be found and deleted - replace explicit 1970 date with a constant EPOCH -- rewrite to use generators rather than storing everything intermediate in lists - to reduce memory impact. +- rewrite to use generators rather than storing everything intermediate in lists - to + reduce memory impact [low priority] - We should ensure logbook.html is utf-8 and stop this crap: file_in = open(logbookfile,'rb') txt = file_in.read().decode("latin1") - - use Fixtures https://docs.djangoproject.com/en/4.1/ref/django-admin/#django-admin-loaddata to cache -data for old logbooks? Not worth it.. - + data for old logbooks? Not worth it.. """ MAX_LOGBOOK_ENTRY_TITLE_LENGTH = 200 BLOG_PARSER_SETTINGS = { # no default, must be explicit diff --git a/parsers/survex.py b/parsers/survex.py index 8bde946..91ce00a 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -24,17 +24,22 @@ It also scans the Loser repo for all the svx files, which it loads individually todo = """ - Lots to do to cut down on unnecessary .save() calls to avoid hitting the db so much. SHould -speed it up noticably. + speed it up noticably. + +- Obscure bug in the *team inheritance and rootblock initialization needs tracking down + +- Learn to use Django .select_related() and .prefetch_related() to speed things up + https://zerotobyte.com/how-to-use-django-select-related-and-prefetch-related/ - LoadSurvexFile() Creates a new current survexfile and valid .survexdirectory The survexblock passed-in is not necessarily the parent. FIX THIS. - When Olly implements LEG in the 'dump3d --legs' utility, then we can use that to get the length of -all the legs in a survex block instead of adding them up oursleves. Which means that we can -ignore all the Units and offset stuff, that troggle will work with survex files with backsights, -repeated readings from distox etc.. Not actually useful for pre 2022 survey data, but good future-proofing. -Also it will be a tiny bit more accurate as these leg lengths are after loop closure fixup. - + all the legs in a survex block instead of adding them up oursleves. Which means that we can + ignore all the Units and offset stuff, that troggle will work with survex files with backsights, + repeated readings from distox etc.. Not actually useful for pre 2022 survey data, + but good future-proofing. + Also it will be a tiny bit more accurate as these leg lengths are after loop closure fixup. """ survexblockroot = None survexomitsroot = None diff --git a/urls.py b/urls.py index c01ab8d..5c29d9f 100644 --- a/urls.py +++ b/urls.py @@ -38,11 +38,15 @@ which is vital to writing code for the webapp. So the URL dispatch is declarativ The API urls return TSV or JSON and are new in July 2020. """ -todo = '''Replace most re_path() with modern and simpler path(). -The admin and logout paths need to stay using re_path() as they -have to be locked to the start. -The final _edit and CATCHALL also have to use re_path(). -Test VERY CAREFULLY for each change. It is fragile. +todo = ''' +- Replace more re_path() with modern and simpler path(). Careful: some have to stay as re_path() + +- The admin and logout paths need to stay using re_path() as they + have to be locked to the start. + +- The final _edit and CATCHALL also have to use re_path(). + +- Test VERY CAREFULLY for each change. It is fragile. ''' # Many of these patterns do not work because troggle spent many years broken and we have