From 9435752f118f098d0c8cfb7fb23d4e17f8bdd486 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Thu, 18 Sep 2025 18:01:41 +0300 Subject: [PATCH] now link to Google Keep exported to-do items --- handbook/troggle/keep_export.html | 254 +++++++++++++++++++++++++ handbook/troggle/process_keep.py | 302 ++++++++++++++++++++++++++++++ 2 files changed, 556 insertions(+) create mode 100644 handbook/troggle/keep_export.html create mode 100644 handbook/troggle/process_keep.py diff --git a/handbook/troggle/keep_export.html b/handbook/troggle/keep_export.html new file mode 100644 index 000000000..d444cfa9e --- /dev/null +++ b/handbook/troggle/keep_export.html @@ -0,0 +1,254 @@ + + + + + + + + Google Keep Export + + + + +

Troggle coding and design to-do lists

+

(Exported from Philip Sargent's Google Keep notes on 2025-09-18 17:56) +

Contact Philip if you wish to share these lists on your Google Keep account. +

+ +
+

Expo Waddington Archive - Winter 2025-26

+
  • Scan all the xeroxes of the survey books 1990-2001 and put paper in recycling. CHECK them all first.
  • Give all non-expo-area Austria stuff to CUCC Library
+
+ +
+

EXPO data glitches

+
  • 346 survex files in :loser: have [ in them as of 21/4/2025. Change to unicode? instead.. (+ the template of course) +grep -nirc --exclude="*.pyc" --exclude="*.3d" "\[" *| grep -v "0$"| grep "svx"| wc
  • fix LoadSurvexLeg() UNSET 'survexfile' variable, setting to caves-1623/76/rift in https://expo.survex.com/survexdebug
  • 2017 files in #57 should be in #59 and need new wallet #57 to hold Elaine & Corin's 2012-DD-06 'Big D' - see index pages
  • 2012-ns-05
  • weird problems with 2015-dl-01, 02 and GMH#BUGGER bad cave '[]' on Entrance object 1623-2024-ND-01A
  • Why are cave ids not appearing in the wallets pages for some (new) caves e.g. 2023#70 not showing 1623-316
  • make .svx for 114 inc leg from GPS to entrance
  • Why is Puffball not parsing as a cave in old logbooks ?
+
+ 1 completed item +
  • Why does 1623.GMH not work as a tag station but 1623.gmh does, when the *fix is GMH ?!
+
+ +
+ +
+

EXPO folk and signup

+
  • Make year_box hyperlink to PersonExpedition on folk list page.
  • auto add year to folk.csv on 1st January .ie a terminal comma and "2026" in header.
  • report number of "years" for each person when running the folk script. I found someone with not enough commas... DataIssue
  • add-person-to current-expo, form for adding to folk list, also as a button on the Person's personal page (for old expoers) ? AND on the folklist page itself?
  • Redo folk.csv
  • regenerate the CSV file from the db, check that this is correct. Then we can have forms to do various things, and regenerate it
  • Need a new field for blog separate from the mugshot field, change the standalone script to work with this too.
  • make folk.htm as part of initial troggle install setup on a new machine
  • create folk.htm as part of databaseReset process instead of as separate script
  • Split the folk.csv into several files
+
+ 1 completed item +
  • sort folk by surname
+
+ +
+ +
+

QMs

+
  • fix long list of bad URLs generated which are meant to be to survex file
  • need more fixtures or factories to test QM pages
  • add ticked-off text to COMMENT field on the QM so that it appears in the reports.
  • Find URL from just the resolution station of the ticked QM
  • Find URL to survexfile from just the location station of the QM, make a link from the QM page
  • Review possible re-design of QM system for TICKing off.
  • QM TICK not unique number, needs to be block-specific
  • put link to tick https://expo.survex.com/handbook/survey/qmentry.html#tick in the QM reports !
  • fix bad error on QM TICK stuff. out of range crash on importing for ;QM22 TICK ...
  • QM tick duplicate? Just number and date, must also be survex lock specific
+
+ 30 completed items +
  • 1627 caves in https://expo.survex.com/caves have bogus links to .3d files, which do exist
  • Check 2018-dm-02 files. 2018-ad-02 is garlic cave
  • cave html file import. Serious bugs in do_entrances, error msgs quote variables not available.
  • Check in survexdir report that the svx file path is always an exact match for the survexdirectory field, in which case the survexdirectory field can be eliminated.
  • WHY are many Caves missing a primary survex file ? see http://localhost:8000/survexdir
  • I seem to have broken the table of the list of survex files which should appear eg in http://localhost:8000/survexfile/145 as there are no sd s attached to a cave and thus no way to get form a cave to a set of sds +REWITE svxcavessingle and svxcavesseveral to work properly again
  • Clean up survex parser removing Survex Directory stuff
  • replace IdentifyCave with lookup function in survex parser
  • class SurvexDirectory is not needed ? primarysurvexfile could just be a proprty of Cave ?? Or does a Cave have many SurvexDirectories? Maybe it does, as they are sub-directories
  • then do jss- and
  • how does 'arge' appear in the aliases list - because it is given as an "Unofficial number" !
  • FIx notable caves getting 1626 as well
  • check 138 works f0r both and separately
  • no wallets found for 1623-2017-21 or 22 even though these are now real caves. Just entrances. Not linked in to survey network. Emailed Becka re this.
  • ** Duplicate cave id count=2 id:'1624-190' cave __str__:'b9'
  • ! Already exists, caveid:1624-101 in area 1624 <QuerySet [<Cave: 1626-101>]>Refactor survexcaveslist(request) and fix check_cave_registered()
  • bad link in AllSurvex to http://localhost:8000/survexfile/langgustl
  • Not discovering 1626/107 as a cave that needs to be put into pending list, despite there being a /1626/107.svx file +AND it gets misallocated to cave 1623-107 (which I guess is understandable as "107" is then an allowable short name)
  • Cave check 107 gets a SD for EACH of 1623, 1626 even when it is 1623-107 !!, but both these SDs have cave=1623-107 +True for 145 too
  • 1626-107 missing from all survex files in http://localhost:8000/survexdir
  • Remove all QM TICK stuff. Keep to former method.
  • survexfile/290 is what the link says, but it shoudl really be surveexfile/cave-1623/290/290.svx !
  • Add link to /survexfile/359 on both the QM reports
  • QM - new report formatted to be tick-list at top camp
  • Add link to /survexfile/359 on the survexfile editing page
  • EditCave page to get data from the 1623-115.html file and parse it, not from database.
  • New report on all SDs showing which Cave they link to and ALERT if the Area is different.
  • Remove all sub-area stuff. Just the 4 Areas. Where is this set ? 249 files in expoweb/cave_data/ have fields of the form: <area>4</area> which we could just ignore... No. keep as comment somehow.. grep "^<area>[^1<]" *
  • Remove Class Area! Just an enumeration of "1623", "1626" etc.
  • Remove all code from aborted TICK system
+
+ +
+ +
+

Troggle TESTS

+
  • Test forms directly, not via request, use https://adamj.eu/tech/2020/06/15/how-to-unit-test-a-django-form/
  • Can I use the overwrite_settings thinggy ? +https://docs.djangoproject.com/en/4.2/topics/testing/tools/#overriding-settings
  • Remove all get_url code ? There are 87 of these in templates. They are tested (mostly) in test suite. Test ALL of them.
  • New test which parses 1623-115.html file
  • New tests to properly exercise all combinations for register new user/old user, as anonymous, as expo, as other logged on user, as admin logged on user..
  • write lots of factories for test data as per https://gogognome.nl/how-to-write-tests-that-need-a-lot-of-data.html
  • /1623/76/76.htm which uses the .lookup() functions#Check there is a test for every entry in urls.py - a bit overkill ?
  • Test new entrances gpx stuff and check wookeys documn#need tests for HTML editor that Martin did using several weird template files
  • Try a Selenium test suite to check for Apache and cgit funnies or https://playwright.dev/python/
  • write tests for editexpopage, wallet edit, drawing upload and git commit
  • write tests for wallets tick lists BEFORE refactoring tick lists code
  • Need tests for PARSERS with dummy input files
  • folk parser
  • survex parser
  • cave & entrance parsers
+
+ +
+

Troggle Photos

+
  • map display showing pins at the GPS locations. Of any subset of photos
  • Dependency injection for settings for inserting settings for inserting photos into different sorts of documents. See Becka email May 2025
  • Index all photos by date, and list photos "on this day" at the bottom of Wallet, Logbook and Survex file pages.
  • Add "Photos on this date", go to placeholder page, on survexfile, wallet, logbookentry pages, views/photos.py
  • Photos, create PHOTOS/year/dateindex.json by resetPhotos.py using parsers/photos.py
  • Run "make" in databaseReset to check if any year of photos needs its dateindex refreshing, from timestamps on files.
  • Extract GPS from exif, put in dateindex.json in each /year/ folder in PHOTOS & GPS
  • Parse and collect Google Photos share for 2023 expo
  • loads of entrance photos in 2012 PHOTOS by filename, but with no GPS data
  • Replace placeholder page (which??) with thumbnails, and gps coords??
  • Given a location, find all photos within 200m. Package with geoproximity search? Or quick sql bounds on easting/northing followed by pythag dist on that subset.
  • Add this "photos within 100m" to each Entrance Description.
  • Sort srtm list by x, and a duplicate by y. Then implement patch search to get closest 100m square before doing exhaustive search.
  • new GIS function "distance" in django supported by MariaDB, sqllite so can use it... but how stable will this be?
  • Display a report page with thumbnails. Put on each CAVE page
  • put git comment field on survex editor, wallet editor, logbook editor.
  • SRTM
  • replace SRTM with 1m laser terrain model. How ?
  • Just DELETE all SRTM altitudes, set to zero, but have a URL to an SRTM-altitude calculator on every entrance?
  • Use camera+gpx to add exif to datestamped photos if we have the GPX +https://github.com/FrancoisSchnell/GPicSync/wiki/Getting-started
+
+ 4 completed items +
  • Put my photos geotagger into scripts folder and document in webpage
  • Use bard to make python version of gpx2survex (in OCAML)
  • Check version of libgdan on django 3.2 on laptop
  • run exif extractor on google expo photos
+
+ +
+ +
+

EXPO Logbooks

+
  • When editing logbook entry, if date is changed, then
  • Forbid year change.
  • change the ID to match.
  • Detect if a logbook has NO images or diagrams? Do a report on all expo logbooks.
  • More than one person as author
  • more than one cave on a logbook
  • add Martin's IMAGE thing to logbook editing.
  • make each entry a separate file.
  • Can we make logbook entry ids persistent in the face of editing and inserting? This should be a goal. Needs redesign.
  • some logbook entry crashes leave a stranded file not checked in to git. Detect this and fix it !
  • edit in diagrams for 2017, 2018, 2019, 2022, 2023, 2024
  • Check 2001 logbook all sanitary with new diagrams scanned
  • Gwyneth Martin was on expo but is not recorded. 1988?
  • Becka: looking for blank lines where someone was listed but did not participate is useful, and I will be making a report to that effect in the database reset parser to check
  • new-expo year functions in troggle,
  • Make 3 html pages into troggle pages to use current_year. Harder than it looks.
  • Parsing. Try Person& PersonExpedition all in python in parsers/people.py and then commit as two bulk transactions. test if links between them work when done like that.
  • New report page per cave listing all logbook entries that reference that cave (separate page as this would be rather long for 204.. )
  • de-pds.awk the old CUCC journal articles in /years/ 1982-84, then hand-edit from gcal to html
  • refactor using item 29 "compose classes instead of deeply nesting Dicts, Lists, Tuples"
+
+ 56 completed items +
  • Stop logbook editor saving entries with a different order of participants every single time !
  • Put name of lbe being edited above the submit button.
  • Take wookey's email 25 oct and make web page on cavelengths
  • Person with no attendances at expo is a bug e.g Alex Faunce (now deleted). Must be removed or it crashes the aliases system.
  • Filter out Emojis that people put in logbook entries as tehy crash MariaDB.. now fixed on teh server with a db setting (I hope)
  • Forbid changing year in new lbe
  • Make default date for new lbe to be yesterday, not just a placeholder
  • Bug: If you create a logbook entry with a date in the wrong year, you have to delete the whole thing and start again. +Otherwise you end up with a entry dated e.g. 7th July 2024, but it will have been stored in the *2023* logbook. Which cocks everything up mightily. STOP THIS HAPPENING. +Each year's worth of stuff gets stored in an entirely different place. Simply changing the date does not untangle the mess.
  • /logbookedit/bad crashes horribly and I can't stop it. Login problem fights redirect.
  • create logbook-blog fold-in instructions
  • add 2016 camping-in-tunnocks-elaine
  • MANUALLY CHECK 2022 logbook re missing bits of blog entries
  • Extract training weekend (blog) entires from logbook for 2019 and 2022 and relocate.
  • edit in Dour's KrakenWakes blog into 2016 logbook (already done Elaine's)
  • Edit in ash's logbokk entries
  • Process uk cavong blog 2023
  • Add/edit a logbook entry, destroy old entry
  • Example parsed edited lbe should correct image urls so that images appear on logbookedit page
  • Save/export logbook creates unique, proper tripids
  • DETECT duplicate id string in _original_ logbook.html ! Make dataissue and make new slug with md5 thing
  • New cp trigger to save db as new logbook, replace old and do git stuff
  • Parser import, preserve tripid and use as slug
  • Import parser preserves sequence of entries, as well as date? Used for export. But can't work for online entries as there is no sequence. Old logbooks have date sequence anyway due to re import.
  • Fix test for logbook entry page
  • Logbook, why does ttile pf lbe on expedition page not get updated when logbook is reloaded?
  • Logbook edit, check that *beetle not forgotten. create test. Oops it is.
  • Finish programming the foreign_friends code to cope with ARGE people without an initial *
  • Logbook edit, check that 'place' is not deleted (even if not recognised) . create test
  • Logbook edit is saving FULL NAMES not the nicknames. hmph. Chnage logbook2005html template to export nicknames
  • Logbook edit, check tripid does not increment on reload
  • Logbook edit is writing back who the author was, even when it had to guess when reading it. Hmph.
  • Logbook check that a cave is recognised from place text
  • Logbook edit : author
  • Manual: move detailed edit instructions to more obscure programming page
  • Create test for logbook form edit page
  • Overflow>26 use unique code for tripid
  • refactor logbook save.add/commit with upload drwg add/commit to use common utility that Martin wrote
  • Hide generated html source in form page
  • Fix tid to be 'date+[ab]' permanent reference not ephemeral in parser and exporter.
  • refactor troggle file save/git my and Martins do same thing
  • what is the *prefix for unknown attendee mean?
  • fix broken logfile reading for 3 bad years.
  • the reload from logbook button seems to be better at identifying all the trips, and the reload link puts things in the right place better, but I am still very amused their results are not the same...
  • add unknown names for 2022 logbook to German list
  • do round-trip 2019 logbook and hand-edit export to remove duplicates of blog entries
  • do round-trip 2018 UKCaving expo training weekend blog
  • check 6 pages 2018 callout book is in logbook
  • 2009 logbook parser misses first 300 lines. Fault in wiki parser.. Also no content for any entry into database
  • Delete logbooks parsing swapped title / people.
  • Finish check for expofiles links in logbooks, some of these are valid to orig large photos or to other documents.
  • Logbook import errors in 2007
  • 1976, 77, 78 logbooks ALL NEED reformatting !
  • Replace id strings with date-related stable ids, not just sequence numbers.
  • 1979 logbook NEEDs reformatting ! Get the old s/w and parse it ! On another laptop?
  • disable Edit This Page for logbook pages. - fix for remaining logbooks
  • logbook-editor laptop docum for Harry Kettle. maybe
+
+ +
+ +
+

EXPO Hardware and system software

+
  • handbook document running tests on the expo server.
  • document deptry for troggle https://github.com/fpgmaas/deptry
  • https://www.tweag.io/blog/2023-03-14-announcing-fawltydeps/
  • document profiling troggle e.g. https://kracekumar.com/post/profiling_django/
  • server has bad CSS for the admin pages. Maybe I uploaded Django4 css
  • document server new apache filter thing??
  • Garden all the .gitignore entries
  • https://github.com/daangemist/kanboard-import-trello
  • Try sqlite for server, e.g. on django-test machine
  • https://gcollazo.com/optimal-sqlite-settings-for-django/
  • https://sqldocs.org/sqlite-database/django-sqlite/
  • https://djangoandy.com/2024/07/08/enabling-wal-in-sqlite-in-django/?form=MG0AV3
  • https://minimalistdjango.com/databases/2023/06/18/sqlite?form=MG0AV3
  • In Cambridge
  • Fix azirophale SD slot
  • Fix Azirophale ssd
  • System
  • get working on MythicBeasts ubuntu server setup for EXPO on 8000
  • use puppet/Bolt to configure full expo installation
  • Install MariaDB and check more severe referential integrity checks work. Failed on upgraded WSL to 22.04, try on fresh install ?
+
+ 14 completed items +
  • move expofiles into WSL too to get speed.
  • Fix tests on server - Path not string in settings
  • Fix spare router password and write on bottom. Return to Wookey.
  • wr841N - why does ethernet connection still come up as CSGcourse on laptop via cable
  • put requirements.txt into venv script
  • Ask wookey re phrase sesrch on server
  • Try django 5.0 pre release, after server upgrade to 4.2
  • edit Handbook to say WSL1 unusable
  • handbook Add pip venv url https://pythonspeed.com/articles/externally-managed-environment-pep-668/
  • sign up azirophale to Matrix chatroom
  • venv script broken with pip3 and python 3.11 - crashes
  • test venv with py3.11 on ubuntu 22.04
  • Buy cuccexpo.eu and set up email redirect for logbook entries and photos ? Nah. Google photo group works fine.
  • Move all repos into \\wsl$ for speed and fix local backup to work
+
+ +
+ +
+

Troggle immediate

+
  • Essentials
  • rebuild essentials2025.gpx and check for Ghast locality on gps prune map
  • Find GhastHole 33T 411454 5283151 47.695674 N 13.819901 E from essentials. Check other caves from that logbook entry by Buck
  • Rebuild all.svx to include gpx from kataster boundaries: gpx/all\_gpx got removed in latest essentials rejig, which is (probably?) fine (why is it now gone?), but that broke all the top-level '1623-and-1626' type files. I've now commented-out all the usages so that all the files work. You (phil) fixed 2 of 6 but forgot the others.
  • Fix bug b entrance without a entrance, notging in cavelist 2025-LB-00
  • Bug https://expo.survex.com/enttags
  • Build a Keep export to HTML utility for all my troggle to-do lists. gkeepapi not working from WSL. Try a new App Password . Try on a "real" ubuntu system, e.g. okchai
  • Add django view design to troggle https://www.loopwerk.io/articles/2025/django-views/?utm_source=www.pythonweekly.com&utm_medium=newsletter&utm_campaign=python-weekly-issue-712-september-4-2025&_bhlid=6ece59f0c3de24278c50871556723844d8ab734a
  • Install skylos to find dead code https://github.com/duriantaco/skylos
  • Make new cave for things found on Buck's logbook entries.
  • Recreate Gschwandalm and other 2024 walks photos+ .gpx tracks for checking entrances
  • add gpslogs to 'on this day' list, as well as photos
  • Wookey gpxfix form to calc a *fix, re-do using Django magic form builder.
  • EXPO Finish expo cave entrance walks, photos, 93, 104 area.
  • This year only: links to surveyers on walletedit page
  • Make grey not pink for non-,tunnelled wallets before 2025
  • Detect and add to Complaints (,but blue) output if sexytopo or topodroid (frank)
  • EXPO: archive off all my 2024 photos, many already downloaded and on GoogleDrive, remove all expo photos from GooglePhotos
  • Does cookie timeout just do git editor setting or does it also cache login, e.g. as 'expo'?
  • get photo of Kai from 2023, dinner photos?
  • Create caves or prospects for all ents in https://expo.survex.com/logbookentry/2025-07-05/2025-07-05d#
  • do the set difference of SurvexStations and SurvexStations which are on entrances: these include the *fix and the programmatically created MapLocations - these are all potential entrances which should be in the undropped list
  • Do not require 1623 prefix for Entrance stations, insert automatically and remove if prrsent. We know which area it is.
+
+ 35 completed items +
  • un refreshed troggle_import_root.pos file from .3d file, when it should always be refreshed! DOn't need to do this for both survex and locations, modify.
  • create missing entrances for Lara file
  • check Lara's template file for new survex file.
  • create new cave 2025-PS-02 with *fix correctly
  • finish entrance visits of July 23
  • re edit and correct http://localhost:8000/handbook/survey/caveentry.html because of new cave addressing things
  • check through all plastic wallets,
  • check mine #30 in index sheet wallets
  • All QMs on all caves are marked as ticked because of a bad regex
  • on the walletform page, make live links to the persons' all_wallets page, so you can easily find the other wallets that someone has done.
  • EXPO copy and resize and archive GooglePhotos from 2025 expo
  • document do_essentials2025.sh
  • I did 2006-01, but check edits on all the other caves I did on Weds.
  • Recreate entrances2025.gpx essentials2025.gpx
  • fix total survey length report to exclude ARGE and in include "wild", un-walleted survex files
  • Check handbook instructions for buck's mistake, corrected in winlaptop, check for linuxlaptop
  • Cookie thing for Anathema/Crowley/Aziraphale shared computer: timeout 1 hour not 2 years. detect the cookie + "public_laptop" with the value "this is a public laptop",
  • re-set the COOKIE_MAX_AGE to be a function cookie_max_age() in all the views that use it.
  • Add link in /drawings page to https://expo.survex.com/repositories/drawings/.git/tree/ and a link in the expoweb/drawings directory
  • configure troggle config to save by machine name not by hardcoded os type
  • check Who/When against folk list
  • Tell harry where to put latex code
  • revise fig on EditThisPage
  • Get photo of Dan Gorst
  • Invalid 1623 areacode may still leave git in bad state. Protect more thoroughly.
  • Make podman container work. permissions problem creating database with databaseReset
  • CaveID on a wallet without a survex file is not indexed on parsing
  • invalid tag station: 2025-ash-01a
  • insert prefix 1626 in Edit Entrance form for tag and other stations if not there already.
  • look for missing ".svx" in survex file name specified in a cave html page.
  • produce a prospecting GPX with just the undescended caves.
  • https://brainbaking.com/post/2025/06/no-time-to-learn-web-framework-x/
  • https://judoscale.com/blog/which-python-framework-is-best
  • Handbook : podman documn.
  • https://expo.survex.com/accounts/register/philip-sargent allows anyone to re-register an already-registered person ! If already registered, abort this form. If logged in as "expo", then log-out automatically before proceeding.
+
+ +
+ +
+

Expo Survex parsing

+
  • Create 'upload new survex file' page
  • Change default survex file to wookey style of splays at the end and docm. of each survey point.
  • Make report of *all* survex blocks (not just by year) which do NOT have a valid *ref AND are by people who were on expo (no ARGE)
  • Look for lengths in feet (Phil Balister) and create data issue
  • Use memoryview & buffer & bytesIO to read from survex files, and shutil() to write. See https://pythonspeed.com/articles/bytesio-reduce-memory-usage/
  • Make link to wallet mentioned in *ref(s) on survex edit page
  • check *ref is correct at line 663 in parsers/survex.py
  • survexfile: Everything seems to use multiplefiles and nothing triggers singlefile?
  • use cavern to calc survey lengths, need to break up individual blocks though, as cavern only reports on the whole file.. Then can ditch a lot of the survex parser. Do in parallel to my code to check. Test on single file upload svx.
  • Why is ropelessconn appreaing as being in file caves-1623/107/0805 in the report http://localhost:8000/survexfilewild/2013 when it should be in 1623/107/107 ?
  • Finish cleaning up root survex block for unseens
  • refactor using item 29 "compose classes instead of deeply nesting Dicts, Lists, Tuples"
  • block essentials 2023-07-31 has parent block essentials (in file 11895 kataster/kataster-boundaries-lukas-plan-2018-07-17/kataster-boundaries.svx)
  • When we pop out of the *include for the kataster boundaries, we are not properly re-setting the current survex file to be the top level one
  • Why zero survey length an 'no dated blocks' message for http://localhost:8000/survexfile/caves-1623/2023-lc-01/2023-lc-01.svx
  • Survex, fix bad inherited *team ! *team 2018 'Pete Lancaster' FAIL personexpedition lookup on *team kataster/kataster-boundaries-lukas-plan-2018-07-17/kataster-boundaries (troggle_unseens) in 'Inst Pete Lancaster'
  • get(id=1) FAILS when only survex is parsed
  • troggle_unseens has me as a surveyor! How? Why??https://expo.survex.com/personexpedition/PhilipSargent/1982
  • why is troggle_unseens appearing in 1982 and is actually a kataster survex file? This is the same survexblockparent thing which confused me so much when trying to parse a single survex file.
  • redo survex parser to defer all team update until the *end point, then do bulk creation
  • should also collect all the .err files, line 1834 in survex parser runs cavern
  • Rrunning survex on isolated .svx files is often pointless. Can I detect when not to do this?
  • run profiler on the other bits of the parser
  • errors giving wrong URL to bad survex file in DataIssues ! Value Error: Clino misread in line ['00aa01', '47.68158', '13.80487', '0'] in caves-1623/115/115cufix http://localhost:8000/survexfile/caves-1623/115/115cufix.svx uncomment the gps00raw.svx to trigger this error, also bad Dates do this
+
+ 14 completed items +
  • Survex long report: why so many duplicate name - bad message
  • Do the same SameDate thing for Survex Files
  • Detect the previously-seen survex path BEFORE loading it and pushing the stack, not after. Refeuse to load it. see line 1721 in process_line(svxline) in PushdownStackScan(). Abort when we see it, not later. Or at least label the svxlinear.log line nicely.
  • Same survexfile is included multiple times. What is the correct way to deal with this?
  • Improve error message in DataIssues for the QM parsing failures
  • Length WRONG for https://expo.survex.com/survexfile/caves-1626/359/Austria_Homecoming_4th_pitch_tube.svx - see Frank Tully wallet summary: 1km not 147m
  • survex *set, check if any begin/end names are affected by this and report if not.
  • or, detect *ref directly in .svx file, without relying on parser: so that recent updates are seen immediately. Even on files which are not fully re-parsed.
  • Survex bad inherited team. Insert warning comment in the survex file.
  • *date not inherited into sub-blocks in caves-1623/290/sciencevsfun
  • re-parse an individual svx file? Yes, do this.
  • also process svx when edited and saved online.
  • What are all the foreign keys to a block that need fixing when a survex file is replaced?
  • is duplicate SurvexFile object created for 264/horizontalentrance - is this still a problem after I fixed to use update_or_create() ?
+
+ +
+ +
+

Caves & Ents

+
  • Document that cave ids must end on a digit.
  • Forbid new cave ids ending in a letter. Tricky as this is read from the filename.. if found, add "-01" ?
  • Disallow CAPITAL letters for entrance identifiers and filenames
  • Make a system to rotate images in Cave Descriptions after upload. Frank's are all sideways.
  • allow entrance *fix (tag) reference to not have a 1623. prefix, since it knows which area the cave is in and can therefore insert it.
  • New quick NewHole form that just does the NewCaveSheet stuff., but does create C+Ent, and also makes a *fix
  • Then delete the <entrance> lines on all cave_data files, but check that editing the <letter> works properly first..
  • Ents
  • Enforce lower case for entrance letters in filenames. Harder than it looks, tied up with the CaveAndENtrance object thing.
  • After an Entrance is created, if the letter is changed e.g. from null to 'b', (this is editable) then the .name is NOT changed to match. Fix this.
  • ents <letter></letter> field - not needed at all now? Use filename, and check that filename is cavename[+letter]
  • ents <letter></letter> field How do we manage user editing of this ? +Ensure it is never used, then ignore the field and remove from user interface ? But it is used because it is manually editable from the editentranceform.
  • write TESTS for entrance data and cave data links.
  • CaveAndEntrance redesign, quote this redesign - is needed (I tried) to connect cave to slug. https://docs.djangoproject.com/en/4.2/ref/models/relations/
  • When renaming an ent station using the form, re-do the setting in db so that the gps position appears immediately, not only after a db reset . Needs .pos regen and re read?
  • Update SRTM ALTITUDE data with fixed data from Radost: fix code and then correct 60 caves, or better, with lidar data.
  • Horrible hack
  • create TESTS for the variant cave content /1623/100 , /1623/100/100, /1623/100/100.html, /1623/l/entrance100.html /1623/100/100/entrance.jpeg (when it is really ine /1623/100/entrance.jpeg
  • Produce a plan for fixing all those hard-coded links in cave_data files 50,50,20,19 in caves/ents 1623/1626 so that the Horrible Hack can be removed. e.g. 110/thing.jpg becomes /1623/110/110/thing.jpg
  • Write a report listing the ones that need fixing.. horrible hack.
  • need a report on all *fix stations which are NOT assigned to an Entrance... 20,000 track points.. hmm.
  • Put missing *ref marker on survex blocks in manywallets table? And on caves in main caves/ list
  • *fix without survex file or station in *entrances file means we don't know which it is.
  • split CaveEdit form into two forms: one for Data and one for Description, but keep a single file for holding the data? And refresh the file from the current db data, not the data returned from the form.
  • fix non exist cave message when running tests. Of course they don't exist. Pre-create GCaveLookup in test setup
  • Ents page
  • Generate single html file of all entrances, unique id for each description etc. Urls for this are encoded in a json export of all ent locations
  • ARGE caves have their *fix locations in the fixedpts 1626 svx file. put the survey point in when the cave is reified.
  • Katastered caves? Rename all svx links inside json files instead of doing it manually.
  • Wookey on cave ids: "I'd add 1000 to that function so it's longer before it goes wrong in the future (we are on >318 already). Also is that really the right way to deal with things that aren't actually integers? "
+
+ 65 completed items +
  • FIX the problem that when creating a cave, the first entrance must not be edited.
  • Fix this: Editing creating a new survex file, Aidan said: +When I typed in the new URL, I just went on editing and then saved. What I should have done was type in the new URL, press enter to go to that URL and then edit.
  • Put name of survex file being edited above the Submit button. +
  • NOT ARGE, most of those coves and survex are VfHO.
  • New Cave form failing with uncreated Cave object
  • How and why the entrance data has not been consistently updated has now been sorted out: just needs doing.. +..OK 17 of the caves now have correct entrance tag locations
  • But there are 25 wallets with incorrect links to 2023 survex files which still need fixing.
  • WHy does 315 still have 2023-JSS-01 in the name when I do _edit_cave ?!
  • Fix the caves where the filename is not quite the same as the slug
  • Write DataIssue if ent filename != caveslug[+letter]
  • Stop reading the slug fields, get everything on an entrance from the filename. Look for the field for validation, but if it is not there, don't worry.
  • fix handbook to says fields not used.
  • why are cave.slug() values sometimes not set correctly, dfaulting to django db id.
  • Make cave aliases list a spearate text file like pendingcaves.txt
  • Write parser for cavealiases.txt file, and have an oldcavealiases.txt file too for the ancient stuff
  • Remove timestamps in cave and entrance files.
  • seveal zillion 'aliases ok' dataissues on server (seems to be from logbook editing)
  • Widen cave description edit fields so that they fill screen.
  • Bug // in EditThsPage url when editing pages which are part of cave descriptions, e.g. http://localhost:8000/1623/161/136desc.htm
  • Make Cave have a unique slug, and use it for everything. Not the separate CaveSlug object
  • remove all <caveslug> lines from cave_data files.
  • Stop reading the <entrance> tag and content on a cave html file, get what we need just from the filenames in entrance_data +Look for the fields for validation, but if it is not there, don't worry.
  • remove all <slug> lines from all entrance_data files.
  • Fix required letter for new entrance only if b or later..
  • REDIRECT /areacode/name and /areacode/name.html to that, and /name/index.html too.
  • Cave description image editor:put button at top, not bottom, and prefill most fields.
  • rewriting cave ids for variant virtual directory, filter out // in url before use.
  • Find and fix all the internal links to which fail now that all Description pages are 1623/110/110.html +Then get rid of the Horrible Hack
  • Uploading photos to new cave still puts them in /t,i,l not in /caveid/t,i,l
  • 2023-BuzzardHole done?
  • Make cave.url always be /areacode/name/name.html
  • Cave description image editor: save to cave folder, not generic shared folder. - does this already if cave url has a folder.
  • new style caves does not find all of them! Not getting some that are explicit on the wallet. Do get filled in when YEAR wallets is run, so do whatever that does, but on all of them on reset
  • amalgamate BS17 and 2012-OK-01 Organhohle
  • 5 hannah caves from 2022 to be created.
  • fix delicate bridge cave 2017
  • amalgamate 2012-ns-07 and 2016-01
  • Do the *fix tht Becka said was OK.
  • Inset a *fix for 1623-114 from my GPS and waypoints
  • Remove <area> stuff from cave subareas? Never used ? Make it NOT <area>. In all the .html cave_data +rename <area> as <subarea>
  • DOCUMENT the *fix process in handbook.
  • What is the *fix register keyword
  • Document *fix See Becka' long post on Matrix 26 Sept 2023
  • Ask/tell Martin re 2 routes to cave page
  • On loading survex files, if no survex file set on Cave, then set it AND make a DataIssue so it can be fixed
  • fix LA34 in 1623 and 1626
  • 2017-CUCC-21 and 2017-21 are the SAME cave, ditto 22
  • Detect entrances with no survey station, orange triangle
  • Detect caves with no survey stations on any entrances
  • Do not default to area 1623 for any date after 2017. Yes, do it for caves referenced in Wallets ONLY.
  • check slug capitalisation for ENTRANCES as well as caves
  • Reify 2023-WW-01
  • download 359 expofiles survey images
  • BuzzardHohle problem creation
  • check unique url when creating cave
  • url suppied to do_ARGE_cave is it cave or survexfile?
  • Port Radost srtm stuff into troggle
  • https://expo.survex.com/expofiles/tools/ Radost's new altitude stuff on ents page
  • On ents page, find cave name for each entry of p stations in last table. Create the inverse index
  • Why do we have a duplicate 78b entrance ?!
  • Why do the numbers on entrances not stack up on parsing.
  • Rewrite gpx2survex in python using bard conversion.
  • Remove /cave/1626-359 way of getting st cave info page? Only /1626/ etc.? Posted discussion on Matrix.
  • Re-architect the gps2survex stuff now that we have code to do everything
  • Need to go and visit 228 and see if it is really there, and photograph entrance. I think it might be same as 114.
+
+ +
+ +
+

EXPO Wallets

+
  • Wallet index pages in top level of /photos, move them
  • Extend to 3-digit wallet numbers
  • Troggle knows about survex files which have the *ref on them, but does not LIST them in the table, only their lengths.. Add them in to the report
  • Flag for wallet "reserved" as taken up the hill.
  • Remove ""you are logged-in individually": "on"" from JSON saved
  • cave id in wallets reports only works if there is a survex file, even when the id is explicit on the wallet. FIX THIS
  • Fix unrecognised cavenames in wallets should produce dataissue !
  • In cave wallet page list the people wallets.persons under the caves
  • Ensure all manywallets are query sets with select related done.
  • Calc the depth of each survex block too, so that we can report it on the annual wallets/surveys reports: ideally use cavern depth? "Down" or -" in compass doesn't work with topo things. use cavern: read the .3d file directly? Cant just do pitches as sexytopo fikes only near vertical.
  • Survex file {sx} was not found in LOSER repo" in models/wallets.py line 209 FIX THIS to look for survex files which have been renamed using the ALIAS to whatever was used and failed, e.g 309.svx is renamed 2023-ASH-09.svx
  • add to wallet edit page the survexblocks and links to survexfiles which we know are relevant because of the *ref. Not just the survexfile explcictly listed in the JSON
  • Rotate page - Wallet scans
  • Play with cache refresh options in rotate page.
  • Want to cache other file images but not current image. do
  • cache or instant-thumbnail for wallets images
  • check ref to svxfile is valid when loading a wallet in scans.py
  • Obscure Bug warning: https://expo.survex.com/cave/scans/1623-290 includes a reference to 2018#49 which is actually 264 rubberman not 290 rubblerumble2 & 3. I think this is a parsing/coding error..
  • Wallets where survex file is not in system yet should give sensible message, re "2022#40 and #44 are misbehaving"
  • Try wallets pages with no data imported, what works? Write tests
  • fpath in every wallet is rooted. Should instead be relative to SCANS_ROOT ?
  • write tests for wallets tick lists BEFORE refactoring tick lists code
  • parser/scans.py should read JSON and do the people, date, caveid thing if there is a survex file which has not got the *ref in it.
  • do get_ticks() tick list properly, not just from on-file JSON
  • These links to non standard wallets do not work, because I have messed up the wallet path for nonstandard wallets. The LUSS one works, because it has no /year/ directory above it. Try out the new scans parser and see if it works now.
  • Redesign: many tick-lists should be on the SurvexFile not on the Wallet!
  • Redesign: multiple caves per wallet: a list in the JSON and a m..n link in database. Or 1..n, m..1 easier?
  • Fixed?
  • on the server, there are DataIssues for 'scans' which do not appear on local PC - but whenever a wallet is created, a JSON file should be created. And none are on the server. IS this now fixed?
  • models/wallets.py fix the broken reprn of archaic wallets without breaking all the tick-lists ! +if len(wyear) != 4 or len(wname) !=6: +is not the way to do it - but is it broken?
  • Things it might do in future (if someone gets around to it) include:
  • - checking the cave number specified matches the folder for the .svx file,
  • - checking that the *ref: filed in the survex file is the same as the wallet name
  • - being more intelligent about .topo files and thus the lack of scan files,
  • - checking the date is in the recent past etc.
+
+ 29 completed items +
  • select wallets with no PersonExpedition on them, exclude survey length from total (arge)
  • Wallets with subdirectories currently give misleading error messages: detect and suppress these, give better msg. +- Loaded All Survex Blocks. + - Update wallets with survex data +! /home/philip/drawings/walletjson/2015DL01/survex/contents.json is not a file: +wyear='2015DL01' (should be eg. '2023') +wname='survex' (should be eg. '2023#13') +self.fpath='/home/philip/expofiles/surveyscans/2024/2024#15/2015DL01/survex' +and +7.74 s to set survexblock:wallet using JSON survexfiles +
  • set walletdate when parsing input for wallets
  • fix bug http://localhost:8000/scanupload/2020:01 +DoesNotExist at /scanupload/2020:01
  • make default wallet be the one after the last one that exists in the system, not #00
  • set walletdate when saving a new or edited wallet
  • Add "Create this wallet" instead of asking user to upload an empty file
  • Update bug for 2022:00 +REDESIGN the nav bar for wallets !
  • when editing wallet metadata, it does NOT store changed wallet name, or people ?! why or how ? BECAUSE a blank entry now triggers the repopulation from other sources! Makes it seem flaky.
  • Wallets: when there is no notes file, add a checkbox to say no notes file required: to enable changing the N red block to a green block
  • TRAINING: need a stupid-person's guide to Wallets page
  • fix bogus Wallet objects created by spiderbots
  • fix bad old wallet display without killing tick boxes in wallets
  • Find out what is causing the black box for survexfiles in some wallet table displays
  • ! In 2007#28 there is an unrecognised cave name 'surface' (out of 2,939 cave names and aliases). reset db & check if fixed.
  • add views/scans.py page to the list on the todo list
  • When creating wallet, check that there are not files or folders in the prposed new /surveyscan/ folder
  • on first parsing,, make wallets store names as a list of Person slugs, not fullnames, or keep as done lazily ?
  • add the participants on an explicit wallet list to .slugpeople so that they get proper URL-linked
  • on the per-person wallet report, and do the same thing for per-cave and per-year wallet reports
  • make separate tick box for electronic surveys NOT simply Therion/Tunnel
  • Relabel Website Updated tickbox
  • Rotate image option in image rename form
  • Make a list of cave ids work as well as a single caveid
  • in wallet editing page (scan upload) produce the coloured bocks for just this wallet.
  • survexfile not showing against reports for https://expo.survex.com/survey_scans/2018%252339/ on server, OK on localhost !
  • Missing notes for Backslash in https://expo.survex.com/logbookentry/2019-07-19/t2019-07-19y
  • And add SameDate *wallets* to the walletform page.
  • -walletslistyear - Failed to find cave object from id 1623-2012-ns-05
+
+ +
+ +
+

EXPO Troggle

+
  • Raise exceptions instead of returning None in troggle.
  • Return a dataclass, not a long tuple in logbook parsing.
  • Make "drawings" a first class object in troggle? Scan?
  • Logbook editing
  • put in "\n" into logbook entries when saving them. No, replace with tinymce.
  • If url begins with expofiles, do not insert /years/
  • Try tinyMCE in /logbookedit/ to see how it works. Use cloud hosting https://www.tiny.cloud/docs/tinymce/6/cloud-quick-start/
  • enable Martin's image add tool to Logbook edit. But where is "current directory"? Needs attention as \logbookedit is not /years/2025/*_edit , which perhaps it should be ?
  • Document images upload process in handbook
  • document new ents tags report in handbook, there are several entrances reports
  • Drawings upload
  • Register the uploaded drawing file (views/uploads.py) using the functions in parsers/drawings.py so that queries of the database can find newly uploaded files without having to do a database reset.
  • Expoweb photo upload
  • re-upload photos of my cave entrances in 2024 (and 2023) so that the GPS data gets stored.
  • Check signal 18/2/25 for Wookey & Olly comments on this: + "1623/41/115.htm" inside "cave_data/1623-115.html" + "1623/41/144.htm" inside "cave_data/1623-144.html" + "1623/258/misc.html" inside "cave_data/1623-258.html" + "1623/41/41.htm" inside "cave_data/1623-41.html"
  • Finish coding 'Using prefetch_related for Reverse Foreign Key Relationships' - use copilot text
  • When getting Class with lots of fields, select only fields needed. i.e. Caves
  • Django query optimisations for scans page report. Hard when doing inverse search on /scans/
  • https://slimsaas.com/blog/django-scaling-performance
  • GPX
  • Check out caves gpx from nat
  • Essentials. make entirely new make_entrances script using GPX as the master format. Actually entrances gpx not essentials.
  • Check becka & martin upload photos to cave description and write documentation. 2023 See emails.
  • find Becka example where ent photo GPS is of the *previous* cave
  • Find photo in potato hut where GPS is outside on the hill
  • When copying survex files in databaseReset.py, use shutil.copyfileobj(ifile, sf) as it is MUCH faster
  • Check Flagged emails for to-do things: Martin Green tags and entrances in 2023.. re-visit sometime...
  • Martin's 10/12/2024 email is jQuery docum which belongs in troggle handbook
  • Generic Error page to produce an online archive of errors: in database? Centralise Generic errors to one common function.
  • EMAIL me/ nerds for any "GENERIC" error pages when they occur.
  • Detect common git and autofix and email nerds with git status checks frequently
  • Generic error page: record a note of who and when and email me. Mostly bots?
  • Diagnostics page which runs git status on each repo and displays results on an HTML page.
  • Install CaveView in os-trog.sh and set a new localsettings route for it. Can do same for TinyMCE?
  • put as much as possible of handwritten to-do lists into Google Keep.
  • Change tuples returned from functions to named-tuples (or first, *_=..) so they can be keyword-addressed rather than using [0] and [1] https://docs.python.org/3/library/collections.html#collections.namedtuple
  • replace number-indexed tuples with named-tuples, such as things returned from matches, look for [0] and [1] in code.
  • Dont use named tuples https://snarky.ca/dont-use-named-tuples-in-new-apis/
  • get BING image on my OSMand using Wookey's instructions
  • get contours on OSMand by downloading it from f-droid on Wave8 phone
  • New Expo Year automation: list of jobs in https://expo.survex.com/handbook/computing/newyear.html
  • 33 occurrences of =open(), including the fairly nasty idiom +return HttpResponse(content=open(imagefile, "rb"), content_type=getmimetype(file)) +which never does close the file handle ? +cd troggle +grep -nir --exclude=".js" --exclude=".pyc" "=\s*open("|wc
  • XML files url-routed to a new INFO page with option to download or to see the XML code
  • replace pyaes python implementation of Fernet with cryptography package when the rust/subinterpreters thing is fixed with wsgi (Aug.2025 fix)
  • Look for all the .filter() where we just use [0] and detect multiple objects returned and do something about them. write expect_single() to encapsulate a .filter() call, and .count()
  • Use select_for_update() and a transaction to prevent conflicts when 2 people try to save the same file. But MariaDB support lacking. Maybe only when asgi working for troggle.
  • archive (wget) the CUCC organisational wikis 2008-2019 https://camcaving.uk/Documents/Expo/Legacy/
  • parse the uploaded drawing file for links to wallets and scan files as done in parsers/drawings.py
  • replace all old os. libraries with pathlib https://docs.python.org/3/library/pathlib.html#correspondence-to-tools-in-the-os-module
  • therion images, omit from report those scan files which work
  • FIX drawings upload form when some bugger has FTP'd or git'ted some files into the repo. --Cannot reproduce fault either locally or on server when editing JSON, either simple file present or when git added but not committed
  • Get the SLACK messages archived off
  • when importing drawings, check for recursive folder structure and infinite loop
  • Why no errors on parsing tunnel files ? Is it only WSL2 ? +RESTORE DataIssues, don't delete
  • Make data entry templates use more Form object to render the HTML, and more ModelForm use in some cases. Makes maintenance easier and provides better in-use validation feedback to users. No Class-based views!
  • Cull items from the online to-do list webpages - mostly cave description and tunneling https://expo.survex.com/handbook/computing/todo-data.html#264
+
+ 265 completed items +
  • statistics incorrect for some years, do not add up the survex files. svx files not listed under wallets of course.
  • Extract GPS from EXIF on uploaded photos and display on /l/ page.
  • Re-copy my handwritten to-do lists and
  • settings.PHOTO_YEAR needs removing, user current_year()
  • Wallet form: if surveying paperless you MUST upload your Topodroid/Sexytopo/etc original files AND an output (jpg or png ideally
  • Check odd the 55 and 255 appear differently e.g. https://expo.survex.com/survexfile/255 https://expo.survex.com/survexfile/55 and also lots of others.
  • get localsettings.py from Muscogee
  • Not in the alias list so not checked? also http://localhost:8000/survexfile/158
  • document uv activate/deactivate
  • get /PHOTOS to Mohawk, from laptop?
  • Change Submit button to say whic cave/survex/logbook is being overwritten. To fix The Underwood Problem.
  • Do Martins thumbnails and image uploads preserve exif locations ?! No!
  • Fix default *team in template and handbook
  • re-register the EXIF gps on my Geshwandalm walk photos using the GPS track, and then update the entrance photos on the website.
  • fix .url file with correct links
  • remove 1623. from entrance names on entrances.gpx
  • Write up displaced photos miles away, e.g the one of the walk on the stoger weg 30km away.
  • Rsync photos with expo server
  • Check rsync all expofiles
  • look at Martin's JavaScript popup for image editing
  • Survexfile/264 works /1623-264 doesnt. Should not be /1623/264?
  • document +/cave_data/cavealiases.txt_edit
  • Document the New Kataster Number procedures in http://localhost:8000/handbook/computing/newkataster.html
  • Delete https://expo.survex.com/1623/2018-AD-02.html
  • make electronic wallets still need plan, elev etc
  • azirophale and aziraphale in the pages, whcih is correct ?
  • create pepper pot using Harry's location 47.68750Ā° N, 13.80789Ā° E
  • QM - fix existing reprots to reflect OPEN and TICKed properly, and teh resolve station point.
  • Move boe nerdery to separate page
  • fix template page https://expo.survex.com/accounts/login/ outdated
  • Fix entrances being lost by Martin's code
  • revise the Name parser for the 19 (?)unrecognised names
  • still not seeing "entrances" dataissues in report
  • cookie to store name of online-editor to add to commit messages for survex files, caves and handbook pages.
  • Do file rename form for Nadia
  • fix tests which now fail with multiple caves per kataster number..
  • Do thumbnails for wallet images below line on rename form
  • FIX BUG in troggle test suite for logins. Due to different hash function? Make test person with a factory method not an imported file. Or due to secrets not being managed properly?
  • /logbookedit/2023 is downloading a previous entry instead of producing a blank one !
  • Return code 1 is OK from git IFF we are on the development machine. Use this to stop daft failures on the test laptop e.g for updating/editing logbook
  • Update my expofiles/ on laptop
  • Sort out untyped 2023 logbook using git history - identify the missing entries still
  • make a delete entry capability for the logbooks.
  • Find out how James does multiple entries
  • make current_year() NOT use user "expo"
  • need file locking for logbook.html?
  • Martin's photo upload stuff just puts them in :expoweb:/i /l /t instead of into :expoweb:/1623/2023-MS-01/i , l, t
  • fix essentials2024.gpx manually
  • Upload GPX form.
  • base.html and menu.html rely on year being passed and have a default of 2024. This should be {% now "Y" %} somehow..
  • Upload callout 2022 2023 to server
  • New Year not working automatically for 2024 logbook without editing python, in parsing logbooks
  • Fix bad image links in https://expo.survex.com/logbookentry/2022-08-16/2022-08-16d
  • http://expo.survex.com/survexfile/1623-2023-JSS-01 shows 2012-ft-01 ! So does http://expo.survex.com/cave/scans/1623-2023-jss-01
  • Do historic expo leaders page becka emails
  • fix bad URLS to years/2022/ukcavingblog_files/
  • search for cave names inside list as error on wallets with a list of caveids
  • make list of cave ids on wallet form be detected individually as caves.
  • Stop noedit bypass by using _edit hack.
  • Botch.
  • Fix 6 caves without 1623 prefix in entrances and cave slugs
  • Make a newcave insist on a new entrance. Test ?!
  • fix unrecognised cave name in scans.py
  • Laptop: do mugshot files for joel, evelyn and maddie
  • Lizie new cave survexfile problem, fix hsndbook! NewCave form should say NewCave not Edit cave, and should have link to right bit of handbook
  • New table on ents page listing everything with lat/long
  • Move EOL & UTF nerdery on http://localhost:8000/handbook/computing/hbmanual1.html to separate page also on newcave page
  • test failure if no therionrefs.log so must run reset before running test on new install. Or touch it as part of setupbefore test?
  • Use radost's expofiles/tools to get altitudes of all entrances from positions
  • Add readme to expofiles/drawings/
  • Edit this Cave: make it read the cave_data html file first, not the stuff already in the db, before presenting it to be edited.
  • New EDIT page for text files such as PENDING (and README?)
  • Done no wallet survex file report
  • Do person slug thing in urls.py +Redesign person name so no regex anywhere. Slugify. see https://expo.survex.com/handbook/troggle/namesredesign.html
  • update lengths script to do 290, 291, HC as well as 115, 161, 258, 264 etc.
  • screwy table formatting on http://localhost:8000/survexfile/264
  • Get the TRELLO messages archived off, wookey did this.
  • why no error message when two *ref in one block in http://localhost:8000/survexfile/caves-1623/204/swings/juicybits.svx
  • Import all old wallets in parser/scan not just smhks - but they seem to be there anyway !?
  • update docm about things which now work in wallets: newcave, newwallet, onlinewallets
  • weird http://localhost:8000/dwgdataraw/290/290+291+295-Oct2022-Pre+Post2022ExpoOK.xml file not found, but the link is there in Drawings and it is...
  • Scan callout book (again), post to website
  • Fix python 3.11 problems in parsers/logbooks.py
  • Remove all caching from logbooks.py, 0.9s faster that's all.
  • remove all Object Store code attempt in logbooks.py. We need the multi-user DB ! Comment out first: how much faster ?
  • check 2019 callouts against 2019 logbook
  • Logbooks
  • create logbook-blog fold in instructions
  • 2017 caving blog needs parsing into the troggle system just like 2019.
  • missing photos in 2017 blog are in expofiles photos ChrisDensham
  • add 2016 camping-in-tunnocks-elaine
  • the reload from logbook button seems to be better at identifying all the trips, and the reload link puts things in the right place better, but I am still very amused their results are not the same...
  • logbook-editor laptop docum for Harry Kettle. maybe
  • do round-trip 2019 logbook and hand-edit export to remove duplicates of blog entries
  • 2019 logbook add scanned diagrams
  • add unknown names for 2022 logbook to German list
  • check blogs against 2019 logbook and copy text and photos
  • Should I write a parser just for UK Caving blog ?
  • do round-trip 2018 UKCaving expo training weekend blog
  • 2022 caving blog - does it exist ?
  • check 6 pages 2018 callout book is in logbook
  • 2009 logbook parser misses first 300 lines. Fault in wiki parser.. Also no content for any entry into database
  • add a test that just copies a file to /drawings, /expofiles/photos /expofiles/surveyscans etc.
  • fix broken logfile reading for 3 bad years.
  • Logbook import errors in 2007
  • NEW FORM to add a logbook entry, which adds to logbook.html and does git commit.
  • Logbooks not parsing swapped title / people properly still.
  • Change documn. re Windows bulk update laptop to use WSL1 ONLY.
  • Remove WSL bits from troglaptop and put in WSLlaptop ?
  • Wallets
  • Wallets where survex file is not in system yet should give sensible message, re "2022#40 and #44 are misbehaving"
  • Add "Create this wallet" instead of asking user to upload an empty file
  • Wallets: whern there is no notes file, add a checkbox to say no notes file required: to enable changing the first red block to a green block
  • Wallet surface/cave radio button, grey out cave id. Check all old wallets for word 'surface'
  • Try wallets pages with no data imported, what works? Write tests
  • fpath in every wallet is rooted. SHould instead be relative to SCANS_ROOT ?
  • when editing wallet metadata, it doen NOT store changed wallet name, or people ?! why or how ? BECAUSE a blank entry now triggers the repopulation from other sources!
  • write tests for wallets tick lists.
  • BUT FIRST change databaseReset to run for only a singe year for a logbook
  • fix APPEND SLASH properly in middleware +https://docs.djangoproject.com/en/5.1/ref/middleware/ +https://gregbrown.co/code/append-or-remove-slash/ +https://djangosnippets.org/snippets/601/ +
  • add a test that just copies a file to /drawings, /expofiles/photos /expofiles/surveyscans etc. to narrow down permissions errors.
  • update handbook re removal of old awk script /handbook/troggle/scriptscurrent.html#survex
  • http://localhost:8000/svxvalid.html
  • Check templates,
  • check templates never (?) used in templates/ folder
  • check templates which don't exit referenced in python code
  • how do I make pydoc work on troggle ? import naming issue?
  • why is 115 or 145 not appearing in http://localhost:8000/survey_scans/ for 1983, because the json is not parsed on import. Fixed in the django template: calls wallet.cave()
  • link in DataIssues does not get to the survex file which caused it ! 2017-dm-01 Pending cave write-up - creating as empty object. No XML file available yet. url: 1623/2017-dm-01
  • do a web page which prints out the contents of the personexpeditions lookup table
  • fix wallets link in person page
  • oconee WSL1
  • check ownership structure on SnowWhite and Muscogee and where the venv is, and who owns it.
  • horrible mess wth file ownerships for expofiles not being writeable by troggle or by rsync scripts
  • horrible mess with git complaining about dubious ownership and refusing to sync
  • venv script barely working, but links not right
  • Do I give up with WSL1 (and thus with oconee?)
  • fix names all being lower case now from survex files
  • fix per-person survey lengths, e.g. Paul Fox is not zero !
  • on *date, reset the date for all the personrole objects attached to the block, as the *team line may have come first..
  • Entrance tag type:Tag : metal, paint :yes' no tag in entrance data model. Already there ?
  • This only count number of logbook trips. Should do survex surveys too http://localhost:8000/person/EmmaWilson +But this means adding 'person' as a Django object link to the wallet, not just having a string with a JSON list of people. +
  • why does scorchio not show people s valid people whereas teh other blocks do ? http://expo.survex.com/survexfile/caves-1623/161/triassic/scorchio.svx
  • still not getting people from survex file in 2022#35, should happen even though the survex file is not parsed ? Invalid *team, fixed? try again..
  • wllet status link incorrect on /personexpedition/GeorgeBreley/2015
  • Move many DataIssue things into separate troggle web pages, e.g. Therion stuff, and group all those pages as Data Integrity inc. current entrances page
  • the exclusion of gpx, kataster etc does not work when they are *included. fix a test soemhow
  • Caves in areas 1624 not shown? survex file list is identical between snowwhite and expo, why are the new survex files not being shown ? Fiters ? +1987 is different /expedition/1987
  • Find and process non-tree svx files,
  • Change reading linear file into memory before parsing it. Redo as generator: 30 MB !
  • unrecognized survex files producing wrong error for survex files in subfolders.
  • some caves have url links and some don't in wallets report pages, e.g. http://localhost:8000/wallets/person/ChrisDensham why ?
  • scans dataissue: prevent duplication on rerun . Do not create if already exists. But why is it run not during import?
  • create walletsindex checklist table on the cave/year/person wallets pages
  • add list of logbook entries and survex files for this same date as the wallet. For the wallet page.
  • It is the viewing of drawings, not the upload, that uses the db instead of just looking in the filesystem. Fix. http://localhost:8000/dwgdataraw/junkk.xml
  • test creates 2020/2020#00 even if it doesn't exist. Stop this happening!
  • fix dirty git repo drawings wallets json. How, How to fix ?
  • wallet reports not showing people if a survex file exists but the JSON is blank e.g. http://expo.survex.com/survey_scans/2022%252342/ +cf difference between 2019#03 and #11
  • big problem, upload test file to completely empty wallet, it inherits old wd JSON data somehow
  • Need bug report when wallet JSON refers to a svx file which is not there (or has been renamed or moved)
  • still getting old wallet data when creating new one
  • fix dates if not ISO format when first seen in survex files or wallet JSON. Parser should creat DataIssue if changing . to - does not fix.
  • fix git origin/master on snowwhite
  • How can the same form have 2 different URLs ?! +which is still named /scanupload/2022:17 or, apparently alternatively, /survey_scans/2022%252317/ (which should be impossible..))
  • works as far back as 2012 (though it fails for logbook entries in 2011, not sure why yet
  • wallets- populate empty wallet field in the uploadscans form (single wallet) not just in the reports
  • Bug in survex passing, picks up ; 2022#15 when there is no *ref or ;ref in caves-1623/290/nicolascage
  • bug in [people] list 2015#38 and 39, one shows people, one doesnot.
  • missing svx files in reports even after data import
  • 2022#25 has no link to survex file, but http://localhost:8000/survexfile/caves-1623/290/pickingbalconysnostril.svx does contain a correct *ref. How is this bug possible ? And is shown in report.
  • why is 2022 #43 not showing that there is a survex file?
  • if there are no files in the expofiles/surveyscans wallet, then the data in the JSON file is NOT VISIBLE and not presented ! Neither are the names in the JSON file presented on any reports. I uploaded desktop.ini to 2022#55 but it still lists as black across the board. 'created' value not useful?
  • 2012#07 and 09 , 07 showing wrong survex file
  • 2022#17 has a blank json file. So why are several bits ticked on the tick list ?
  • 2012#20 and 21 one showing S as green the other not. Depends on ".svx" in waldata["survex file"] field somehow
  • edit /handbook/survey/onlinewallet.html to update
  • put url link to view contetns.json directly from the upload page to check against data displayed.
  • Find Frames in use and fix so not used
  • Put hugtheplug email text into readme file in that scanfolder
  • finish cleanup of parsers/scans.py of all wallets contents using Path.rglob
  • Move aliases to person from personexpefition
  • drawings bug, image files not rendering +http://localhost:8000/survey_scans/2006%252306/ +http://localhost:8000/survey_scans/2017%252327/plan-1.jpeg +but http://localhost:8000/survey_scans/2004%252318/ is fine + +
  • https://www.aboutchromebooks.com/news/chromeos-109-adds-sftp-to-the-terminal-on-chromebooks/
  • Django 4.2 changes admin pages CSS cosde. FIX.
  • ExpeditionDay is attached to PersonRole on a survex file, but not on a PersonTrip attached to a logbookentry
  • remove use of save)carefully in parsers/people and parsers/QMs
  • ExpeditionDay, unused in Logbooks. But is it used in TEMPLATES ?!
  • document where all.svx is used or mentioned and replace with _<whtever the file is>.svx
  • rename PersonTrip as PersonLogEntry
  • survex parser is not inheriting-down a *team into sub-survexblocks which do not have any *team entries
  • remove expeditionday from Expedition
  • remove nickname from PersonExpedition
  • do bulk creation for the deferred DataIssues in store_data_issues()
  • SurvexBlock, why is the 'cave' field empty ?
  • run ruff on more of the code, the views
  • todo
  • update http://localhost:8000/todo/anything
  • Update todo section at top of each file of python code,
  • Make todo properly visible on troggle without iframe
  • Move KeninWassermelon survey but check first whether expoweb files links would break
  • Bug in 2018#26 https://expo.survex.com/logbookentry/2018-07-24/t2018-07-24a_homecoming
  • Timesramp in cave and ents files in expoweb
  • Entrsnce field for date last visited.
  • Many names in survex fikes still not being recognised, but fix 'on expo' name check first
  • 1983 wallets not parsed in databasereset
  • wallets- Make repo master for contents json
  • Git commit upload drawing file. Huh? It does this. TEST it.
  • Drawings upload does not show live filelist
  • Update db wallets contents on wallet save, check.
  • If svx file has *ref then link it to that wallet in json !
  • Populate empty date and people on wallets on parser import 'scans'
  • Readme.txt. Missing in troggle/Readme on barbie
  • Rename file on photo upload
  • Copy note from matrix and make page on names in handbook
  • Do the same SameDate thing for logbook entries
  • Survex parser: detect bad dates using single digit day number. Dot is correct separator. Also detect date ranges properly.
  • Add catch-all free text box to wallets JSON
  • Img rotate extrndion for chrome on laptop
  • Wallet to wallet relationship when notes are drawn up into plan and elev in a different wallet. OR free text field.
  • http://localhost:8000/survexfile/caves-1623/290/nicolascage.svx has *ref #14 but appears in list as #15 ! both JSON are correct. Both svx are correct. Recent import done.
  • Becka: I don't understand why, in a minority of cases (eg wallet 2022 #14 and #35) the cave (eg 1623/290) doesn't get automatically filled in. In other cases that looked just the same to me they did get filled in.
  • Add same date survex files and logbook entries to the wallet page
  • Update handbook scripts: obsolege re wallwts.py
  • wallets- Update handbook re wallets - again. http://localhost:8000/handbook/survey/onlinewallet.html
  • Use include for upper table in the 3 wallets reports
  • When teplacing a pending file with a real ine does it barf ?
  • go through becka's list of moved svx files and check dataissues
  • copy all contents.json to e.g. drawings/wallets/2015#25.json and commit on import or when wallet is updated.
  • weird bug on https://expo.survex.com/cave/scans/1626-359
  • weird bug in http://localhost:8000/cave/scans/1623-204 truncating id
  • Date before name bug in parsing survex files, produce a dataissue msg
  • https://expo.survex.com/survexfile/caves/ incorrect links for 354 ie /survexfile/354 but /survexfile/359 resolves fine. Hmm.
  • http://localhost:8000/survexfile/361 not resolved
  • do therion *ref doing regex for e.g. in the field 2019#12
  • https://expo.survex.com//survexfile/55 correct, but https://expo.survex.com//survexfile/255 incorrect, both are caves in both areas 1626-255 seems to be missing now..
  • Fix link to cav4s on ents page
  • we need a report of wallets for all the wallets ordered by Person, like the one per cave
  • fix git for online edited svx files
  • Fix test with upload scanfile which complains a bit on git add but not commit.
  • CSV QM import does not process the 'ticked' field.
  • QM single end page does not have link to the survex file from which it was extracted
  • remove links to old walletsindex pages
  • Add ollys comment re survexport
  • GPS guide not Photography Guide in handbook menu. Possibly Prospecting Guide ?
  • QMs, remove the whole linked-logbook thing. We only have QMs from CSV or survex.
  • Fix http://localhost:8000/cave/qms/1623-264/2019-lipstickdipstick2B
  • work through the remaning PENDING caves and create 1623-XX.html files for them properly, and rename the files with 1623- prefix for cave_data too
  • UTF8 stuff
  • Detect failed UTF8 file at runtime and redirect to an error page with helpful message in expo.py. Will usually happen in EditThisPage or in editing survex files.
  • create test to check for locale is UTF8 +However in the WSGI environment the system/locale encodings are:sys.getdefaultencoding() = utf-8 +sys.getfilesystemencoding() = ascii +locale.getdefaultlocale() = (None, None) +locale.getpreferredencoding() = ANSI_X3.4-1968
  • Scans list, but for just one cave
  • Fix new 2019 logbook entries
  • logbook parsing mismatch : total 1,906 log entries parsed in all expeditions ** total trips in ObjStore:1,942
  • QMreport page basic
  • Why no table 1 on /people on recent django 3.2? - because of Django template security chnage
  • Do <details> and <summary> in to-do pages
  • python generator to read/process survex files
  • Fix duplicated entries on person expedition page, wookey
  • System
  • Install Expo s/w on SnowWhite WSL2 and check documn works.
  • pip install pep8radius https://stackoverflow.com/questions/14328406/tool-to-convert-python-code-to-be-pep8-compliant
  • get working on MythicBeasts ubuntu server setup for EXPO on 8000
  • use Bolt to configure full expo installation
  • Bug in Expo page, see 2001. Only 2 months shown, not Sept. days
+
+ +
+ +
+ + diff --git a/handbook/troggle/process_keep.py b/handbook/troggle/process_keep.py new file mode 100644 index 000000000..7f7b05aa8 --- /dev/null +++ b/handbook/troggle/process_keep.py @@ -0,0 +1,302 @@ +import os +import json + +""" This was written almost entirely by Google Gemini 2.5 (apps and code) on 18 Sept. 2025 + +Google Takeout is accessible from the browser when logged in to your Google account. +It can be set up to export dozens of different types of data. Here we only export "Keep" +data. +After running this, move the generated file to expoweb/handbook/troggle +and git add/commit/push +Philip Sargent +""" +# --- CONFIGURATION --- +# šŸ“Œ IMPORTANT: Replace this with the actual path to your Google Keep Takeout folder. +TAKEOUT_DIRECTORY = 'Takeout/Keep' # Example for Linux/macOS +# + +def process_keep_files_to_dict(directory_path): + """ + Scans a directory for Google Keep JSON files, filters for a specific + label, prints the content, and returns the data as a list of dictionaries. + + Args: + directory_path (str): The path to the folder containing the JSON files. + + Returns: + list: A list of dictionaries, where each dictionary represents a note. + """ + print(f"šŸ“ Processing files in: {directory_path}\n") + + # This list will store the dictionary for each matching note + extracted_notes = [] + + if not os.path.isdir(directory_path): + print(f"āŒ Error: Directory not found at '{directory_path}'.") + print("Please update the TAKEOUT_DIRECTORY variable with the correct path.") + return extracted_notes + + # Iterate over every file in the specified directory + for filename in os.listdir(directory_path): + if filename.endswith('.json'): + file_path = os.path.join(directory_path, filename) + + try: + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # --- Label Check --- + has_expo_label = False + if 'labels' in data: + if any(label.get('name', '').lower() == 'expo' for label in data['labels']): + has_expo_label = True + + if not has_expo_label: + continue # Skip this file + + # --- Data Extraction & Storage --- + + # Create a dictionary to hold the current note's data + current_note = { + 'title': data.get('title') or 'Untitled Note', + 'source_file': filename, + 'content_type': None, + 'color': data.get('color', 'DEFAULT'), + 'content': None + } + + # As before, print to the console for immediate feedback + print("-" * 40) + print(f"āœ… Title: {current_note['title']}") + + # Handle text notes + if 'textContent' in data: + content = data['textContent'] + current_note['content_type'] = 'text' + current_note['content'] = content + print(content) + + # Handle list notes + elif 'listContent' in data: + current_note['content_type'] = 'list' + list_items = [] + checked_items = [] + # print("Tasks:") + for item in data['listContent']: + # Create a clean dictionary for the list item + item_data = { + 'text': item.get('text', ''), + 'is_checked': item.get('isChecked', False) + } + list_items.append(item_data) + if item_data['is_checked']: + checked_items.append(item_data) + + # Print the list item to the console + status = 'x' if item_data['is_checked'] else ' ' + # print(f" [{status}] {item_data['text']}") + print(f"Tasks: {len(list_items)} of which {len(checked_items)} are done.") + current_note['content'] = list_items + + # Add the completed dictionary for this note to our main list + extracted_notes.append(current_note) + + # print("-" * 40 + "\n") + + except Exception as e: + print(f"ā—ļø An unexpected error occurred with file {filename}: {e}") + + if not extracted_notes: + print("No notes with the 'EXPO' label were found in the directory.") + + return extracted_notes + +import html +from datetime import datetime + +def write_html_output(notes_data, filename="keep_export.html"): + """ + Generates an HTML file from the extracted Keep notes data. + + Args: + notes_data (list): A list of note dictionaries. + filename (str): The name of the output HTML file. + """ + + # These are the standard Google Keep colors. + color_map = { + "DEFAULT": "#ffffff", + "RED": "#f28b82", + "ORANGE": "#fbbc04", + "YELLOW": "#fff475", + "GREEN": "#ccff90", + "TEAL": "#a7ffeb", + "BLUE": "#cbf0f8", + "DARK_BLUE": "#aecbfa", + "PURPLE": "#d7aefb", + "PINK": "#fdcfe8", + "BROWN": "#e6c9a8", + "GRAY": "#e8eaed", + } + + # --- HTML Head and CSS Styling --- + # The CSS is embedded directly in the HTML file for simplicity. + html_head = f""" + + + + + + Google Keep Export + + +""" + + # --- HTML Body Generation --- + html_body_content = "" + for note in notes_data: + # Sanitize title to prevent HTML injection issues + title = html.escape(note['title']) + + # Start building the content for this specific note + note_content_html = "" + + # MODIFICATION 2: Get the color and apply it as an inline style + note_color_name = note.get('color', 'DEFAULT') + # Safely get the color from the map, falling back to the default white + bg_color = color_map.get(note_color_name, color_map['DEFAULT']) + + # Handle text notes + if note['content_type'] == 'text': + # Sanitize text content and replace newlines with
tags + text = html.escape(note['content']) + note_content_html = f'

{text}

' + + # Handle list notes + elif note['content_type'] == 'list': + unchecked_items = [] + checked_items = [] + + for item in note['content']: + # Sanitize list item text + item_text = html.escape(item['text']) + if item['is_checked']: + checked_items.append( + f'
  • {item_text}
    • ' + ) + else: + unchecked_items.append( + f'
  • {item_text}
    • ' + ) + + # Combine unchecked items + note_content_html += '' + + # Add checked items inside a collapsible
    element + if checked_items: + item_count = len(checked_items) + plural_s = 's' if item_count > 1 else '' + note_content_html += f""" +
    + {item_count} completed item{plural_s} +
      {"".join(checked_items)}
    +
    + """ + + # Combine title and content into a single note card + html_body_content += f""" +
    +

    {title}

    + {note_content_html} +
    + """ + # --- Final HTML Assembly --- + full_html = f""" +{html_head} + +

    Troggle coding and design to-do lists

    +

    (Exported from Philip Sargent's Google Keep notes on {datetime.now().strftime('%Y-%m-%d %H:%M')}) +

    Contact Philip if you wish to share these lists on your Google Keep account. +

    + {html_body_content} +
    + + +""" + + # --- Writing to File --- + try: + with open(filename, 'w', encoding='utf-8') as f: + f.write(full_html) + print(f"\nšŸŽ‰ Successfully wrote {len(notes_data)} notes to '{filename}'") + except Exception as e: + print(f"ā—ļø An error occurred while writing the file: {e}") + +# --- Run the program --- +if __name__ == "__main__": + all_notes_data = process_keep_files_to_dict(TAKEOUT_DIRECTORY) + + if all_notes_data: + print("\n" + "="*50) + print("šŸŽ‰ Successfully processed all matching files.") + print(f"Total notes extracted: {len(all_notes_data)}") + # print("Data has been stored in the 'all_notes_data' variable.") + + print("="*50) + write_html_output(all_notes_data) \ No newline at end of file