From b1c5b03104b802849b5c6cd9b397bc0bccc803bd Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sat, 8 Jul 2023 17:13:57 +0100 Subject: [PATCH 01/10] link to 2023 in menus --- templates/menu.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/menu.html b/templates/menu.html index 4b15a5f..69787e6 100644 --- a/templates/menu.html +++ b/templates/menu.html @@ -38,7 +38,7 @@ {% endif %}
  • Areas
    • -
  • Troggle
    • +
  • Troggle
    • From 3d43c0ec12cdf339301bf12d3110be0d97edbee6 Mon Sep 17 00:00:00 2001 From: Expo on server Date: Sat, 8 Jul 2023 22:42:06 +0100 Subject: [PATCH 02/10] Allow for parent directories to be created when created photos. --- core/views/editor_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/views/editor_helpers.py b/core/views/editor_helpers.py index 17ae04b..7aa9c75 100644 --- a/core/views/editor_helpers.py +++ b/core/views/editor_helpers.py @@ -95,7 +95,7 @@ def new_image_form(request, path): # Create directories if required for full_path in image_path, thumb_path, desc_path: print(full_path, full_path.parent) - full_path.parent.mkdir(parents=False, exist_ok=True) + full_path.parent.mkdir(parents=True, exist_ok=True) try: change_message = form.cleaned_data["change_message"] write_and_commit( From beab42323bd9b7e436f16a165e295e05ba726caf Mon Sep 17 00:00:00 2001 From: Expo on server Date: Sat, 8 Jul 2023 23:43:57 +0100 Subject: [PATCH 03/10] Added CaveSlugs to Admin --- core/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin.py b/core/admin.py index b7c0d2b..1d8201f 100644 --- a/core/admin.py +++ b/core/admin.py @@ -3,7 +3,7 @@ from django.core import serializers from django.http import HttpResponse from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance -from troggle.core.models.logbooks import QM, LogbookEntry, PersonLogEntry +from troggle.core.models.logbooks import QM, LogbookEntry, PersonLogEntry, CaveSlug from troggle.core.models.survex import ( DrawingFile, SingleScan, @@ -139,6 +139,7 @@ admin.site.register(Cave, CaveAdmin) admin.site.register(Area) admin.site.register(CaveAndEntrance) admin.site.register(Entrance, EntranceAdmin) +admin.site.register(CaveSlug) admin.site.register(SurvexBlock, SurvexBlockAdmin) admin.site.register(DrawingFile, DrawingFileAdmin) admin.site.register(Expedition) From 086537cb5633d7c40a692501bacd126291f5f0a7 Mon Sep 17 00:00:00 2001 From: Expo on server Date: Sat, 8 Jul 2023 23:45:32 +0100 Subject: [PATCH 04/10] =Added string to caveslugs so admin interface works. Added explanation for the rational for cave slugs. --- core/models/logbooks.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/core/models/logbooks.py b/core/models/logbooks.py index 9a13b59..b0bde61 100644 --- a/core/models/logbooks.py +++ b/core/models/logbooks.py @@ -11,8 +11,15 @@ from troggle.core.models.troggle import Expedition, TroggleModel """ todo = """ -- Can we rewrite things to eliminate the CaveSlug and objects? Surely - foreign keys work fine ?! +- Can we rewrite things to eliminate the CaveSlug and objects? No + Surely foreign keys work fine ?! No + + Foreign keys do not allow for there being multiple ways to refer to a cave, eg 1623-1999-03 aka 1623-204 + Having slugs allows for much more loose coupling to caves, which removes alot of the need to reset the database, which interupts work flow. + It also means we do not have to be creating temporary cave objects in the database, where we do not have the underlying file in cave_data. + + To Do move Cave Slug back to troggle.core.models + """ @@ -25,6 +32,9 @@ class CaveSlug(models.Model): slug = models.SlugField(max_length=50, unique=True) primary = models.BooleanField(default=False) + def __str__(self): + return f"{self.slug}: {self.cave}" + class LogbookEntry(TroggleModel): """Single parsed entry from Logbook From 4d48dd43867ee8c25202b2b18c9407bee0278240 Mon Sep 17 00:00:00 2001 From: Expo laptop Crowley Date: Sun, 9 Jul 2023 11:30:50 +0200 Subject: [PATCH 05/10] Logbook debugging --- parsers/logbooks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/parsers/logbooks.py b/parsers/logbooks.py index e7c7363..5b9bb6e 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -57,7 +57,7 @@ LOGBOOK_PARSER_SETTINGS = { LOGBOOKS_DIR = "years" # subfolder of settings.EXPOWEB ENTRIES = { - "2023": 1, + "2023": 11, "2022": 90, "2019": 55, "2018": 95, @@ -295,7 +295,7 @@ def parser_html(year, expedition, txt, seq=""): headmatch = re.match(r"(?i)(?s).*]*>(.*?) 0: frontpath = Path(settings.EXPOWEB, LOGBOOKS_DIR, year, "frontmatter.html") with open(frontpath, "w") as front: @@ -305,7 +305,7 @@ def parser_html(year, expedition, txt, seq=""): endmatch = re.match(r"(?i)(?s).*([\s\S]*?)(?= 0: endpath = Path(settings.EXPOWEB, LOGBOOKS_DIR, year, "endmatter.html") with open(endpath, "w") as end: @@ -333,7 +333,7 @@ def parser_html(year, expedition, txt, seq=""): if s: tripid, tripid1, tripdate, trippeople, triptitle, triptext, tu = s.groups() else: # allow title and people to be swapped in order - msg = f" !- {year} Can't parse:{logbook_entry_count} '{trippara[:50]}'..." + msg = f" !- {year} Can't parse:{logbook_entry_count} '{trippara[:55]}'...'{trippara}'" print(msg) DataIssue.objects.create(parser="logbooks", message=msg) @@ -353,7 +353,7 @@ def parser_html(year, expedition, txt, seq=""): tripid, tripid1, tripdate, triptitle, trippeople, triptext, tu = s2.groups() else: # if not re.search(r"Rigging Guide", trippara): - msg = f" !- Logbook. Can't parse entry on 2nd pass:{logbook_entry_count} '{trippara[:50]}'..." + msg = f" !- Logbook. Can't parse entry on 2nd pass:{logbook_entry_count} '{trippara[:55]}'...'{trippara}'" print(msg) DataIssue.objects.create(parser="logbooks", message=msg) continue From 389fb6c409bc6438276f5ef7552c35f1e2cd723c Mon Sep 17 00:00:00 2001 From: Expo laptop Crowley Date: Mon, 10 Jul 2023 09:18:17 +0200 Subject: [PATCH 06/10] add month --- templates/expedition.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/expedition.html b/templates/expedition.html index ad6430d..c0b4724 100644 --- a/templates/expedition.html +++ b/templates/expedition.html @@ -37,7 +37,7 @@ an "S" for a survey trip. The colours of the "T" and "S" a Caver {% for d in dates %} - {{d.day}} + {{d.day}}/{{d.month}} {% endfor %} From 5a7f197bcb370638e6d54a4358ddca204a3e3ebb Mon Sep 17 00:00:00 2001 From: Expo laptop Crowley Date: Mon, 10 Jul 2023 09:24:11 +0200 Subject: [PATCH 07/10] hack to make server reset quickly --- parsers/survex.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parsers/survex.py b/parsers/survex.py index 70fc987..5394de6 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -739,7 +739,10 @@ class LoadingSurvex: print(f" {survexblock.survexfile=}") # raise - setdate_on_survexblock(year) + if year: + setdate_on_survexblock(year) + else: + setdate_on_survexblock("1970") if survexblock.date: # do not actually need a distict variable 'currentdate' but it makes the code clearer self.currentdate = survexblock.date From d0ccc46260a7752e0bbc8afb867e2d979d9890ae Mon Sep 17 00:00:00 2001 From: Expo laptop Crowley Date: Mon, 10 Jul 2023 10:03:23 +0200 Subject: [PATCH 08/10] better test & msg --- parsers/survex.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/parsers/survex.py b/parsers/survex.py index 5394de6..58763c5 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -739,9 +739,10 @@ class LoadingSurvex: print(f" {survexblock.survexfile=}") # raise - if year: + try: setdate_on_survexblock(year) - else: + except NameError: + print(">> why is year not set ?! {survexblock.survexfile.path}") setdate_on_survexblock("1970") if survexblock.date: # do not actually need a distict variable 'currentdate' but it makes the code clearer From b3e2f34960a4c49017c160cebbd191512fff0524 Mon Sep 17 00:00:00 2001 From: Expo laptop Crowley Date: Mon, 10 Jul 2023 10:33:57 +0200 Subject: [PATCH 09/10] hack fix --- parsers/survex.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/parsers/survex.py b/parsers/survex.py index 58763c5..25bb028 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1381,7 +1381,13 @@ class LoadingSurvex: survexblock.survexfile.cave.slug() self.fix_undated(survexblock) # null-op if already set - expoyear = str(survexblock.date.year) + try: + expoyear = str(survexblock.date.year) + except NameError: + print(">> why is survexblock.date.year not set ?! in LoadSurvexQM()/n {survexblock.survexfile.path}") + expoyear = "1970" + + try: qm = QM.objects.create( From 8ff438942dd0981d2dd442dae3e61fbd5a84280e Mon Sep 17 00:00:00 2001 From: Expo laptop Crowley Date: Mon, 10 Jul 2023 12:49:14 +0200 Subject: [PATCH 10/10] handle shortform months and days in svx file --- parsers/survex.py | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/parsers/survex.py b/parsers/survex.py index 25bb028..405771d 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -695,12 +695,15 @@ class LoadingSurvex: ) oline = line + perps = get_people_on_trip(survexblock) # What, you don't know Judge Dredd slang ? + if len(line) > 10: message = "! DATE Warning LONG DATE '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path) print(self.insp+message) stash_data_issue(parser='svxdate', message=message, url=None, sb=(survexblock.survexfile.path)) if line[10] == "-": # ie a range, just look at first date line = line[0:10] + if len(line) == 10: year = line[:4] # TO DO set to correct Austrian timezone Europe/Vienna ? @@ -708,7 +711,6 @@ class LoadingSurvex: survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d") elif len(line) == 7: year = line[:4] - perps = get_people_on_trip(survexblock) # What, you don't know Judge Dredd slang ? message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}" print(self.insp + message) stash_data_issue( @@ -717,13 +719,36 @@ class LoadingSurvex: survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m") # sets to first of month elif len(line) == 4: year = line[:4] - perps = get_people_on_trip(survexblock) message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}" print(self.insp + message) stash_data_issue( parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) ) survexblock.date = datetime.strptime(line, "%Y") # sets to January 1st + elif len(line) == 9 or len(line) == 8: + year = line[:4] + message = f"! DATE format WARNING, single digit day or month number,'{oline}' [{line[-5]}][{line[-2]}] ({survexblock}) {survexblock.survexfile.path}" + print(self.insp + message) + stash_data_issue( + parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) + ) + if line[-2] == "-" or line[-2] == ".": + line = line[:-1] + '0' + line[-1] + survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d") + print(f"! DATE -2 '{line}' '{survexblock.date}'") + elif line[-5] == "-" or line[-5] == ".": + line = line[:-4] + '0' + line[-4:] + survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d") + print(f"! DATE -5 '{line}' '{survexblock.date}'") + else: + year = line[:4] + message = ( + f"! DATE Error SHORT LINE '{line}' '{oline}-{survexblock}' ({type(survexblock)}) {survexblock.survexfile.path}" + ) + print(self.insp + message) + stash_data_issue( + parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) + ) else: # these errors are reporting the wrong survexblock, which is actually a SurvexFile (!) # see To Do notes on how to trigger this. Still needs investigating.. @@ -737,13 +762,16 @@ class LoadingSurvex: print(f" {type(survexblock)=}") # survexblock.parent fails as a SurvexFile has no .parent ...ugh. print(f" {survexblock.survexpath=}") print(f" {survexblock.survexfile=}") + # Not setting 'year' crashes entire import on databaseReset. + year = line[:4] + perps = get_people_on_trip(survexblock) # raise try: setdate_on_survexblock(year) except NameError: - print(">> why is year not set ?! {survexblock.survexfile.path}") - setdate_on_survexblock("1970") + print(f">> why is year not set ?! {survexblock.survexfile.path}") + setdate_on_survexblock("1976") if survexblock.date: # do not actually need a distict variable 'currentdate' but it makes the code clearer self.currentdate = survexblock.date @@ -1383,8 +1411,8 @@ class LoadingSurvex: self.fix_undated(survexblock) # null-op if already set try: expoyear = str(survexblock.date.year) - except NameError: - print(">> why is survexblock.date.year not set ?! in LoadSurvexQM()/n {survexblock.survexfile.path}") + except: + print(f">> why is survexblock not set ?! in LoadSurvexQM()/n {survexblock.survexfile.path}") expoyear = "1970"