From 9e11c0814ef1475cd51ee444acdf52244bd0cca6 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 26 Apr 2021 17:23:23 +0100 Subject: [PATCH] missing entrance .html file now handled differently --- parsers/caves.py | 144 +++++++++++++++++++++++++++-------------------- settings.py | 6 ++ 2 files changed, 89 insertions(+), 61 deletions(-) diff --git a/parsers/caves.py b/parsers/caves.py index ce2eaa7..3a58748 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -23,6 +23,52 @@ entrances_xslug = {} caves_xslug = {} areas_xslug = {} +def dummy_entrance(k, slug, msg="DUMMY"): + '''Returns an empty entrance object for either a PENDING cave or a DUMMY entrance if + user forgot to provide one when creating the cave + ''' + ent = Entrance( + name = k, + entrance_description = "Dummy entrance: auto-created when registering a new cave " + + "and you forgot to create an entrance for it. Click on 'Edit' to enter the correct data, then 'Submit'.", + marking = '?') + if ent: + ent.save() # must save to have id before foreign keys work. + try: # Now create a entrance slug ID + es = EntranceSlug.objects.update_or_create(entrance = ent, + slug = slug, primary = False) + except: + message = f" ! {k:11s} {msg} entrance create failure" + DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}') + print(message) + + ent.cached_primary_slug = slug + ent.filename = slug + ".html" + ent.save() + return ent + else: + message = f" ! {k:11s} {msg} cave SLUG '{slug}' create failure" + DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}') + print(message) + raise + +def set_dummy_entrance(id, slug, cave, msg="DUMMY"): + '''Entrance field either missing or holds a null string instead of a filename in a cave_data file. + ''' + global entrances_xslug + try: + entrance = dummy_entrance(id, slug, msg="DUMMY") + letter = "" + entrances_xslug[slug] = entrance + ce = CaveAndEntrance.objects.update_or_create(cave = cave, entrance_letter = "", entrance = entrance) + message = f' ! Entrance Dummy setting WORKED, slug:"{slug}" cave id:"{id}" ' + DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}') + print(message) + except: + message = f' ! Entrance Dummy setting failure, slug:"{slug}" cave id :"{id}" ' + DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}') + print(message) + def do_pending_cave(k, url, area_1623): ''' default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists @@ -30,12 +76,14 @@ def do_pending_cave(k, url, area_1623): ''' default_note = f"_Survex file found in loser repo but no description in expoweb


\n" default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then " - default_note += f"
\n - search in the Expo for that year e.g. {k[0:4]} to find a " + default_note += f"

\n\n - (1) search in the survex file for the *ref to find a " + default_note += f"relevant wallet, e.g.2009#11 and read the notes image files
\n - " + default_note += f"

\n\n - (2) search in the Expo for that year e.g. {k[0:4]} to find a " default_note += f"relevant logbook entry, then
\n - " default_note += f"click on 'Edit this cave' and copy the information you find in the survex file and the logbook" - default_note += f"
\n - " + default_note += f"

\n\n - " default_note += f"When you Submit it will create a file file in expoweb/cave_data/ " - default_note += f"
\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. " + default_note += f"

\n\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. " default_note += f"and then Submit to save it. NB your entrance info will not be visible after a reboot of the server until a programmer has edited parser/caves.py (python code) to remove the cave form the [pending] list. But it won't be lost." slug = "1623-" + k @@ -65,41 +113,9 @@ def do_pending_cave(k, url, area_1623): message = f' ! {k:11s} PENDING cave create failure' DataIssue.objects.create(parser='caves', message=message) print(message) - - ent = Entrance( - name = k, - entrance_description = "Dummy entrance: auto-created when registering a new cave." + - "This file WILL NOT BE LOADED while the cave is in the pending[] list in parsers/caves.py", - marking = '?') - if ent: - ent.save() # must save to have id before foreign keys work. - try: # Now create a entrance slug ID - es = EntranceSlug.objects.update_or_create(entrance = ent, - slug = slug, primary = False) - except: - message = f" ! {k:11s} PENDING entrance create failure" - DataIssue.objects.create(parser='caves', message=message) - print(message) - - # Now we will actually write this default entrance slugfile. Yes it's naughty, but an extra - # entrance file is less hassle than a missing one! And people always forget to save the entrance file - # when they are saving the edited cave file. - - #This WILL then casue an error when the parser then tries to import this file and we get an import failure because - # the entrance exists in the db. - ent.cached_primary_slug = slug - ent.filename = slug + ".html" - ent.save() - if not ent.get_file_path().is_file(): - # don't overwrite if by some chnace it exists - # but the pre-existing file won't be parsed later until the cave is removed from the pending list - ent.writeDataFile() - else: - message = f" ! {k:11s} PENDING cave SLUG '{slug}' create failure" - DataIssue.objects.create(parser='caves', message=message) - print(message) try: + ent = dummy_entrance(k, slug, msg="PENDING") ceinsts = CaveAndEntrance.objects.update_or_create(cave = cave, entrance_letter = "", entrance = ent) for ceinst in ceinsts: if str(ceinst) == str(cave): # magic runes... why is the next value a Bool? @@ -116,10 +132,8 @@ def readcaves(): '''Reads the xml-format HTML files in the EXPOWEB repo, not from the loser repo. ''' # For those caves which do not have cave_data/1623-xxx.html XML files even though they exist and have surveys - pending = ["2007-06", "2009-02", - "2010-06", "2010-07", "2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06", - "2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888", - "2018-pf-01", "2018-pf-02"] + # should put this in a simple list which can be edited using 'Edit this file' + pending = settings.PENDING with transaction.atomic(): print(" - Deleting Caves and Entrances") @@ -153,11 +167,12 @@ def readcaves(): print(" - settings.CAVEDESCRIPTIONS: ", settings.CAVEDESCRIPTIONS) print(" - Reading Entrances from entrance descriptions xml files") for filename in next(os.walk(settings.ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files - if filename.endswith('.html'): - if Path(filename).stem[5:] in pending: - print(f'Skipping pending entrance dummy file <{filename}>') - else: - readentrance(filename) + # if filename.endswith('.html'): + # if Path(filename).stem[5:] in pending: + # print(f'Skipping pending entrance dummy file <{filename}>') + # else: + # readentrance(filename) + readentrance(filename) print(" - Reading Caves from cave descriptions xml files") for filename in next(os.walk(settings.CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files @@ -298,8 +313,8 @@ def readcave(filename): description_file = getXML(cavecontents, "description_file", maxItems = 1, context = context) url = getXML(cavecontents, "url", maxItems = 1, context = context) entrances = getXML(cavecontents, "entrance", context = context) - - if len(non_public) == 1 and len(slugs) >= 1 and len(official_name) == 1 and len(areas) >= 1 and len(kataster_code) == 1 and len(kataster_number) == 1 and len(unofficial_number) == 1 and len(explorers) == 1 and len(underground_description) == 1 and len(equipment) == 1 and len(references) == 1 and len(survey) == 1 and len(kataster_status) == 1 and len(underground_centre_line) == 1 and len(notes) == 1 and len(length) == 1 and len(depth) == 1 and len(extent) == 1 and len(survex_file) == 1 and len(description_file ) == 1 and len(url) == 1 and len(entrances) >= 1: + + if len(non_public) == 1 and len(slugs) >= 1 and len(official_name) == 1 and len(areas) >= 1 and len(kataster_code) == 1 and len(kataster_number) == 1 and len(unofficial_number) == 1 and len(explorers) == 1 and len(underground_description) == 1 and len(equipment) == 1 and len(references) == 1 and len(survey) == 1 and len(kataster_status) == 1 and len(underground_centre_line) == 1 and len(notes) == 1 and len(length) == 1 and len(depth) == 1 and len(extent) == 1 and len(survex_file) == 1 and len(description_file ) == 1 and len(url) == 1: try: c, state = Cave.objects.update_or_create(non_public = {"True": True, "False": False, "true": True, "false": False,}[non_public[0]], official_name = official_name[0], @@ -363,21 +378,28 @@ def readcave(filename): primary = False - for entrance in entrances: - eslug = getXML(entrance, "entranceslug", maxItems = 1, context = context)[0] - letter = getXML(entrance, "letter", maxItems = 1, context = context)[0] - try: - if eslug in entrances_xslug: - entrance = entrances_xslug[eslug] + if not entrances or len(entrances) < 1: + # missing entrance link in cave_data/1623-* .html file + set_dummy_entrance(slug[5:], slug, c, msg="DUMMY") + else: + for entrance in entrances: + eslug = getXML(entrance, "entranceslug", maxItems = 1, context = context)[0] + letter = getXML(entrance, "letter", maxItems = 1, context = context)[0] + if len(entrances) == 1 and not eslug: # may be empty: + set_dummy_entrance(slug[5:], slug, c, msg="DUMMY") else: - entrance = Entrance.objects.get(entranceslug__slug = eslug) - entrances_xslug[eslug] = entrance - ce = CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance) - except: - message = f' ! Entrance setting failure, slug:"{slug}" letter:"{letter}" cave:"{c}" filename:"{filename}"' - DataIssue.objects.create(parser='caves', message=message) - print(message) - + try: + if eslug in entrances_xslug: + entrance = entrances_xslug[eslug] + else: + entrance = Entrance.objects.get(entranceslug__slug = eslug) + entrances_xslug[eslug] = entrance + ce = CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance) + except: + message = f' ! Entrance setting failure, slug:"{slug}" #entrances:{len(entrances)} {entrance} letter:"{letter}" cave:"{c}" filename:"cave_data/{filename}"' + DataIssue.objects.create(parser='caves', message=message) + print(message) + if survex_file[0]: if not (Path(SURVEX_DATA) / survex_file[0]).is_file(): message = f' ! {slug:12} survex filename does not exist :LOSER:"{survex_file[0]}" in "{filename}"' diff --git a/settings.py b/settings.py index 8d34a9c..3420b60 100644 --- a/settings.py +++ b/settings.py @@ -105,6 +105,12 @@ LOGBOOK_PARSER_SETTINGS = { "1982": ("1982/log.htm", "Parseloghtml01"), } +# Caves for which survex files exist, but are not otherwise registered +PENDING = ["2007-06", "2009-02", + "2010-06", "2010-07", "2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06", + "2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888", + "2018-pf-01", "2018-pf-02"] + APPEND_SLASH = False # never relevant because we have urls that match unknown files and produce an 'edit this page' response SMART_APPEND_SLASH = True