forked from expo/troggle
missing entrance .html file now handled differently
This commit is contained in:
Philip Sargent
parent
72fa8a5883
commit
9e11c0814e
144
parsers/caves.py
144
parsers/caves.py
@ -23,6 +23,52 @@ entrances_xslug = {}
|
||||
caves_xslug = {}
|
||||
areas_xslug = {}
|
||||
|
||||
def dummy_entrance(k, slug, msg="DUMMY"):
|
||||
'''Returns an empty entrance object for either a PENDING cave or a DUMMY entrance if
|
||||
user forgot to provide one when creating the cave
|
||||
'''
|
||||
ent = Entrance(
|
||||
name = k,
|
||||
entrance_description = "Dummy entrance: auto-created when registering a new cave " +
|
||||
"and you forgot to create an entrance for it. Click on 'Edit' to enter the correct data, then 'Submit'.",
|
||||
marking = '?')
|
||||
if ent:
|
||||
ent.save() # must save to have id before foreign keys work.
|
||||
try: # Now create a entrance slug ID
|
||||
es = EntranceSlug.objects.update_or_create(entrance = ent,
|
||||
slug = slug, primary = False)
|
||||
except:
|
||||
message = f" ! {k:11s} {msg} entrance create failure"
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}')
|
||||
print(message)
|
||||
|
||||
ent.cached_primary_slug = slug
|
||||
ent.filename = slug + ".html"
|
||||
ent.save()
|
||||
return ent
|
||||
else:
|
||||
message = f" ! {k:11s} {msg} cave SLUG '{slug}' create failure"
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}')
|
||||
print(message)
|
||||
raise
|
||||
|
||||
def set_dummy_entrance(id, slug, cave, msg="DUMMY"):
|
||||
'''Entrance field either missing or holds a null string instead of a filename in a cave_data file.
|
||||
'''
|
||||
global entrances_xslug
|
||||
try:
|
||||
entrance = dummy_entrance(id, slug, msg="DUMMY")
|
||||
letter = ""
|
||||
entrances_xslug[slug] = entrance
|
||||
ce = CaveAndEntrance.objects.update_or_create(cave = cave, entrance_letter = "", entrance = entrance)
|
||||
message = f' ! Entrance Dummy setting WORKED, slug:"{slug}" cave id:"{id}" '
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}')
|
||||
print(message)
|
||||
except:
|
||||
message = f' ! Entrance Dummy setting failure, slug:"{slug}" cave id :"{id}" '
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}')
|
||||
print(message)
|
||||
|
||||
def do_pending_cave(k, url, area_1623):
|
||||
'''
|
||||
default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists
|
||||
@ -30,12 +76,14 @@ def do_pending_cave(k, url, area_1623):
|
||||
'''
|
||||
default_note = f"_Survex file found in loser repo but no description in expoweb <br><br><br>\n"
|
||||
default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then "
|
||||
default_note += f"<br>\n - search in the Expo for that year e.g. <a href='/expedition/{k[0:4]}'>{k[0:4]}</a> to find a "
|
||||
default_note += f"<br><br>\n\n - (1) search in the survex file for the *ref to find a "
|
||||
default_note += f"relevant wallet, e.g.<a href='/survey_scans/2009%252311/'>2009#11</a> and read the notes image files <br>\n - "
|
||||
default_note += f"<br><br>\n\n - (2) search in the Expo for that year e.g. <a href='/expedition/{k[0:4]}'>{k[0:4]}</a> to find a "
|
||||
default_note += f"relevant logbook entry, then <br>\n - "
|
||||
default_note += f"click on 'Edit this cave' and copy the information you find in the survex file and the logbook"
|
||||
default_note += f"<br>\n - "
|
||||
default_note += f"<br><br>\n\n - "
|
||||
default_note += f"When you Submit it will create a file file in expoweb/cave_data/ "
|
||||
default_note += f"<br>\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. "
|
||||
default_note += f"<br><br>\n\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. "
|
||||
default_note += f"and then Submit to save it. NB your entrance info will not be visible after a reboot of the server until a programmer has edited parser/caves.py (python code) to remove the cave form the [pending] list. But it won't be lost."
|
||||
|
||||
slug = "1623-" + k
|
||||
@ -65,41 +113,9 @@ def do_pending_cave(k, url, area_1623):
|
||||
message = f' ! {k:11s} PENDING cave create failure'
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
print(message)
|
||||
|
||||
ent = Entrance(
|
||||
name = k,
|
||||
entrance_description = "Dummy entrance: auto-created when registering a new cave." +
|
||||
"This file WILL NOT BE LOADED while the cave is in the pending[] list in parsers/caves.py",
|
||||
marking = '?')
|
||||
if ent:
|
||||
ent.save() # must save to have id before foreign keys work.
|
||||
try: # Now create a entrance slug ID
|
||||
es = EntranceSlug.objects.update_or_create(entrance = ent,
|
||||
slug = slug, primary = False)
|
||||
except:
|
||||
message = f" ! {k:11s} PENDING entrance create failure"
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
print(message)
|
||||
|
||||
# Now we will actually write this default entrance slugfile. Yes it's naughty, but an extra
|
||||
# entrance file is less hassle than a missing one! And people always forget to save the entrance file
|
||||
# when they are saving the edited cave file.
|
||||
|
||||
#This WILL then casue an error when the parser then tries to import this file and we get an import failure because
|
||||
# the entrance exists in the db.
|
||||
ent.cached_primary_slug = slug
|
||||
ent.filename = slug + ".html"
|
||||
ent.save()
|
||||
if not ent.get_file_path().is_file():
|
||||
# don't overwrite if by some chnace it exists
|
||||
# but the pre-existing file won't be parsed later until the cave is removed from the pending list
|
||||
ent.writeDataFile()
|
||||
else:
|
||||
message = f" ! {k:11s} PENDING cave SLUG '{slug}' create failure"
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
print(message)
|
||||
|
||||
try:
|
||||
ent = dummy_entrance(k, slug, msg="PENDING")
|
||||
ceinsts = CaveAndEntrance.objects.update_or_create(cave = cave, entrance_letter = "", entrance = ent)
|
||||
for ceinst in ceinsts:
|
||||
if str(ceinst) == str(cave): # magic runes... why is the next value a Bool?
|
||||
@ -116,10 +132,8 @@ def readcaves():
|
||||
'''Reads the xml-format HTML files in the EXPOWEB repo, not from the loser repo.
|
||||
'''
|
||||
# For those caves which do not have cave_data/1623-xxx.html XML files even though they exist and have surveys
|
||||
pending = ["2007-06", "2009-02",
|
||||
"2010-06", "2010-07", "2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06",
|
||||
"2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888",
|
||||
"2018-pf-01", "2018-pf-02"]
|
||||
# should put this in a simple list which can be edited using 'Edit this file'
|
||||
pending = settings.PENDING
|
||||
|
||||
with transaction.atomic():
|
||||
print(" - Deleting Caves and Entrances")
|
||||
@ -153,11 +167,12 @@ def readcaves():
|
||||
print(" - settings.CAVEDESCRIPTIONS: ", settings.CAVEDESCRIPTIONS)
|
||||
print(" - Reading Entrances from entrance descriptions xml files")
|
||||
for filename in next(os.walk(settings.ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
|
||||
if filename.endswith('.html'):
|
||||
if Path(filename).stem[5:] in pending:
|
||||
print(f'Skipping pending entrance dummy file <{filename}>')
|
||||
else:
|
||||
readentrance(filename)
|
||||
# if filename.endswith('.html'):
|
||||
# if Path(filename).stem[5:] in pending:
|
||||
# print(f'Skipping pending entrance dummy file <{filename}>')
|
||||
# else:
|
||||
# readentrance(filename)
|
||||
readentrance(filename)
|
||||
|
||||
print(" - Reading Caves from cave descriptions xml files")
|
||||
for filename in next(os.walk(settings.CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
|
||||
@ -298,8 +313,8 @@ def readcave(filename):
|
||||
description_file = getXML(cavecontents, "description_file", maxItems = 1, context = context)
|
||||
url = getXML(cavecontents, "url", maxItems = 1, context = context)
|
||||
entrances = getXML(cavecontents, "entrance", context = context)
|
||||
|
||||
if len(non_public) == 1 and len(slugs) >= 1 and len(official_name) == 1 and len(areas) >= 1 and len(kataster_code) == 1 and len(kataster_number) == 1 and len(unofficial_number) == 1 and len(explorers) == 1 and len(underground_description) == 1 and len(equipment) == 1 and len(references) == 1 and len(survey) == 1 and len(kataster_status) == 1 and len(underground_centre_line) == 1 and len(notes) == 1 and len(length) == 1 and len(depth) == 1 and len(extent) == 1 and len(survex_file) == 1 and len(description_file ) == 1 and len(url) == 1 and len(entrances) >= 1:
|
||||
|
||||
if len(non_public) == 1 and len(slugs) >= 1 and len(official_name) == 1 and len(areas) >= 1 and len(kataster_code) == 1 and len(kataster_number) == 1 and len(unofficial_number) == 1 and len(explorers) == 1 and len(underground_description) == 1 and len(equipment) == 1 and len(references) == 1 and len(survey) == 1 and len(kataster_status) == 1 and len(underground_centre_line) == 1 and len(notes) == 1 and len(length) == 1 and len(depth) == 1 and len(extent) == 1 and len(survex_file) == 1 and len(description_file ) == 1 and len(url) == 1:
|
||||
try:
|
||||
c, state = Cave.objects.update_or_create(non_public = {"True": True, "False": False, "true": True, "false": False,}[non_public[0]],
|
||||
official_name = official_name[0],
|
||||
@ -363,21 +378,28 @@ def readcave(filename):
|
||||
|
||||
primary = False
|
||||
|
||||
for entrance in entrances:
|
||||
eslug = getXML(entrance, "entranceslug", maxItems = 1, context = context)[0]
|
||||
letter = getXML(entrance, "letter", maxItems = 1, context = context)[0]
|
||||
try:
|
||||
if eslug in entrances_xslug:
|
||||
entrance = entrances_xslug[eslug]
|
||||
if not entrances or len(entrances) < 1:
|
||||
# missing entrance link in cave_data/1623-* .html file
|
||||
set_dummy_entrance(slug[5:], slug, c, msg="DUMMY")
|
||||
else:
|
||||
for entrance in entrances:
|
||||
eslug = getXML(entrance, "entranceslug", maxItems = 1, context = context)[0]
|
||||
letter = getXML(entrance, "letter", maxItems = 1, context = context)[0]
|
||||
if len(entrances) == 1 and not eslug: # may be empty: <entranceslug></entranceslug>
|
||||
set_dummy_entrance(slug[5:], slug, c, msg="DUMMY")
|
||||
else:
|
||||
entrance = Entrance.objects.get(entranceslug__slug = eslug)
|
||||
entrances_xslug[eslug] = entrance
|
||||
ce = CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance)
|
||||
except:
|
||||
message = f' ! Entrance setting failure, slug:"{slug}" letter:"{letter}" cave:"{c}" filename:"{filename}"'
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
print(message)
|
||||
|
||||
try:
|
||||
if eslug in entrances_xslug:
|
||||
entrance = entrances_xslug[eslug]
|
||||
else:
|
||||
entrance = Entrance.objects.get(entranceslug__slug = eslug)
|
||||
entrances_xslug[eslug] = entrance
|
||||
ce = CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance)
|
||||
except:
|
||||
message = f' ! Entrance setting failure, slug:"{slug}" #entrances:{len(entrances)} {entrance} letter:"{letter}" cave:"{c}" filename:"cave_data/{filename}"'
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
print(message)
|
||||
|
||||
if survex_file[0]:
|
||||
if not (Path(SURVEX_DATA) / survex_file[0]).is_file():
|
||||
message = f' ! {slug:12} survex filename does not exist :LOSER:"{survex_file[0]}" in "{filename}"'
|
||||
|
||||
@ -105,6 +105,12 @@ LOGBOOK_PARSER_SETTINGS = {
|
||||
"1982": ("1982/log.htm", "Parseloghtml01"),
|
||||
}
|
||||
|
||||
# Caves for which survex files exist, but are not otherwise registered
|
||||
PENDING = ["2007-06", "2009-02",
|
||||
"2010-06", "2010-07", "2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06",
|
||||
"2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888",
|
||||
"2018-pf-01", "2018-pf-02"]
|
||||
|
||||
APPEND_SLASH = False # never relevant because we have urls that match unknown files and produce an 'edit this page' response
|
||||
SMART_APPEND_SLASH = True
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user