mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2025-12-14 19:47:12 +00:00
reformatted using black
This commit is contained in:
600
parsers/caves.py
600
parsers/caves.py
@@ -6,49 +6,48 @@ from pathlib import Path
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
|
||||
from troggle.core.models.caves import (Area, Cave, CaveAndEntrance, CaveSlug,
|
||||
Entrance, EntranceSlug, GetCaveLookup)
|
||||
from troggle.core.models.caves import Area, Cave, CaveAndEntrance, CaveSlug, Entrance, EntranceSlug, GetCaveLookup
|
||||
from troggle.core.models.troggle import DataIssue
|
||||
from troggle.settings import (CAVEDESCRIPTIONS, ENTRANCEDESCRIPTIONS, EXPOWEB,
|
||||
SURVEX_DATA)
|
||||
from troggle.settings import CAVEDESCRIPTIONS, ENTRANCEDESCRIPTIONS, EXPOWEB, SURVEX_DATA
|
||||
|
||||
'''Reads all the cave description data by parsing the xml files (stored as e.g. :EXPOWEB:/cave_data/1623-161.html )
|
||||
"""Reads all the cave description data by parsing the xml files (stored as e.g. :EXPOWEB:/cave_data/1623-161.html )
|
||||
and creating the various Cave, Entrance and necessary Area objects.
|
||||
|
||||
This is the first import that happens after the database is reinitialised.
|
||||
So is the first thing that creates tables.
|
||||
|
||||
'''
|
||||
"""
|
||||
|
||||
todo='''
|
||||
todo = """
|
||||
- Cannot use Edit This Page for pendingcaves.txt_edit as Edit This Page is expecting an html file.
|
||||
So we will need a separate file-editing capability just for this configuration file ?!
|
||||
|
||||
- crashes on MariaDB in databasereset.py on server when deleting Caves and complains Area needs a non null parent, But this is not true.
|
||||
The only solution we have found is to let it crash, then stop and restart MariaDB (requires a logon able to sudo)
|
||||
and then restart the databasereset.py again. (status as of July 2022)
|
||||
'''
|
||||
"""
|
||||
entrances_xslug = {}
|
||||
caves_xslug = {}
|
||||
areas_xslug = {}
|
||||
|
||||
|
||||
def dummy_entrance(k, slug, msg="DUMMY"):
|
||||
'''Returns an empty entrance object for either a PENDING cave or a DUMMY entrance if
|
||||
"""Returns an empty entrance object for either a PENDING cave or a DUMMY entrance if
|
||||
user forgot to provide one when creating the cave
|
||||
'''
|
||||
"""
|
||||
ent = Entrance(
|
||||
name = k,
|
||||
entrance_description = "Dummy entrance: auto-created when registering a new cave " +
|
||||
"and you forgot to create an entrance for it. Click on 'Edit' to enter the correct data, then 'Submit'.",
|
||||
marking = '?')
|
||||
name=k,
|
||||
entrance_description="Dummy entrance: auto-created when registering a new cave "
|
||||
+ "and you forgot to create an entrance for it. Click on 'Edit' to enter the correct data, then 'Submit'.",
|
||||
marking="?",
|
||||
)
|
||||
if ent:
|
||||
ent.save() # must save to have id before foreign keys work.
|
||||
try: # Now create a entrance slug ID
|
||||
es = EntranceSlug(entrance = ent,
|
||||
slug = slug, primary = False)
|
||||
ent.save() # must save to have id before foreign keys work.
|
||||
try: # Now create a entrance slug ID
|
||||
es = EntranceSlug(entrance=ent, slug=slug, primary=False)
|
||||
except:
|
||||
message = f" ! {k:11s} {msg}-{slug} entrance create failure"
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'{slug}')
|
||||
DataIssue.objects.create(parser="caves", message=message, url=f"{slug}")
|
||||
print(message)
|
||||
|
||||
ent.cached_primary_slug = slug
|
||||
@@ -57,46 +56,48 @@ def dummy_entrance(k, slug, msg="DUMMY"):
|
||||
return ent
|
||||
else:
|
||||
message = f" ! {k:11s} {msg} cave SLUG '{slug}' create failure"
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'{slug}')
|
||||
DataIssue.objects.create(parser="caves", message=message, url=f"{slug}")
|
||||
print(message)
|
||||
raise
|
||||
raise
|
||||
|
||||
|
||||
def set_dummy_entrance(id, slug, cave, msg="DUMMY"):
|
||||
'''Entrance field either missing or holds a null string instead of a filename in a cave_data file.
|
||||
'''
|
||||
"""Entrance field either missing or holds a null string instead of a filename in a cave_data file."""
|
||||
global entrances_xslug
|
||||
try:
|
||||
entrance = dummy_entrance(id, slug, msg="DUMMY")
|
||||
letter = ""
|
||||
entrances_xslug[slug] = entrance
|
||||
ce = CaveAndEntrance.objects.update_or_create(cave = cave, entrance_letter = "", entrance = entrance)
|
||||
message = f' ! Warning: Dummy Entrance successfully set for entrance {id} on cave {cave}'
|
||||
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'{cave.url}')
|
||||
ce = CaveAndEntrance.objects.update_or_create(cave=cave, entrance_letter="", entrance=entrance)
|
||||
message = f" ! Warning: Dummy Entrance successfully set for entrance {id} on cave {cave}"
|
||||
|
||||
DataIssue.objects.create(parser="caves", message=message, url=f"{cave.url}")
|
||||
print(message)
|
||||
except:
|
||||
#raise
|
||||
# raise
|
||||
message = f' ! Entrance Dummy setting failure, slug:"{slug}" cave id :"{id}" '
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'{cave.url}')
|
||||
DataIssue.objects.create(parser="caves", message=message, url=f"{cave.url}")
|
||||
print(message)
|
||||
|
||||
|
||||
|
||||
def do_pending_cave(k, url, area):
|
||||
'''
|
||||
default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists
|
||||
in expoweb/cave_data/1623-"k".html
|
||||
|
||||
"""
|
||||
default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists
|
||||
in expoweb/cave_data/1623-"k".html
|
||||
|
||||
Note that at this point in importing the data we have not yet seen the survex files, so we can't
|
||||
look inside the relevant survex file to find the year and so we con't provide helpful links.
|
||||
'''
|
||||
"""
|
||||
|
||||
def get_survex_file(k):
|
||||
'''Guesses at and finds a survex file for this pending cave.
|
||||
"""Guesses at and finds a survex file for this pending cave.
|
||||
Convoluted. Needs rewriting
|
||||
'''
|
||||
"""
|
||||
if k[0:3] == "162":
|
||||
id = Path(k[5:])
|
||||
else:
|
||||
id = Path(k)
|
||||
|
||||
|
||||
survex_file = f"caves-{area.short_name}/{id}/{id}.svx"
|
||||
if Path(settings.SURVEX_DATA, survex_file).is_file():
|
||||
return survex_file
|
||||
@@ -104,7 +105,7 @@ def do_pending_cave(k, url, area):
|
||||
survex_file = f"caves-{area.short_name}/{id}.svx"
|
||||
if Path(settings.SURVEX_DATA, survex_file).is_file():
|
||||
return survex_file
|
||||
|
||||
|
||||
survex_file = ""
|
||||
d = Path(settings.SURVEX_DATA, f"caves-{area.short_name}/{id}")
|
||||
if d.is_dir():
|
||||
@@ -113,8 +114,8 @@ def do_pending_cave(k, url, area):
|
||||
for f in dir:
|
||||
if f.suffix == ".svx":
|
||||
survex_file = f.relative_to(settings.SURVEX_DATA)
|
||||
chk = min(5, len(f.name)-1)
|
||||
if str(f.name)[:chk].lower() == str(id.name)[:chk].lower(): # bodge which mostly works
|
||||
chk = min(5, len(f.name) - 1)
|
||||
if str(f.name)[:chk].lower() == str(id.name)[:chk].lower(): # bodge which mostly works
|
||||
prime_suspect = survex_file
|
||||
if prime_suspect:
|
||||
survex_file = prime_suspect
|
||||
@@ -124,71 +125,81 @@ def do_pending_cave(k, url, area):
|
||||
return survex_file
|
||||
|
||||
slug = k
|
||||
|
||||
|
||||
g = GetCaveLookup()
|
||||
with transaction.atomic():
|
||||
if slug in g:
|
||||
message = f" ! {k:18} cave listed in pendingcaves.txt already exists."
|
||||
DataIssue.objects.create(parser='caves', message=message, url=url)
|
||||
DataIssue.objects.create(parser="caves", message=message, url=url)
|
||||
print(message)
|
||||
return
|
||||
|
||||
|
||||
|
||||
default_note = f"_Survex file found in loser repo but no description in expoweb <br><br><br>\n"
|
||||
default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then "
|
||||
default_note += f"<br><br>\n\n - (0) look in the <a href=\"/noinfo/cave-number-index\">cave number index</a> for notes on this cave, "
|
||||
default_note += f"<br><br>\n\n - (1) search in the survex file for the *ref to find a "
|
||||
default_note += f"relevant wallet, e.g.<a href='/survey_scans/2009%252311/'>2009#11</a> and read the notes image files <br>\n - "
|
||||
default_note += f"<br><br>\n\n - (2) search in the Expo for that year e.g. <a href='/expedition/2009'>2009</a> to find a "
|
||||
default_note += f"relevant logbook entry, remember that the date may have been recorded incorrectly, "
|
||||
default_note += f"so check for trips i.e. logbook entries involving the same people as were listed in the survex file, "
|
||||
default_note += f"and you should also check the scanned copy of the logbook (linked from each logbook entry page) "
|
||||
default_note += f"just in case a vital trip was not transcribed, then <br>\n - "
|
||||
default_note += f"click on 'Edit this cave' and copy the information you find in the survex file and the logbook"
|
||||
default_note += f"and delete all the text in the 'Notes' section - which is the text you are reading now."
|
||||
default_note += f"<br><br>\n\n - Only two fields on this form are essential. "
|
||||
default_note = f"_Survex file found in loser repo but no description in expoweb <br><br><br>\n"
|
||||
default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then "
|
||||
default_note += f'<br><br>\n\n - (0) look in the <a href="/noinfo/cave-number-index">cave number index</a> for notes on this cave, '
|
||||
default_note += f"<br><br>\n\n - (1) search in the survex file for the *ref to find a "
|
||||
default_note += f"relevant wallet, e.g.<a href='/survey_scans/2009%252311/'>2009#11</a> and read the notes image files <br>\n - "
|
||||
default_note += (
|
||||
f"<br><br>\n\n - (2) search in the Expo for that year e.g. <a href='/expedition/2009'>2009</a> to find a "
|
||||
)
|
||||
default_note += f"relevant logbook entry, remember that the date may have been recorded incorrectly, "
|
||||
default_note += (
|
||||
f"so check for trips i.e. logbook entries involving the same people as were listed in the survex file, "
|
||||
)
|
||||
default_note += (
|
||||
f"and you should also check the scanned copy of the logbook (linked from each logbook entry page) "
|
||||
)
|
||||
default_note += f"just in case a vital trip was not transcribed, then <br>\n - "
|
||||
default_note += (
|
||||
f"click on 'Edit this cave' and copy the information you find in the survex file and the logbook"
|
||||
)
|
||||
default_note += f"and delete all the text in the 'Notes' section - which is the text you are reading now."
|
||||
default_note += f"<br><br>\n\n - Only two fields on this form are essential. "
|
||||
default_note += f"Documentation of all the fields on 'Edit this cave' form is in <a href='/handbook/survey/caveentryfields.html'>handbook/survey/caveentryfields</a>"
|
||||
default_note += f"<br><br>\n\n - "
|
||||
default_note += f"You will also need to create a new entrance from the 'Edit this cave' page. Ignore the existing dummy one, it will evaporate on the next full import."
|
||||
default_note += f"<br><br>\n\n - "
|
||||
default_note += f"When you Submit it will create a new file in expoweb/cave_data/ "
|
||||
default_note += f"<br><br>\n\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. "
|
||||
default_note += f"<br><br>\n\n - "
|
||||
default_note += f"You will also need to create a new entrance from the 'Edit this cave' page. Ignore the existing dummy one, it will evaporate on the next full import."
|
||||
default_note += f"<br><br>\n\n - "
|
||||
default_note += f"When you Submit it will create a new file in expoweb/cave_data/ "
|
||||
default_note += (
|
||||
f"<br><br>\n\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. "
|
||||
)
|
||||
default_note += f"and then Submit to save it (if you forget to do this, a dummy entrance will be created for your new cave description)."
|
||||
default_note += f"<br><br>\n\n - Finally, you need to find a nerd to edit the file '<var>expoweb/cave_data/pending.txt</var>' "
|
||||
default_note += f"to remove the line <br><var>{slug}</var><br> as it is no longer 'pending' but 'done. Well Done."
|
||||
default_note += f"<br><br>\n\n - Finally, you need to find a nerd to edit the file '<var>expoweb/cave_data/pending.txt</var>' "
|
||||
default_note += (
|
||||
f"to remove the line <br><var>{slug}</var><br> as it is no longer 'pending' but 'done. Well Done."
|
||||
)
|
||||
|
||||
survex_file = get_survex_file(k)
|
||||
|
||||
|
||||
cave = Cave(
|
||||
unofficial_number = k,
|
||||
underground_description = "Pending cave write-up - creating as empty object. No XML file available yet.",
|
||||
survex_file = survex_file,
|
||||
url = url,
|
||||
notes = default_note)
|
||||
unofficial_number=k,
|
||||
underground_description="Pending cave write-up - creating as empty object. No XML file available yet.",
|
||||
survex_file=survex_file,
|
||||
url=url,
|
||||
notes=default_note,
|
||||
)
|
||||
if cave:
|
||||
cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key.
|
||||
cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key.
|
||||
cave.area.add(area)
|
||||
cave.save()
|
||||
message = f" ! {k:18} {cave.underground_description} url: {url}"
|
||||
DataIssue.objects.create(parser='caves', message=message, url=url)
|
||||
DataIssue.objects.create(parser="caves", message=message, url=url)
|
||||
print(message)
|
||||
|
||||
try: # Now create a cave slug ID
|
||||
cs = CaveSlug.objects.update_or_create(cave = cave,
|
||||
slug = slug, primary = False)
|
||||
|
||||
try: # Now create a cave slug ID
|
||||
cs = CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False)
|
||||
except:
|
||||
message = f" ! {k:11s} PENDING cave SLUG create failure"
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
else:
|
||||
message = f' ! {k:11s} PENDING cave create failure'
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
message = f" ! {k:11s} PENDING cave create failure"
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
|
||||
try:
|
||||
ent = dummy_entrance(k, slug, msg="PENDING")
|
||||
ceinsts = CaveAndEntrance.objects.update_or_create(cave = cave, entrance_letter = "", entrance = ent)
|
||||
ceinsts = CaveAndEntrance.objects.update_or_create(cave=cave, entrance_letter="", entrance=ent)
|
||||
for ceinst in ceinsts:
|
||||
if str(ceinst) == str(cave): # magic runes... why is the next value a Bool?
|
||||
ceinst.cave = cave
|
||||
@@ -196,15 +207,14 @@ def do_pending_cave(k, url, area):
|
||||
break
|
||||
except:
|
||||
message = f" ! {k:11s} PENDING entrance + cave UNION create failure '{cave}' [{ent}]"
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
|
||||
|
||||
|
||||
def readentrance(filename):
|
||||
'''Reads an enrance description from the .html file
|
||||
"""Reads an enrance description from the .html file
|
||||
Convoluted. Sorry.This is as I inherited it and I haven't fiddled with it. Needs rewriting
|
||||
'''
|
||||
"""
|
||||
global entrances_xslug
|
||||
global caves_xslug
|
||||
global areas_xslug
|
||||
@@ -213,181 +223,214 @@ def readentrance(filename):
|
||||
with open(os.path.join(ENTRANCEDESCRIPTIONS, filename)) as f:
|
||||
contents = f.read()
|
||||
context = filename
|
||||
#print("Reading file ENTRANCE {} / {}".format(ENTRANCEDESCRIPTIONS, filename))
|
||||
entrancecontentslist = getXML(contents, "entrance", maxItems = 1, context = context)
|
||||
# print("Reading file ENTRANCE {} / {}".format(ENTRANCEDESCRIPTIONS, filename))
|
||||
entrancecontentslist = getXML(contents, "entrance", maxItems=1, context=context)
|
||||
if len(entrancecontentslist) != 1:
|
||||
message = f'! BAD ENTRANCE at "{filename}"'
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
else:
|
||||
entrancecontents = entrancecontentslist[0]
|
||||
non_public = getXML(entrancecontents, "non_public", maxItems = 1, context = context)
|
||||
name = getXML(entrancecontents, "name", maxItems = 1, context = context)
|
||||
slugs = getXML(entrancecontents, "slug", context = context)
|
||||
entrance_description = getXML(entrancecontents, "entrance_description", maxItems = 1, context = context)
|
||||
explorers = getXML(entrancecontents, "explorers", maxItems = 1, context = context)
|
||||
map_description = getXML(entrancecontents, "map_description", maxItems = 1, context = context)
|
||||
location_description = getXML(entrancecontents, "location_description", maxItems = 1, context = context)
|
||||
lastvisit = getXML(entrancecontents, "last visit date", maxItems = 1, minItems = 0, context = context)
|
||||
approach = getXML(entrancecontents, "approach", maxItems = 1, context = context)
|
||||
underground_description = getXML(entrancecontents, "underground_description", maxItems = 1, context = context)
|
||||
photo = getXML(entrancecontents, "photo", maxItems = 1, context = context)
|
||||
marking = getXML(entrancecontents, "marking", maxItems = 1, context = context)
|
||||
marking_comment = getXML(entrancecontents, "marking_comment", maxItems = 1, context = context)
|
||||
findability = getXML(entrancecontents, "findability", maxItems = 1, context = context)
|
||||
findability_description = getXML(entrancecontents, "findability_description", maxItems = 1, context = context)
|
||||
alt = getXML(entrancecontents, "alt", maxItems = 1, context = context)
|
||||
northing = getXML(entrancecontents, "northing", maxItems = 1, context = context)
|
||||
easting = getXML(entrancecontents, "easting", maxItems = 1, context = context)
|
||||
tag_station = getXML(entrancecontents, "tag_station", maxItems = 1, context = context)
|
||||
exact_station = getXML(entrancecontents, "exact_station", maxItems = 1, context = context)
|
||||
other_station = getXML(entrancecontents, "other_station", maxItems = 1, context = context)
|
||||
other_description = getXML(entrancecontents, "other_description", maxItems = 1, context = context)
|
||||
bearings = getXML(entrancecontents, "bearings", maxItems = 1, context = context)
|
||||
url = getXML(entrancecontents, "url", maxItems = 1, context = context)
|
||||
#if len(non_public) == 1 and len(slugs) >= 1 and len(name) >= 1 and len(entrance_description) == 1 and len(explorers) == 1 and len(map_description) == 1 and len(location_description) == 1 and len(lastvisit) == 1 and len(approach) == 1 and len(underground_description) == 1 and len(marking) == 1 and len(marking_comment) == 1 and len(findability) == 1 and len(findability_description) == 1 and len(alt) == 1 and len(northing) == 1 and len(easting) == 1 and len(tag_station) == 1 and len(exact_station) == 1 and len(other_station) == 1 and len(other_description) == 1 and len(bearings) == 1 and len(url) == 1:
|
||||
e, state = Entrance.objects.update_or_create(name = name[0],
|
||||
non_public = {"True": True, "False": False, "true": True, "false": False,}[non_public[0]],
|
||||
entrance_description = entrance_description[0],
|
||||
explorers = explorers[0],
|
||||
map_description = map_description[0],
|
||||
location_description = location_description[0],
|
||||
lastvisit = lastvisit[0],
|
||||
approach = approach[0],
|
||||
underground_description = underground_description[0],
|
||||
photo = photo[0],
|
||||
marking = marking[0],
|
||||
marking_comment = marking_comment[0],
|
||||
findability = findability[0],
|
||||
findability_description = findability_description[0],
|
||||
alt = alt[0],
|
||||
northing = northing[0],
|
||||
easting = easting[0],
|
||||
tag_station = tag_station[0],
|
||||
exact_station = exact_station[0],
|
||||
other_station = other_station[0],
|
||||
other_description = other_description[0],
|
||||
bearings = bearings[0],
|
||||
url = url[0],
|
||||
filename = filename,
|
||||
cached_primary_slug = slugs[0])
|
||||
non_public = getXML(entrancecontents, "non_public", maxItems=1, context=context)
|
||||
name = getXML(entrancecontents, "name", maxItems=1, context=context)
|
||||
slugs = getXML(entrancecontents, "slug", context=context)
|
||||
entrance_description = getXML(entrancecontents, "entrance_description", maxItems=1, context=context)
|
||||
explorers = getXML(entrancecontents, "explorers", maxItems=1, context=context)
|
||||
map_description = getXML(entrancecontents, "map_description", maxItems=1, context=context)
|
||||
location_description = getXML(entrancecontents, "location_description", maxItems=1, context=context)
|
||||
lastvisit = getXML(entrancecontents, "last visit date", maxItems=1, minItems=0, context=context)
|
||||
approach = getXML(entrancecontents, "approach", maxItems=1, context=context)
|
||||
underground_description = getXML(entrancecontents, "underground_description", maxItems=1, context=context)
|
||||
photo = getXML(entrancecontents, "photo", maxItems=1, context=context)
|
||||
marking = getXML(entrancecontents, "marking", maxItems=1, context=context)
|
||||
marking_comment = getXML(entrancecontents, "marking_comment", maxItems=1, context=context)
|
||||
findability = getXML(entrancecontents, "findability", maxItems=1, context=context)
|
||||
findability_description = getXML(entrancecontents, "findability_description", maxItems=1, context=context)
|
||||
alt = getXML(entrancecontents, "alt", maxItems=1, context=context)
|
||||
northing = getXML(entrancecontents, "northing", maxItems=1, context=context)
|
||||
easting = getXML(entrancecontents, "easting", maxItems=1, context=context)
|
||||
tag_station = getXML(entrancecontents, "tag_station", maxItems=1, context=context)
|
||||
exact_station = getXML(entrancecontents, "exact_station", maxItems=1, context=context)
|
||||
other_station = getXML(entrancecontents, "other_station", maxItems=1, context=context)
|
||||
other_description = getXML(entrancecontents, "other_description", maxItems=1, context=context)
|
||||
bearings = getXML(entrancecontents, "bearings", maxItems=1, context=context)
|
||||
url = getXML(entrancecontents, "url", maxItems=1, context=context)
|
||||
# if len(non_public) == 1 and len(slugs) >= 1 and len(name) >= 1 and len(entrance_description) == 1 and len(explorers) == 1 and len(map_description) == 1 and len(location_description) == 1 and len(lastvisit) == 1 and len(approach) == 1 and len(underground_description) == 1 and len(marking) == 1 and len(marking_comment) == 1 and len(findability) == 1 and len(findability_description) == 1 and len(alt) == 1 and len(northing) == 1 and len(easting) == 1 and len(tag_station) == 1 and len(exact_station) == 1 and len(other_station) == 1 and len(other_description) == 1 and len(bearings) == 1 and len(url) == 1:
|
||||
e, state = Entrance.objects.update_or_create(
|
||||
name=name[0],
|
||||
non_public={
|
||||
"True": True,
|
||||
"False": False,
|
||||
"true": True,
|
||||
"false": False,
|
||||
}[non_public[0]],
|
||||
entrance_description=entrance_description[0],
|
||||
explorers=explorers[0],
|
||||
map_description=map_description[0],
|
||||
location_description=location_description[0],
|
||||
lastvisit=lastvisit[0],
|
||||
approach=approach[0],
|
||||
underground_description=underground_description[0],
|
||||
photo=photo[0],
|
||||
marking=marking[0],
|
||||
marking_comment=marking_comment[0],
|
||||
findability=findability[0],
|
||||
findability_description=findability_description[0],
|
||||
alt=alt[0],
|
||||
northing=northing[0],
|
||||
easting=easting[0],
|
||||
tag_station=tag_station[0],
|
||||
exact_station=exact_station[0],
|
||||
other_station=other_station[0],
|
||||
other_description=other_description[0],
|
||||
bearings=bearings[0],
|
||||
url=url[0],
|
||||
filename=filename,
|
||||
cached_primary_slug=slugs[0],
|
||||
)
|
||||
primary = True
|
||||
for slug in slugs:
|
||||
#print("entrance slug:{} filename:{}".format(slug, filename))
|
||||
# print("entrance slug:{} filename:{}".format(slug, filename))
|
||||
try:
|
||||
cs = EntranceSlug.objects.update_or_create(entrance = e,
|
||||
slug = slug,
|
||||
primary = primary)
|
||||
cs = EntranceSlug.objects.update_or_create(entrance=e, slug=slug, primary=primary)
|
||||
except:
|
||||
# need to cope with duplicates
|
||||
message = f" ! FAILED to get precisely one ENTRANCE when updating using: cave_entrance/{filename}"
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/')
|
||||
kents = EntranceSlug.objects.all().filter(entrance = e,
|
||||
slug = slug,
|
||||
primary = primary)
|
||||
DataIssue.objects.create(parser="caves", message=message, url=f"/cave/{slug}/edit/")
|
||||
kents = EntranceSlug.objects.all().filter(entrance=e, slug=slug, primary=primary)
|
||||
for k in kents:
|
||||
message = " ! - DUPLICATE in db. entrance:"+ str(k.entrance) + ", slug:" + str(k.slug())
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/')
|
||||
message = " ! - DUPLICATE in db. entrance:" + str(k.entrance) + ", slug:" + str(k.slug())
|
||||
DataIssue.objects.create(parser="caves", message=message, url=f"/cave/{slug}/edit/")
|
||||
print(message)
|
||||
for k in kents:
|
||||
if k.slug() != None:
|
||||
print(" ! - OVERWRITING this one: slug:"+ str(k.slug()))
|
||||
print(" ! - OVERWRITING this one: slug:" + str(k.slug()))
|
||||
k.notes = "DUPLICATE entrance found on import. Please fix\n" + k.notes
|
||||
c = k
|
||||
primary = False
|
||||
# else: # more than one item in long list. But this is not an error, and the max and min have been checked by getXML
|
||||
# slug = Path(filename).stem
|
||||
# message = f' ! ABORT loading this entrance. in "{filename}"'
|
||||
# DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/')
|
||||
# print(message)
|
||||
# slug = Path(filename).stem
|
||||
# message = f' ! ABORT loading this entrance. in "{filename}"'
|
||||
# DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/')
|
||||
# print(message)
|
||||
|
||||
|
||||
def readcave(filename):
|
||||
'''Reads an enrance description from the .html file
|
||||
"""Reads an enrance description from the .html file
|
||||
Convoluted. Sorry.This is as I inherited it and I haven't fiddled with it. Needs rewriting
|
||||
Assumes any area it hasn't seen before is a subarea of 1623
|
||||
'''
|
||||
"""
|
||||
global entrances_xslug
|
||||
global caves_xslug
|
||||
global areas_xslug
|
||||
|
||||
|
||||
# Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
|
||||
with open(os.path.join(CAVEDESCRIPTIONS, filename)) as f:
|
||||
contents = f.read()
|
||||
context = filename
|
||||
cavecontentslist = getXML(contents, "cave", maxItems = 1, context = context)
|
||||
cavecontentslist = getXML(contents, "cave", maxItems=1, context=context)
|
||||
if len(cavecontentslist) != 1:
|
||||
message = f'! BAD CAVE at "{filename}"'
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
else:
|
||||
cavecontents = cavecontentslist[0]
|
||||
non_public = getXML(cavecontents, "non_public", maxItems = 1, context = context)
|
||||
slugs = getXML(cavecontents, "caveslug", maxItems = 1, context = context)
|
||||
official_name = getXML(cavecontents, "official_name", maxItems = 1, context = context)
|
||||
areas = getXML(cavecontents, "area", context = context)
|
||||
kataster_code = getXML(cavecontents, "kataster_code", maxItems = 1, context = context)
|
||||
kataster_number = getXML(cavecontents, "kataster_number", maxItems = 1, context = context)
|
||||
unofficial_number = getXML(cavecontents, "unofficial_number", maxItems = 1, context = context)
|
||||
explorers = getXML(cavecontents, "explorers", maxItems = 1, context = context)
|
||||
underground_description = getXML(cavecontents, "underground_description", maxItems = 1, context = context)
|
||||
equipment = getXML(cavecontents, "equipment", maxItems = 1, context = context)
|
||||
references = getXML(cavecontents, "references", maxItems = 1, context = context)
|
||||
survey = getXML(cavecontents, "survey", maxItems = 1, context = context)
|
||||
kataster_status = getXML(cavecontents, "kataster_status", maxItems = 1, context = context)
|
||||
underground_centre_line = getXML(cavecontents, "underground_centre_line", maxItems = 1, context = context)
|
||||
notes = getXML(cavecontents, "notes", maxItems = 1, context = context)
|
||||
length = getXML(cavecontents, "length", maxItems = 1, context = context)
|
||||
depth = getXML(cavecontents, "depth", maxItems = 1, context = context)
|
||||
extent = getXML(cavecontents, "extent", maxItems = 1, context = context)
|
||||
survex_file = getXML(cavecontents, "survex_file", maxItems = 1, context = context)
|
||||
description_file = getXML(cavecontents, "description_file", maxItems = 1, context = context)
|
||||
url = getXML(cavecontents, "url", maxItems = 1, context = context)
|
||||
entrances = getXML(cavecontents, "entrance", context = context)
|
||||
|
||||
if len(non_public) == 1 and len(slugs) >= 1 and len(official_name) == 1 and len(areas) >= 1 and len(kataster_code) == 1 and len(kataster_number) == 1 and len(unofficial_number) == 1 and len(explorers) == 1 and len(underground_description) == 1 and len(equipment) == 1 and len(references) == 1 and len(survey) == 1 and len(kataster_status) == 1 and len(underground_centre_line) == 1 and len(notes) == 1 and len(length) == 1 and len(depth) == 1 and len(extent) == 1 and len(survex_file) == 1 and len(description_file ) == 1 and len(url) == 1:
|
||||
non_public = getXML(cavecontents, "non_public", maxItems=1, context=context)
|
||||
slugs = getXML(cavecontents, "caveslug", maxItems=1, context=context)
|
||||
official_name = getXML(cavecontents, "official_name", maxItems=1, context=context)
|
||||
areas = getXML(cavecontents, "area", context=context)
|
||||
kataster_code = getXML(cavecontents, "kataster_code", maxItems=1, context=context)
|
||||
kataster_number = getXML(cavecontents, "kataster_number", maxItems=1, context=context)
|
||||
unofficial_number = getXML(cavecontents, "unofficial_number", maxItems=1, context=context)
|
||||
explorers = getXML(cavecontents, "explorers", maxItems=1, context=context)
|
||||
underground_description = getXML(cavecontents, "underground_description", maxItems=1, context=context)
|
||||
equipment = getXML(cavecontents, "equipment", maxItems=1, context=context)
|
||||
references = getXML(cavecontents, "references", maxItems=1, context=context)
|
||||
survey = getXML(cavecontents, "survey", maxItems=1, context=context)
|
||||
kataster_status = getXML(cavecontents, "kataster_status", maxItems=1, context=context)
|
||||
underground_centre_line = getXML(cavecontents, "underground_centre_line", maxItems=1, context=context)
|
||||
notes = getXML(cavecontents, "notes", maxItems=1, context=context)
|
||||
length = getXML(cavecontents, "length", maxItems=1, context=context)
|
||||
depth = getXML(cavecontents, "depth", maxItems=1, context=context)
|
||||
extent = getXML(cavecontents, "extent", maxItems=1, context=context)
|
||||
survex_file = getXML(cavecontents, "survex_file", maxItems=1, context=context)
|
||||
description_file = getXML(cavecontents, "description_file", maxItems=1, context=context)
|
||||
url = getXML(cavecontents, "url", maxItems=1, context=context)
|
||||
entrances = getXML(cavecontents, "entrance", context=context)
|
||||
|
||||
if (
|
||||
len(non_public) == 1
|
||||
and len(slugs) >= 1
|
||||
and len(official_name) == 1
|
||||
and len(areas) >= 1
|
||||
and len(kataster_code) == 1
|
||||
and len(kataster_number) == 1
|
||||
and len(unofficial_number) == 1
|
||||
and len(explorers) == 1
|
||||
and len(underground_description) == 1
|
||||
and len(equipment) == 1
|
||||
and len(references) == 1
|
||||
and len(survey) == 1
|
||||
and len(kataster_status) == 1
|
||||
and len(underground_centre_line) == 1
|
||||
and len(notes) == 1
|
||||
and len(length) == 1
|
||||
and len(depth) == 1
|
||||
and len(extent) == 1
|
||||
and len(survex_file) == 1
|
||||
and len(description_file) == 1
|
||||
and len(url) == 1
|
||||
):
|
||||
try:
|
||||
c, state = Cave.objects.update_or_create(non_public = {"True": True, "False": False, "true": True, "false": False,}[non_public[0]],
|
||||
official_name = official_name[0],
|
||||
kataster_code = kataster_code[0],
|
||||
kataster_number = kataster_number[0],
|
||||
unofficial_number = unofficial_number[0],
|
||||
explorers = explorers[0],
|
||||
underground_description = underground_description[0],
|
||||
equipment = equipment[0],
|
||||
references = references[0],
|
||||
survey = survey[0],
|
||||
kataster_status = kataster_status[0],
|
||||
underground_centre_line = underground_centre_line[0],
|
||||
notes = notes[0],
|
||||
length = length[0],
|
||||
depth = depth[0],
|
||||
extent = extent[0],
|
||||
survex_file = survex_file[0],
|
||||
description_file = description_file[0],
|
||||
url = url[0],
|
||||
filename = filename)
|
||||
c, state = Cave.objects.update_or_create(
|
||||
non_public={
|
||||
"True": True,
|
||||
"False": False,
|
||||
"true": True,
|
||||
"false": False,
|
||||
}[non_public[0]],
|
||||
official_name=official_name[0],
|
||||
kataster_code=kataster_code[0],
|
||||
kataster_number=kataster_number[0],
|
||||
unofficial_number=unofficial_number[0],
|
||||
explorers=explorers[0],
|
||||
underground_description=underground_description[0],
|
||||
equipment=equipment[0],
|
||||
references=references[0],
|
||||
survey=survey[0],
|
||||
kataster_status=kataster_status[0],
|
||||
underground_centre_line=underground_centre_line[0],
|
||||
notes=notes[0],
|
||||
length=length[0],
|
||||
depth=depth[0],
|
||||
extent=extent[0],
|
||||
survex_file=survex_file[0],
|
||||
description_file=description_file[0],
|
||||
url=url[0],
|
||||
filename=filename,
|
||||
)
|
||||
except:
|
||||
print(" ! FAILED to get only one CAVE when updating using: "+filename)
|
||||
print(" ! FAILED to get only one CAVE when updating using: " + filename)
|
||||
kaves = Cave.objects.all().filter(kataster_number=kataster_number[0])
|
||||
for k in kaves:
|
||||
message = " ! - DUPLICATES in db. kataster:"+ str(k.kataster_number) + ", slug:" + str(k.slug())
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
message = " ! - DUPLICATES in db. kataster:" + str(k.kataster_number) + ", slug:" + str(k.slug())
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
for k in kaves:
|
||||
if k.slug() != None:
|
||||
print(" ! - OVERWRITING this one: slug:"+ str(k.slug()))
|
||||
print(" ! - OVERWRITING this one: slug:" + str(k.slug()))
|
||||
k.notes = "DUPLICATE kataster number found on import. Please fix\n" + k.notes
|
||||
c = k
|
||||
|
||||
|
||||
for area_slug in areas:
|
||||
if area_slug in areas_xslug:
|
||||
newArea = areas_xslug[area_slug]
|
||||
else:
|
||||
area = Area.objects.filter(short_name = area_slug)
|
||||
area = Area.objects.filter(short_name=area_slug)
|
||||
if area:
|
||||
newArea = area[0]
|
||||
else:
|
||||
newArea = Area(short_name = area_slug, super = Area.objects.get(short_name = "1623"))
|
||||
newArea = Area(short_name=area_slug, super=Area.objects.get(short_name="1623"))
|
||||
newArea.save()
|
||||
areas_xslug[area_slug] = newArea
|
||||
c.area.add(newArea)
|
||||
@@ -396,17 +439,15 @@ def readcave(filename):
|
||||
if slug in caves_xslug:
|
||||
cs = caves_xslug[slug]
|
||||
else:
|
||||
try: # we want to overwrite a PENDING cave if we are now importing the 1623-xxx.html file for it
|
||||
cs = CaveSlug.objects.update_or_create(cave = c,
|
||||
slug = slug,
|
||||
primary = primary)
|
||||
try: # we want to overwrite a PENDING cave if we are now importing the 1623-xxx.html file for it
|
||||
cs = CaveSlug.objects.update_or_create(cave=c, slug=slug, primary=primary)
|
||||
caves_xslug[slug] = cs
|
||||
except Exception as ex:
|
||||
# This fails to do an update! It just crashes.. to be fixed
|
||||
message = f" ! Cave update/create failure : {slug}, skipping file cave_data/{context} with exception\nException: {ex.__class__}"
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
|
||||
|
||||
primary = False
|
||||
|
||||
if not entrances or len(entrances) < 1:
|
||||
@@ -414,80 +455,87 @@ def readcave(filename):
|
||||
set_dummy_entrance(slug[5:], slug, c, msg="DUMMY")
|
||||
else:
|
||||
for entrance in entrances:
|
||||
eslug = getXML(entrance, "entranceslug", maxItems = 1, context = context)[0]
|
||||
letter = getXML(entrance, "letter", maxItems = 1, context = context)[0]
|
||||
if len(entrances) == 1 and not eslug: # may be empty: <entranceslug></entranceslug>
|
||||
eslug = getXML(entrance, "entranceslug", maxItems=1, context=context)[0]
|
||||
letter = getXML(entrance, "letter", maxItems=1, context=context)[0]
|
||||
if len(entrances) == 1 and not eslug: # may be empty: <entranceslug></entranceslug>
|
||||
set_dummy_entrance(slug[5:], slug, c, msg="DUMMY")
|
||||
else:
|
||||
try:
|
||||
if eslug in entrances_xslug:
|
||||
entrance = entrances_xslug[eslug]
|
||||
else:
|
||||
entrance = Entrance.objects.get(entranceslug__slug = eslug)
|
||||
entrance = Entrance.objects.get(entranceslug__slug=eslug)
|
||||
entrances_xslug[eslug] = entrance
|
||||
ce = CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance)
|
||||
ce = CaveAndEntrance.objects.update_or_create(
|
||||
cave=c, entrance_letter=letter, entrance=entrance
|
||||
)
|
||||
except:
|
||||
message = f' ! Entrance setting failure, slug:"{slug}" #entrances:{len(entrances)} {entrance} letter:"{letter}" cave:"{c}" filename:"cave_data/{filename}"'
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'{c.url}_edit/')
|
||||
DataIssue.objects.create(parser="caves", message=message, url=f"{c.url}_edit/")
|
||||
print(message)
|
||||
|
||||
|
||||
if survex_file[0]:
|
||||
if not (Path(SURVEX_DATA) / survex_file[0]).is_file():
|
||||
message = f' ! {slug:12} survex filename does not exist :LOSER:"{survex_file[0]}" in "{filename}"'
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'/{slug[0:4]}/{slug}_cave_edit/')
|
||||
DataIssue.objects.create(parser="caves", message=message, url=f"/{slug[0:4]}/{slug}_cave_edit/")
|
||||
print(message)
|
||||
|
||||
|
||||
if description_file[0]: # if not an empty string
|
||||
if description_file[0]: # if not an empty string
|
||||
message = f' - {slug:12} Note (not an error): complex description filename "{description_file[0]}" inside "{CAVEDESCRIPTIONS}/{filename}"'
|
||||
DataIssue.objects.create(parser='caves ok', message=message, url=f'/{slug}_cave_edit/')
|
||||
DataIssue.objects.create(parser="caves ok", message=message, url=f"/{slug}_cave_edit/")
|
||||
print(message)
|
||||
|
||||
if not (Path(EXPOWEB) / description_file[0]).is_file():
|
||||
message = f' ! {slug:12} description filename "{EXPOWEB}/{description_file[0]}" does not refer to a real file'
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'/{slug}_cave_edit/')
|
||||
DataIssue.objects.create(parser="caves", message=message, url=f"/{slug}_cave_edit/")
|
||||
print(message)
|
||||
#c.description_file="" # done only once, to clear out cruft.
|
||||
#c.save()
|
||||
else: # more than one item in long list
|
||||
# c.description_file="" # done only once, to clear out cruft.
|
||||
# c.save()
|
||||
else: # more than one item in long list
|
||||
message = f' ! ABORT loading this cave. in "{filename}"'
|
||||
DataIssue.objects.create(parser='caves', message=message, url=f'/{slug}_cave_edit/')
|
||||
DataIssue.objects.create(parser="caves", message=message, url=f"/{slug}_cave_edit/")
|
||||
print(message)
|
||||
|
||||
def getXML(text, itemname, minItems = 1, maxItems = None, printwarnings = True, context = ""):
|
||||
"""Reads a single XML tag
|
||||
"""
|
||||
|
||||
def getXML(text, itemname, minItems=1, maxItems=None, printwarnings=True, context=""):
|
||||
"""Reads a single XML tag"""
|
||||
items = re.findall("<%(itemname)s>(.*?)</%(itemname)s>" % {"itemname": itemname}, text, re.S)
|
||||
if len(items) < minItems and printwarnings:
|
||||
message = " ! %(count)i x %(itemname)s found, at least %(min)i expected. Load ABORT. " % {"count": len(items),
|
||||
"itemname": itemname,
|
||||
"min": minItems} + " in file " + context
|
||||
DataIssue.objects.create(parser='caves', message=message, url=""+context)
|
||||
message = (
|
||||
" ! %(count)i x %(itemname)s found, at least %(min)i expected. Load ABORT. "
|
||||
% {"count": len(items), "itemname": itemname, "min": minItems}
|
||||
+ " in file "
|
||||
+ context
|
||||
)
|
||||
DataIssue.objects.create(parser="caves", message=message, url="" + context)
|
||||
print(message)
|
||||
|
||||
|
||||
if maxItems is not None and len(items) > maxItems and printwarnings:
|
||||
message = " ! %(count)i x %(itemname)s found, no more than %(max)i expected in this XML unit. Load ABORT. " % {"count": len(items),
|
||||
"itemname": itemname,
|
||||
"max": maxItems} + " in file " + context
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
message = (
|
||||
" ! %(count)i x %(itemname)s found, no more than %(max)i expected in this XML unit. Load ABORT. "
|
||||
% {"count": len(items), "itemname": itemname, "max": maxItems}
|
||||
+ " in file "
|
||||
+ context
|
||||
)
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
if minItems == 0:
|
||||
if not items:
|
||||
items = [ "" ]
|
||||
items = [""]
|
||||
return items
|
||||
|
||||
|
||||
|
||||
def readcaves():
|
||||
'''Reads the xml-format HTML files in the EXPOWEB repo, not from the loser repo.
|
||||
'''
|
||||
"""Reads the xml-format HTML files in the EXPOWEB repo, not from the loser repo."""
|
||||
# For those caves which do not have cave_data/1623-xxx.html XML files even though they exist and have surveys
|
||||
# should put this in a simple list
|
||||
# should put this in a simple list
|
||||
pending = set()
|
||||
fpending = Path(CAVEDESCRIPTIONS, "pendingcaves.txt")
|
||||
if fpending.is_file():
|
||||
with open(fpending, "r") as fo:
|
||||
cids = fo.readlines()
|
||||
for cid in cids:
|
||||
pending.add(cid.strip().rstrip('\n').upper())
|
||||
pending.add(cid.strip().rstrip("\n").upper())
|
||||
|
||||
with transaction.atomic():
|
||||
print(" - Deleting Caves and Entrances")
|
||||
@@ -505,55 +553,53 @@ def readcaves():
|
||||
except:
|
||||
pass
|
||||
# Clear the cave data issues and the caves as we are reloading
|
||||
DataIssue.objects.filter(parser='areas').delete()
|
||||
DataIssue.objects.filter(parser='caves').delete()
|
||||
DataIssue.objects.filter(parser='caves ok').delete()
|
||||
DataIssue.objects.filter(parser='entrances').delete()
|
||||
|
||||
DataIssue.objects.filter(parser="areas").delete()
|
||||
DataIssue.objects.filter(parser="caves").delete()
|
||||
DataIssue.objects.filter(parser="caves ok").delete()
|
||||
DataIssue.objects.filter(parser="entrances").delete()
|
||||
|
||||
print(" - Creating Areas 1623, 1624, 1627 and 1626")
|
||||
# This crashes on the server with MariaDB even though a null parent is explicitly allowed.
|
||||
area_1623= Area.objects.create(short_name = "1623", super=None)
|
||||
area_1623 = Area.objects.create(short_name="1623", super=None)
|
||||
area_1623.save()
|
||||
area_1624= Area.objects.create(short_name = "1624", super=None)
|
||||
area_1624.save()
|
||||
area_1626= Area.objects.create(short_name = "1626", super=None)
|
||||
area_1624 = Area.objects.create(short_name="1624", super=None)
|
||||
area_1624.save()
|
||||
area_1626 = Area.objects.create(short_name="1626", super=None)
|
||||
area_1626.save()
|
||||
area_1627= Area.objects.create(short_name = "1627", super=None)
|
||||
area_1627 = Area.objects.create(short_name="1627", super=None)
|
||||
area_1627.save()
|
||||
|
||||
|
||||
with transaction.atomic():
|
||||
print(" - settings.CAVEDESCRIPTIONS: ", CAVEDESCRIPTIONS)
|
||||
print(" - Reading Entrances from entrance descriptions xml files")
|
||||
for filename in next(os.walk(ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
|
||||
for filename in next(os.walk(ENTRANCEDESCRIPTIONS))[2]: # Should be a better way of getting a list of files
|
||||
# if filename.endswith('.html'):
|
||||
# if Path(filename).stem[5:] in pending:
|
||||
# print(f'Skipping pending entrance dummy file <{filename}>')
|
||||
# else:
|
||||
# readentrance(filename)
|
||||
# if Path(filename).stem[5:] in pending:
|
||||
# print(f'Skipping pending entrance dummy file <{filename}>')
|
||||
# else:
|
||||
# readentrance(filename)
|
||||
readentrance(filename)
|
||||
|
||||
print(" - Reading Caves from cave descriptions xml files")
|
||||
for filename in next(os.walk(CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
|
||||
if filename.endswith('.html'):
|
||||
for filename in next(os.walk(CAVEDESCRIPTIONS))[2]: # Should be a better way of getting a list of files
|
||||
if filename.endswith(".html"):
|
||||
readcave(filename)
|
||||
|
||||
print (" - Setting up all the variously useful alias names")
|
||||
print(" - Setting up all the variously useful alias names")
|
||||
mycavelookup = GetCaveLookup()
|
||||
|
||||
print (" - Setting pending caves")
|
||||
|
||||
print(" - Setting pending caves")
|
||||
# Do this last, so we can detect if they are created and no longer 'pending'
|
||||
|
||||
with transaction.atomic():
|
||||
for k in pending:
|
||||
|
||||
|
||||
if k[0:3] == "162":
|
||||
areanum = k[0:4]
|
||||
url = f'{areanum}/{k[5:]}' # Note we are not appending the .htm as we are modern folks now.
|
||||
url = f"{areanum}/{k[5:]}" # Note we are not appending the .htm as we are modern folks now.
|
||||
else:
|
||||
areanum = "1623"
|
||||
url = f'1623/{k}'
|
||||
|
||||
url = f"1623/{k}"
|
||||
|
||||
area = area_1623
|
||||
if areanum == "1623":
|
||||
@@ -564,12 +610,10 @@ def readcaves():
|
||||
area = area_1626
|
||||
if areanum == "1627":
|
||||
area = area_1627
|
||||
try:
|
||||
try:
|
||||
do_pending_cave(k, url, area)
|
||||
except:
|
||||
message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum}"
|
||||
DataIssue.objects.create(parser='caves', message=message)
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
raise
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user