mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2024-11-25 16:51:54 +00:00
931 lines
41 KiB
Python
931 lines
41 KiB
Python
import os
|
|
import re
|
|
import string
|
|
import sys
|
|
|
|
from pathlib import Path
|
|
|
|
from django.conf import settings
|
|
from django.db import transaction
|
|
|
|
from troggle.core.models.caves import Cave, CaveAndEntrance, Entrance, GetCaveLookup
|
|
from troggle.core.models.logbooks import CaveSlug
|
|
from troggle.core.models.survex import SurvexStation
|
|
from troggle.core.models.troggle import DataIssue
|
|
from troggle.settings import CAVEDESCRIPTIONS, ENTRANCEDESCRIPTIONS, EXPOWEB, SURVEX_DATA
|
|
|
|
"""Reads all the cave description data and entrance description data
|
|
by parsing the xml files stored as e.g.
|
|
:EXPOWEB:/cave_data/1623-161.html
|
|
or
|
|
:EXPOWEB:/entrance_data/1623-161g.html
|
|
|
|
and creating the various Cave, Entrance and necessary Area objects.
|
|
|
|
This is the first import that happens after the database is reinitialised.
|
|
So is the first thing that creates tables.
|
|
|
|
"""
|
|
|
|
todo = """
|
|
- When reading cave data, to start off wit we do not know the cave id (slug) so we can't give a useful url in
|
|
the error message, but we do have the filename. Systematize this, and the same thing with reading entrance files.
|
|
|
|
- Cannot use Edit This Page for pendingcaves.txt_edit as Edit This Page is expecting an html file.
|
|
So we will need a separate file-editing capability just for this configuration file ?!
|
|
|
|
- we want to overwrite a PENDING cave if we are now importing the 1623-xxx.html file for it
|
|
|
|
- rewrite archaic regex
|
|
re.findall("<%(itemname)s>(.*?)</%(itemname)s>" % {"itemname": itemname}, text, re.S)
|
|
in modern form and pre-compile it.
|
|
|
|
- crashes on MariaDB in databasereset.py on server when deleting Caves and complains Area needs a
|
|
non null parent, But this is not true. The only solution we have found is to let it crash, then
|
|
stop and restart MariaDB (requires a logon able to sudo) and then restart the databasereset.py
|
|
again. (status as of July 2022). May not happen now that class Area is removed (Sept.2023).
|
|
"""
|
|
AREACODES = {"1623", "1624", "1626", "1627"} # NB set not dict
|
|
ARGEAREAS = {"1626", "1627", "1624"} # NB set not dict
|
|
|
|
entrances_xslug = {}
|
|
caves_xslug = {}
|
|
areas_xslug = {}
|
|
|
|
|
|
def dummy_entrance(k, slug, msg="DUMMY"):
|
|
"""Returns an empty entrance object for either a PENDING cave or a DUMMY entrance if
|
|
user forgot to provide one when creating the cave
|
|
"""
|
|
ent = Entrance.objects.create( # creates object and saves into db
|
|
name=k, slug=k, filename = k + ".html",
|
|
entrance_description="Dummy entrance: auto-created when registering a new cave "
|
|
+ "and you forgot to create an entrance for it. Click on 'Edit' to enter the correct data, then 'Submit'.",
|
|
marking="?",
|
|
)
|
|
if ent:
|
|
return ent
|
|
else:
|
|
message = f" ! {k:11s} {msg}-{slug} {k} entrance create failure"
|
|
DataIssue.objects.create(parser="entrances", message=message, url=f"{slug}")
|
|
print(message)
|
|
raise # caught and handled by calling routine.
|
|
|
|
|
|
|
|
def set_dummy_entrance(id, slug, cave, msg="DUMMY"):
|
|
"""Called only when reading the cave and entrance html files
|
|
|
|
Called when the Entrance field in a cave_data file is either missing or
|
|
holds a null string instead of a filename.
|
|
|
|
Previously, the lack of an entrance where an entrance was expected, caused troggle to crash in several places.
|
|
But it is more robust now, so this is not necessary... we hope.
|
|
|
|
Also, Cave and Entrance editing now expects there to be a real file (since April 2023), so creating this
|
|
dummy is actually harmful. So this is commented out, pending removal after further experience.
|
|
|
|
global variable entrances_xslug is simply a cache of references to Entrance objects
|
|
to speed things up when parsing a lot of caves and entrances. All DB actions are time-consuming
|
|
so
|
|
"""
|
|
global entrances_xslug
|
|
message = f" - Note: Missing Entrance for entrance '{id}' on cave '{cave}' - Is this a problem?"
|
|
DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.url}")
|
|
|
|
def add_cave_to_pending_list(id, wallet, message):
|
|
"""(id, f"Wallet {wallet} - Could not find id <{id}>")
|
|
"""
|
|
wurl = f"/walletedit/{wallet.walletname}".replace('#', ':')
|
|
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
|
|
|
|
def create_new_cave(svxpath, svxid=None, msg=None):
|
|
"""This is called only when a new survex file is edited online which has a path on the
|
|
:loser: repo which is not recognised as a known cave.
|
|
ALSO called by survex parser when it finds a cave it doesn't recognise
|
|
"""
|
|
# e.g. svxpath = "caves-1623/666/beast" .svx - from the *inlcude tree
|
|
# e.g. svxid = "caves-1623/666/beast"
|
|
print(f"Create new cave at {svxpath} - {msg}")
|
|
#
|
|
survex_file = ""
|
|
if svxid:
|
|
sv = Path(settings.SURVEX_DATA, svxid + ".svx")
|
|
if sv.is_file:
|
|
survex_file = svxid + ".svx"
|
|
else:
|
|
sv = Path(settings.SURVEX_DATA, svxpath + ".svx")
|
|
if sv.is_file:
|
|
survex_file = svxpath + ".svx"
|
|
|
|
if survex_file:
|
|
# message = f"Found a survex file {survex_file=} {svxpath=} {svxid=} "
|
|
# DataIssue.objects.create(parser="caves", message=message)
|
|
# print(message, file=sys.stderr)
|
|
# print(message)
|
|
pass
|
|
else:
|
|
message = f"NOT found a survex file {svxpath=} {svxid=}"
|
|
DataIssue.objects.create(parser="caves", message=message)
|
|
print(message, file=sys.stderr)
|
|
print(message)
|
|
|
|
parts = svxpath.split("/")
|
|
a = parts[0][-4:]
|
|
caveid = parts[1]
|
|
print(f"parts {parts}, {a}, {caveid}")
|
|
# double check
|
|
if a[0:3] == "162":
|
|
areacode = a[0:4]
|
|
url = f"{areacode}/{caveid}/{caveid}.html" # Note we are appending the .html as we are believe in backwards compatability. This is to fix Martin's new 2023 app.
|
|
else:
|
|
print(f"WARNING: parsers/caves/create_new_cave called with svxpath '{svxpath}'. Surely it should start 'caves-162*'? {msg}")
|
|
areacode = "1623"
|
|
url = f"1623/{caveid}/{caveid}.html"
|
|
|
|
k = f"{areacode}-{caveid}"
|
|
|
|
caves = Cave.objects.filter(unofficial_number=caveid, areacode =areacode)
|
|
if caves:
|
|
message = f" ! Already exists, caveid:{k} in areacode {areacode} {caves} - {msg}"
|
|
DataIssue.objects.create(parser="caves", message=message)
|
|
print(message)
|
|
return caves[0]
|
|
|
|
urltest = Cave.objects.filter(url=url)
|
|
if urltest:
|
|
message = f" ! Cave {urltest[0]} already exists with this url {url}. Can't create new cave {slug} from {svxpath} "
|
|
DataIssue.objects.create(parser="caves", message=message, url=url)
|
|
print(message)
|
|
return urltest[0]
|
|
try:
|
|
cave = do_pending_cave(k, caveid, url, areacode, msg)
|
|
except:
|
|
message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areacode} - {msg}"
|
|
DataIssue.objects.create(parser="caves", message=message)
|
|
print(message)
|
|
raise
|
|
|
|
# But this sets the survex file on the Cave from the first one we find, not necessarily the best survex file for this cave
|
|
cave.survex_file=survex_file # primary survex file for Cave
|
|
cave.areacode=areacode
|
|
cave.save()
|
|
return cave
|
|
|
|
def do_ARGE_cave(slug, caveid, areacode, svxid):
|
|
"""Only called by survex parser, NOT the cave parser.
|
|
Creates a new Cave object, but with abbreviated data as the survex file (from ARGE) is all we have.
|
|
We already know the survex file.
|
|
We already know that the cave doesn't exist... though there are bugs..
|
|
|
|
Assumes anything in the ARGE list of cave areas is Arge, which is not true for 1626...
|
|
|
|
caveid may be kataster number or it may be e.g. LA34
|
|
"""
|
|
|
|
default_note = "This is (probably) an ARGE cave where we only have the survex file and no other information"
|
|
url = f"{areacode}/{caveid}/{caveid}.html"
|
|
|
|
urltest = Cave.objects.filter(url=url)
|
|
if urltest:
|
|
message = f" ! Cave {urltest[0]} already exists with this url {url}. Can't create new ARGE cave {slug} from {svxid}"
|
|
DataIssue.objects.create(parser="caves", message=message, url=url)
|
|
print(message)
|
|
return urltest[0]
|
|
|
|
numtest = Cave.objects.filter(unofficial_number=caveid.upper(), areacode=areacode)
|
|
if numtest:
|
|
message = f" ! Cave {numtest[0]} already exists with this areacode {areacode} and unofficial_number {caveid.upper()}. Can't create new ARGE cave {slug}"
|
|
DataIssue.objects.create(parser="caves", message=message, url=url)
|
|
print(message)
|
|
return numtest[0]
|
|
|
|
sv = Path(settings.SURVEX_DATA, svxid + ".svx")
|
|
if sv.is_file:
|
|
with open(sv, "r") as s:
|
|
line1 = s.readline()
|
|
line2 = s.readline()
|
|
line3 = s.readline()
|
|
rest = s.readlines()
|
|
else:
|
|
print(f"not correct svxid {svxid} {sv}", file=sys.stderr)
|
|
|
|
print(f"{caveid} {rest}")
|
|
passages = "\n"
|
|
for line in rest:
|
|
if line.strip().startswith("*begin"):
|
|
passages = f"{passages}{line}"
|
|
cave = Cave(
|
|
underground_description="ARGE cave.<br>3 lines of the survexfile, then all the *begin lines:<br><pre>" + line1 +line2 +line3 +passages +"</pre>",
|
|
unofficial_number="ARGE",
|
|
survex_file= f"{svxid}.svx",
|
|
url=url,
|
|
notes=default_note,
|
|
areacode=areacode,
|
|
)
|
|
if cave:
|
|
try:
|
|
kn = int(caveid)
|
|
cave.kataster_number=kn # should only set this if all digit
|
|
except:
|
|
# must be unofficial 'number' or name
|
|
cave.unofficial_number=caveid
|
|
|
|
cave.save()
|
|
|
|
try: # Now create a cave slug ID
|
|
CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False)
|
|
except:
|
|
message = f" ! {slug:11s} ARGE CaveSLUG create failure {caveid=} {url=} {areacode=} {svxid=}"
|
|
DataIssue.objects.create(parser="caves", message=message)
|
|
print(message)
|
|
else:
|
|
message = f" ! {slug:11s} ARGE cave create failure {caveid=} {url=} {areacode=} {svxid=}"
|
|
DataIssue.objects.create(parser="caves", message=message)
|
|
print(message)
|
|
return None
|
|
|
|
return cave
|
|
|
|
def do_pending_cave(slug, caveid, url, areacode, msg=None):
|
|
"""
|
|
default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists
|
|
in expoweb/cave_data/1623-"k".html
|
|
|
|
Note that at this point (parsing caves) in importing the data we have not yet seen the survex files, so we can't
|
|
look inside the relevant survex file to find the year and so we con't provide helpful links.
|
|
This also gets called when parsing survex files, when we do have this info.
|
|
"""
|
|
|
|
def get_survex_file(k):
|
|
"""Guesses at and finds a survex file for this pending cave.
|
|
Convoluted. Needs rewriting.
|
|
Pointless if this cave is being created because we found a survex file...
|
|
|
|
One problem is that the Cave name may have different capitalisation from the survex filename,
|
|
e.g. 2018-NTU-02 has a survex file 2018-ntu-02.svx
|
|
"""
|
|
if k[0:3] == "162":
|
|
id = Path(k[5:])
|
|
else:
|
|
id = Path(k)
|
|
|
|
survex_file = f"caves-{areacode}/{id}/{id}.svx"
|
|
if Path(settings.SURVEX_DATA, survex_file).is_file():
|
|
return survex_file
|
|
else:
|
|
survex_file = f"caves-{areacode}/{id}.svx"
|
|
if Path(settings.SURVEX_DATA, survex_file).is_file():
|
|
return survex_file
|
|
|
|
# This should find the file even if the capitalisation is different, or if the directory name is totally different
|
|
survex_file = ""
|
|
d = Path(settings.SURVEX_DATA, f"caves-{areacode}/{id}")
|
|
if d.is_dir():
|
|
prime_suspect = ""
|
|
dir = d.iterdir()
|
|
for f in dir:
|
|
if f.suffix == ".svx":
|
|
survex_file = f.relative_to(settings.SURVEX_DATA)
|
|
chk = min(5, len(f.name) - 1)
|
|
if str(f.name)[:chk].lower() == str(id.name)[:chk].lower(): # bodge which mostly works
|
|
prime_suspect = survex_file
|
|
if prime_suspect:
|
|
survex_file = prime_suspect
|
|
# message = f" ! {k:14} Found a survex file which might be the right one: {survex_file} - {msg}"
|
|
# DataIssue.objects.create(parser='caves', message=message, url=url)
|
|
# print(message)
|
|
if Path(settings.SURVEX_DATA, survex_file).is_file():
|
|
return survex_file
|
|
return ""
|
|
|
|
g = GetCaveLookup()
|
|
with transaction.atomic():
|
|
if slug in g:
|
|
message = f" ! {slug:18} cave listed in pendingcaves.txt already exists. - {msg}"
|
|
DataIssue.objects.create(parser="caves", message=message, url=url)
|
|
print(message)
|
|
return
|
|
|
|
default_note = "A reference has been found to this cave id in a survex file in the loser repo, or in a wallet metadata"
|
|
default_note += " in a JSON file in the drawings repo, but no Cave Description exists in expoweb (in /cave_data/)<br><br><br>\n"
|
|
default_note += "INSTRUCTIONS: FIRST read the notes in <a href='/cave_data/pendingcaves.txt'>pendingcaves.txt</a><br />"
|
|
default_note += "Next open 'This survex file' (link above the CaveView panel) to find the date and info. Then "
|
|
default_note += '<br><br>\n\n - (0) look in the <a href="/noinfo/cave-number-index">cave number index</a> for notes on this cave, '
|
|
default_note += "<br><br>\n\n - (1) search in the survex file for the *ref to find a "
|
|
default_note += "relevant wallet, e.g.<a href='/survey_scans/2009%252311/'>2009#11</a> and read the notes image files <br>\n - "
|
|
default_note += (
|
|
"<br><br>\n\n - (2) search in the Expo for that year e.g. <a href='/expedition/2009'>2009</a> to find a "
|
|
)
|
|
default_note += "relevant logbook entry, remember that the date may have been recorded incorrectly, "
|
|
default_note += (
|
|
"so check for trips i.e. logbook entries involving the same people as were listed in the survex file, "
|
|
)
|
|
default_note += (
|
|
"and you should also check the scanned copy of the logbook (linked from each logbook entry page) "
|
|
)
|
|
default_note += "just in case a vital trip was not transcribed, then <br>\n - "
|
|
default_note += (
|
|
"click on 'Edit this cave' and copy the information you find in the survex file and the logbook"
|
|
)
|
|
default_note += "and delete all the text in the 'Notes' section - which is the text you are reading now."
|
|
default_note += "<br><br>\n\n - Only two fields on this form are essential. "
|
|
default_note += "Documentation of all the fields on 'Edit this cave' form is in <a href='/handbook/survey/caveentryfields.html'>handbook/survey/caveentryfields</a>"
|
|
default_note += "<br><br>\n\n - "
|
|
default_note += "You will also need to create a new entrance from the 'Edit this cave' page. Ignore the existing dummy one, it will evaporate on the next full import."
|
|
default_note += "<br><br>\n\n - "
|
|
default_note += "When you Submit it will create a new file in expoweb/cave_data/ "
|
|
default_note += (
|
|
"<br><br>\n\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. "
|
|
)
|
|
default_note += "and then Submit to save it (if you forget to do this, a dummy entrance will be created for your new cave description)."
|
|
default_note += "<br><br>\n\n - Finally, you need to find a nerd to edit the file '<var>expoweb/cave_data/pending.txt</var>' "
|
|
default_note += (
|
|
f"to remove the line <br><var>{slug}</var><br> as it is no longer 'pending' but 'done. Well Done."
|
|
)
|
|
urltest = Cave.objects.filter(url=url)
|
|
if urltest:
|
|
message = f" ! Cave {urltest[0]} already exists with this url {url}. Can't create new cave {slug}"
|
|
DataIssue.objects.create(parser="caves", message=message, url=url)
|
|
print(message)
|
|
return urltest[0]
|
|
|
|
survex_file = get_survex_file(slug)
|
|
|
|
cave = Cave(
|
|
unofficial_number=caveid,
|
|
underground_description="Pending cave write-up - No cave description created yet.",
|
|
survex_file=survex_file,
|
|
url=url,
|
|
notes=default_note,
|
|
areacode=areacode,
|
|
)
|
|
if cave:
|
|
cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key.
|
|
message = f" ! {slug:18} Pending cave write-up url: {url} - {survex_file=} - {msg}"
|
|
DataIssue.objects.create(parser="caves", message=message, url=url)
|
|
print(message)
|
|
|
|
try: # Now create a cave slug ID
|
|
CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False)
|
|
except:
|
|
message = f" ! {slug:11s} PENDING CaveSLUG {slug} create failure - {msg}"
|
|
DataIssue.objects.create(parser="caves", message=message)
|
|
print(message)
|
|
else:
|
|
message = f" ! {slug:11s} PENDING cave create failure - {msg}"
|
|
DataIssue.objects.create(parser="caves", message=message)
|
|
print(message)
|
|
|
|
return cave
|
|
|
|
def getXML(text, itemname, minItems=1, maxItems=None, context=""):
|
|
"""Reads a single XML tag
|
|
Should throw exception rather than producing error message here,
|
|
then handle exception in calling routine where it has the context.
|
|
|
|
This always succeeds, but it produces error message on the terminal and in the
|
|
DataIssues log.
|
|
"""
|
|
items = re.findall("<%(itemname)s>(.*?)</%(itemname)s>" % {"itemname": itemname}, text, re.S)
|
|
if len(items) < minItems:
|
|
message = (
|
|
" ! %(count)i x %(itemname)s found, at least %(min)i expected. Load may ABORT. "
|
|
% {"count": len(items), "itemname": itemname, "min": minItems}
|
|
+ " in file "
|
|
+ context
|
|
)
|
|
DataIssue.objects.create(parser="caves", message=message, url="" + context)
|
|
print(message)
|
|
|
|
if maxItems is not None and len(items) > maxItems:
|
|
message = (
|
|
" ! %(count)i x %(itemname)s found, no more than %(max)i expected in this XML unit. Load may ABORT. "
|
|
% {"count": len(items), "itemname": itemname, "max": maxItems}
|
|
+ " in file "
|
|
+ context
|
|
)
|
|
DataIssue.objects.create(parser="caves", message=message, url="" + context)
|
|
print(message)
|
|
if minItems == 0:
|
|
if not items:
|
|
items = [""]
|
|
return items
|
|
|
|
|
|
def boolify(boolstrs):
|
|
return {
|
|
"True": True,
|
|
"False": False,
|
|
"true": True,
|
|
"false": False}[boolstrs[0]]
|
|
|
|
def validate_station(station):
|
|
"""It is possible to break troggle entirely by getting this wrong.
|
|
These station identifiers are matched against other stations using .endswith()
|
|
in parsers/locations.py
|
|
so a simple number here will match hundreds of SUrvexStation objects
|
|
It should be, e.g. "1623.p240"
|
|
|
|
We will test them against survex stations after we have loaded them.
|
|
"""
|
|
if station == "":
|
|
return True
|
|
|
|
# CANNOT test against locations as we have not read the survex files yet. Hmph.
|
|
|
|
# Must have the right format in its name
|
|
dot = station.find(".")
|
|
if dot == -1:
|
|
print(dot)
|
|
# no full stop found. Bad station identifier.
|
|
# should just skip really, and log an error
|
|
raise
|
|
else:
|
|
return True
|
|
|
|
def read_entrance(filename, ent=None):
|
|
"""Reads an entrance description from the .html file.
|
|
Runs on initial full import, and also whenever an entrance is edited online.
|
|
|
|
If not called as part of initial import, then the global lists will not be correct
|
|
but this is OK, a search will find them in the db.
|
|
|
|
Args:
|
|
filename: The name of the .html file.
|
|
ent: The entrance object, if it already exists.
|
|
|
|
Returns:
|
|
The entrance object, or a new entrance object if `ent` is None.
|
|
"""
|
|
def getXMLmax1(field):
|
|
return getXML(entrancecontents, field, maxItems=1, context=context)
|
|
|
|
global entrances_xslug
|
|
global caves_xslug
|
|
global areas_xslug
|
|
|
|
# Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
|
|
with open(os.path.join(ENTRANCEDESCRIPTIONS, filename)) as f:
|
|
contents = f.read()
|
|
context = filename
|
|
|
|
# validate filename, check areacode
|
|
ent_area = filename[:4]
|
|
if ent_area not in AREACODES:
|
|
message = f'! BAD AREA CODE in "{filename}". Not recognised.'
|
|
DataIssue.objects.create(parser="entrances", message=message, url=f"/entrance_data/{filename}_edit")
|
|
print(message)
|
|
|
|
entrancecontentslist = getXML(contents, "entrance", maxItems=1, context=context)
|
|
if len(entrancecontentslist) != 1:
|
|
message = f'! BAD ENTRANCE DATA in "{filename}". More than one entrance. Edit file manually, click.'
|
|
DataIssue.objects.create(parser="entrances", message=message, url=f"/entrance_data/{filename}_edit")
|
|
print(message)
|
|
return None
|
|
|
|
entrancecontents = entrancecontentslist[0]
|
|
slugs = getXML(entrancecontents, "slug", context=context)
|
|
slug = slugs[0]
|
|
|
|
if len(slugs) >1:
|
|
# Only ever one of these per entrance in the expo dataset
|
|
message = f" ! - More than one slug for an entrance: {entrance}, slugs: {slugs}. Ignoring all except first."
|
|
DataIssue.objects.create(parser="entrances", message=message, url=f"/cave/{slug}/edit/")
|
|
print(message)
|
|
|
|
lastvisit = getXML(entrancecontents, "lastvisit", maxItems=1, minItems=0, context=f"/cave/{slug}/edit/")
|
|
lat_wgs84 = getXML(entrancecontents, "lat_wgs84", maxItems=1, minItems=0, context=f"/cave/{slug}/edit/")
|
|
long_wgs84 = getXML(entrancecontents, "long_wgs84", maxItems=1, minItems=0, context=f"/cave/{slug}/edit/")
|
|
|
|
alt = getXMLmax1("alt")
|
|
approach = getXMLmax1("approach")
|
|
bearings = getXMLmax1("bearings")
|
|
entrance_description = getXMLmax1("entrance_description")
|
|
explorers = getXMLmax1("explorers")
|
|
findability = getXMLmax1("findability")
|
|
findability_description = getXMLmax1("findability_description")
|
|
location_description = getXMLmax1("location_description")
|
|
map_description = getXMLmax1("map_description")
|
|
marking = getXMLmax1("marking")
|
|
marking_comment = getXMLmax1("marking_comment")
|
|
name = getXMLmax1("name")
|
|
non_public = getXMLmax1("non_public")
|
|
other_description = getXMLmax1("other_description")
|
|
other_station = getXMLmax1("other_station")
|
|
photo = getXMLmax1("photo")
|
|
tag_station = getXMLmax1("tag_station")
|
|
underground_description = getXMLmax1("underground_description")
|
|
|
|
if not ent:
|
|
ent, state = Entrance.objects.update_or_create(slug=slug)
|
|
|
|
ent.name=name[0]
|
|
ent.non_public=boolify(non_public)
|
|
ent.alt=alt[0]
|
|
ent.approach=approach[0]
|
|
ent.bearings=bearings[0]
|
|
ent.lat_wgs84=lat_wgs84[0]
|
|
ent.long_wgs84=long_wgs84[0]
|
|
ent.entrance_description=entrance_description[0]
|
|
ent.explorers=explorers[0]
|
|
ent.filename=filename
|
|
ent.findability=findability[0]
|
|
ent.findability_description=findability_description[0]
|
|
ent.lastvisit=lastvisit[0]
|
|
ent.location_description=location_description[0]
|
|
ent.map_description=map_description[0]
|
|
ent.marking=marking[0]
|
|
ent.marking_comment=marking_comment[0]
|
|
ent.other_description=other_description[0]
|
|
ent.other_station=other_station[0]
|
|
ent.photo=photo[0]
|
|
# ent.slug=slugs[0] # set algorithically
|
|
ent.tag_station=tag_station[0]
|
|
ent.underground_description=underground_description[0]
|
|
|
|
for st in [ent.other_station, ent.tag_station]:
|
|
#validate_station(st)
|
|
try:
|
|
validate_station(st)
|
|
except:
|
|
message = f" ! BAD ENTRANCE TAG '{st}' in '{filename}'. Must format like '1623.p204'. Edit file manually, click."
|
|
#http://localhost:8000/1623/2023-EBH-01/1623-2023-EBH-01:1623-2023-EBH-01_entrance_edit
|
|
DataIssue.objects.create(parser="entrances", message=message, url=f"/1623/{slug}/{slug}:{slug}_entrance_edit")
|
|
print(message)
|
|
ent.save()
|
|
return ent
|
|
|
|
def read_cave(filename, cave=None):
|
|
"""Reads an entrance description from the .html file
|
|
Convoluted. Sorry. Needs rewriting
|
|
|
|
If not called as part of initial import, then the global lists will not be correct
|
|
but this is OK, a search will find them in the db.
|
|
|
|
Attempted to use standard python3.11 xml library but fails on HTML entities (2023-04-23)
|
|
import xml.etree.ElementTree as ET
|
|
tree = ET.parse(fn)
|
|
xml_root = tree.getroot()
|
|
for t in ["html", "head", "body", "cave","non_public", "caveslug", "official_name","entrance"]:
|
|
elements = xml_root.findall(t)
|
|
|
|
"""
|
|
def getXMLmax1(field):
|
|
return getXML(cavecontents, field, maxItems=1, context=context)
|
|
|
|
def do_entrances():
|
|
"""For both bulk import and individual re-reading of cave_data file,
|
|
fix the entrances
|
|
What is Class CaveAndEntrance for?
|
|
"""
|
|
for e in entrances:
|
|
|
|
eslug = getXML(e, "entranceslug", maxItems=1, context=context)[0]
|
|
# if eslug.endswith(('a','b','c','d','e','f')):
|
|
# print(f"! Entrance {eslug}")
|
|
if eslug.endswith('a b'):
|
|
message = f' - Entrance has weird name slug:"{eslug}" cave:"{cave}" caveslug:"{slug}" filename:"cave_data/{filename}"'
|
|
DataIssue.objects.create(parser="xEntrances", message=message, url=f"{cave.url}_cave_edit/")
|
|
# print(message)
|
|
|
|
letter = getXML(e, "letter", maxItems=1, context=context)[0]
|
|
|
|
if len(entrances) > 1 and letter =="":
|
|
# Usually the second entrance is 'b', but the first is still unlettered. So probably 'a'
|
|
letter = eslug[-1].lower()
|
|
if letter.lower() not in list(string.ascii_lowercase):
|
|
letter = "a"
|
|
message = f"- Warning - Empty 'letter' field for '{eslug}' in multiple-entrance cave '{cave}', setting to {letter}."
|
|
#eurl = f"{cave.url}_cave_edit/"
|
|
eurl = Path(cave.url).parent + f"{cave.slug()}_cave_edit/"
|
|
# edit recognizer: (?P<path>.*)/(?P<slug>[^/]+)_cave_edit/$
|
|
DataIssue.objects.create(parser="entrances", message=message, url=eurl)
|
|
print(message)
|
|
|
|
if len(entrances) == 1 and not eslug: # may be empty: <entranceslug></entranceslug>
|
|
msg="DUMMY: no entrance slug read from file, so assume textually same as cave slug"
|
|
set_dummy_entrance(slug[5:], slug, c, msg=msg)
|
|
print(f"! {msg}\n- {slug} {c}")
|
|
else:
|
|
if eslug in entrances_xslug:
|
|
# print(f"eslug {eslug} found eslug in xslug cache ")
|
|
entrance = entrances_xslug[eslug]
|
|
else:
|
|
# print(f"eslug {eslug} looking up entrance ")
|
|
try:
|
|
entrance = Entrance.objects.get(slug=eslug)
|
|
entrances_xslug[eslug] = entrance
|
|
except:
|
|
message = f"! Fail entrance loading {eslug} /entrance_data/{eslug} file does not exist or loading it failed."
|
|
DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.url}_cave_edit/")
|
|
print(message)
|
|
return
|
|
|
|
if eslug != f"{entrance}":
|
|
message = f"eslug {eslug} using different entrance {entrance} to set CaveAndEntrance"
|
|
DataIssue.objects.create(parser="xEntrances", message=message, url=f"{cave.areacode}/{cave.areacode}-{cave.url}_cave_edit/")
|
|
print(message)
|
|
try:
|
|
# this fails if there is not an unambiguous letter set.
|
|
CaveAndEntrance.objects.update_or_create(
|
|
cave=cave, entranceletter=letter, entrance=entrance
|
|
)
|
|
except:
|
|
print(f"! Entrance setting failure {slug}")
|
|
message = f' ! Entrance setting failure, slug:"{slug}" #entrances:{len(entrances)} letter:"{letter}" cave:"{cave}" filename:"cave_data/{filename}"\n{e}'
|
|
DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.areacode}/{cave.areacode}-{cave.url}_cave_edit/")
|
|
print(message)
|
|
def reload_entrances():
|
|
"""For individual re-reading of a cave_data file when editing,
|
|
also re-read the entrance_data files
|
|
"""
|
|
for eslug in entrances_xslug:
|
|
entrance = entrances_xslug[eslug]
|
|
read_entrance(entrance.filename, ent=entrance)
|
|
entrance.save()
|
|
|
|
def do_caveslugstuff():
|
|
"""This may be a fossil. We only have one slug per cave in troggle.
|
|
Pending destruction of this whole concept and Class CaveSlug
|
|
What is Class CaveSlug for?
|
|
"""
|
|
primary = True # this sets the first thing we find to be primary=True and all the others =False
|
|
for slug in slugs:
|
|
if slug in caves_xslug:
|
|
cs = caves_xslug[slug]
|
|
else:
|
|
try:
|
|
cs = CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=primary)
|
|
caves_xslug[slug] = cs
|
|
except Exception as ex:
|
|
#raise
|
|
# This fails to do an update! It just crashes.. to be fixed
|
|
message = f" ! CaveSlug update/create failure : {slug}, skipping file cave_data/{context} with exception\nException: {ex.__class__}"
|
|
DataIssue.objects.create(parser="caves", message=message, url=f"{cave.url}_edit/")
|
|
print(message)
|
|
primary = False
|
|
|
|
def check_directory(areacode, caveid, url, cave):
|
|
dir = Path(settings.EXPOWEB, areacode, caveid)
|
|
dir_l = Path(settings.EXPOWEB, areacode, caveid.lower())
|
|
dir_u = Path(settings.EXPOWEB, areacode, caveid.upper())
|
|
if dir.is_dir():
|
|
return
|
|
if dir_l.is_dir() or dir_u.is_dir():
|
|
message = f" ! Cave URL capitalisation incorrect '{dir}' is not a directory but different capitalisation is. {url=} "
|
|
DataIssue.objects.create(parser="caves", message=message, url=f"{cave.slug()}_cave_edit/")
|
|
print(message)
|
|
return
|
|
if cave.filename:
|
|
# not a pending cave, yet the directory does not exist. This is FINE. Many don't (yet)
|
|
pass
|
|
|
|
def check_slug(areacode, kataster_number, unofficial_number, url):
|
|
if kataster_number:
|
|
if slug == f"{areacode}-{kataster_number}":
|
|
return
|
|
message = f" ! Cave Slug incorrect (kataster): '{slug}' != '{areacode}-{kataster_number}' {url=} in file {filename}"
|
|
else:
|
|
if slug == f"{areacode}-{unofficial_number}":
|
|
return
|
|
message = f" ! Cave Slug incorrect (unofficial): '{slug}' != '{areacode}-{unofficial_number}' {url=} in file {filename}"
|
|
if slug.lower() == f"{areacode}-{unofficial_number.lower()}":
|
|
message = f" ! Cave Slug capitalisation incorrect (unofficial): '{slug}' != '{areacode}-{unofficial_number}' {url=} in file {filename}"
|
|
DataIssue.objects.create(parser="caves", message=message, url=f"{cave.slug}_cave_edit/")
|
|
print(message)
|
|
|
|
global entrances_xslug
|
|
global caves_xslug
|
|
global areas_xslug
|
|
|
|
# Note: we want to overwrite a PENDING cave if we are now importing the 1623-xxx.html file for it
|
|
|
|
# Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
|
|
fn = settings.CAVEDESCRIPTIONS / filename
|
|
context = f"/cave_data/{filename}_edit" # the expo html editor, not the specialist cave data editor. oops.
|
|
|
|
# print(f" - Reading Cave from cave descriptions file {fn}")
|
|
if not fn.exists():
|
|
message = f" ! Cave_data file reading problem filename:'cave_data/{filename}'"
|
|
DataIssue.objects.create(parser="caves", message=message, url=context)
|
|
print(message)
|
|
return None
|
|
|
|
with open(fn) as f:
|
|
contents = f.read()
|
|
cavecontentslist = getXML(contents, "cave", maxItems=1, context=context)
|
|
|
|
if len(cavecontentslist) != 1:
|
|
message = f'! BAD CAVE DATA in "{filename}". More than one cave. Edit file manually, click.'
|
|
DataIssue.objects.create(parser="caves", message=message, url=context)
|
|
print(message)
|
|
return None
|
|
|
|
cavecontents = cavecontentslist[0]
|
|
slugs = getXML(cavecontents, "caveslug", maxItems=1, context=context)
|
|
if len(slugs) > 1:
|
|
message = f" ! - More than one slug for a cave: {cave}, slugs: {slugs}. Ignoring all except first."
|
|
DataIssue.objects.create(parser="caves", message=message, url=context)
|
|
print(message)
|
|
slug = slugs[0]
|
|
|
|
non_public = getXMLmax1("non_public")
|
|
official_name = getXMLmax1("official_name")
|
|
kataster_code = getXMLmax1("kataster_code")
|
|
kataster_number = getXMLmax1("kataster_number")
|
|
unofficial_number = getXMLmax1("unofficial_number")
|
|
explorers = getXMLmax1("explorers")
|
|
underground_description = getXMLmax1("underground_description")
|
|
equipment = getXMLmax1("equipment")
|
|
references = getXMLmax1("references")
|
|
survey = getXMLmax1("survey")
|
|
kataster_status = getXMLmax1("kataster_status")
|
|
underground_centre_line = getXMLmax1("underground_centre_line")
|
|
notes = getXMLmax1("notes")
|
|
length = getXMLmax1("length")
|
|
depth = getXMLmax1("depth")
|
|
extent = getXMLmax1("extent")
|
|
survex_file = getXMLmax1("survex_file")
|
|
description_file = getXMLmax1("description_file")
|
|
|
|
contextguess = f"/{slug[0:4]}/{slug}_cave_edit/" # guess as we havent read areacode yet
|
|
|
|
manual_edit = True
|
|
if not cave:
|
|
# we are parsing using databaseReset.py not an online edit
|
|
# we have already checked for uniqueness so we do not need the 'update' thinggy
|
|
manual_edit = False
|
|
|
|
try:
|
|
cave, state = Cave.objects.update_or_create(filename=filename) # replace with slug when CaveSlug tidied up
|
|
except:
|
|
print(" ! FAILED to get only one CAVE in db when updating using: " + filename)
|
|
kaves = Cave.objects.all().filter(filename=filename) # replace with slug when CaveSlug tidied up
|
|
for k in kaves:
|
|
message = " ! - DUPLICATES in db. kataster:" + str(k.kataster_number) + ", slug:" + str(k.slug())
|
|
DataIssue.objects.create(parser="caves", message=message, url=contextguess)
|
|
print(message)
|
|
for k in kaves:
|
|
if k.slug() is not None:
|
|
print(" ! - OVERWRITING this one: slug:" + str(k.slug()))
|
|
k.notes = "DUPLICATE kataster number found on import. Please fix\n" + k.notes
|
|
cave = k
|
|
|
|
# From here on the code applies to both edited and newly-imported caves (mostly!)
|
|
do_caveslugstuff() # needs cave!=None
|
|
|
|
areas = getXML(cavecontents, "area", context=contextguess) # can be multiple <area> tags
|
|
for area_slug in areas:
|
|
if area_slug in AREACODES: # ignore sub areas which are in another <area> tag
|
|
cave.areacode = area_slug
|
|
else:
|
|
cave.subarea = area_slug
|
|
if not cave.areacode:
|
|
if slug[0:4] in AREACODES:
|
|
cave.areacode = slug[0:4]
|
|
context = f"/{cave.areacode}/{slug}_cave_edit/"
|
|
|
|
|
|
cave.non_public=boolify(non_public)
|
|
cave.official_name=official_name[0]
|
|
cave.kataster_code=kataster_code[0]
|
|
cave.kataster_number=kataster_number[0]
|
|
cave.unofficial_number=unofficial_number[0]
|
|
cave.explorers=explorers[0]
|
|
cave.underground_description=underground_description[0]
|
|
cave.equipment=equipment[0]
|
|
cave.references=references[0]
|
|
cave.survey=survey[0]
|
|
cave.kataster_status=kataster_status[0]
|
|
cave.underground_centre_line=underground_centre_line[0]
|
|
cave.notes=notes[0]
|
|
cave.length=length[0]
|
|
cave.depth=depth[0]
|
|
cave.extent=extent[0]
|
|
cave.survex_file=survex_file[0]
|
|
cave.description_file=description_file[0]
|
|
# cave.url=url[0] # set algorithically:
|
|
cave.url = f"{cave.areacode}/{cave.number()}/{cave.number()}.html"
|
|
check_directory(cave.areacode, cave.number(), cave.url, cave)
|
|
|
|
check_slug(cave.areacode,cave.kataster_number, cave.unofficial_number, cave.url)
|
|
|
|
entrances = getXML(cavecontents, "entrance", context=context)
|
|
do_entrances()
|
|
# print(f"- {entrances_xslug=}")
|
|
if not entrances or len(entrances) < 1:
|
|
# missing entrance link in cave_data/1623-* .html file
|
|
set_dummy_entrance(slug[5:], slug, cave, msg="DUMMY: no entrances")
|
|
else:
|
|
do_entrances()
|
|
if manual_edit:
|
|
reload_entrances()
|
|
|
|
if survex_file[0]:
|
|
if not (Path(SURVEX_DATA) / survex_file[0]).is_file():
|
|
message = f' ! {slug:12} survex filename does not exist :LOSER:"{survex_file[0]}" in "{filename}"'
|
|
DataIssue.objects.create(parser="caves", message=message, url=f"/{slug[0:4]}/{slug}_cave_edit/")
|
|
print(message)
|
|
# else:
|
|
# print(f"{slug:12} survex filename UNSET")
|
|
|
|
|
|
if description_file[0]: # if not an empty string
|
|
message = f' - {slug:12} Note (not an error): complex description filename "{description_file[0]}" inside "cave_data/{filename}"'
|
|
DataIssue.objects.create(parser="caves ok", message=message, url=f"/{slug}_cave_edit/")
|
|
# print(message)
|
|
|
|
if not (Path(EXPOWEB) / description_file[0]).is_file():
|
|
message = f' ! {slug:12} description filename "{EXPOWEB}/{description_file[0]}" does not refer to a real file'
|
|
DataIssue.objects.create(parser="caves", message=message, url=f"/{slug}_cave_edit/")
|
|
print(message)
|
|
cave.save()
|
|
return cave
|
|
|
|
def readcaves():
|
|
"""Called from databaseReset mass importer.
|
|
Reads the xml-format HTML 'cave' files in the EXPOWEB repo, the survex files from the loser repo.
|
|
"""
|
|
# Pending is for those caves which do not have cave_data/1623-xxx.html XML files even though
|
|
# they exist and have surveys.
|
|
with transaction.atomic():
|
|
pending = set()
|
|
fpending = Path(CAVEDESCRIPTIONS, "pendingcaves.txt")
|
|
if fpending.is_file():
|
|
with open(fpending, "r") as fo:
|
|
cids = fo.readlines()
|
|
for cid in cids:
|
|
pcaveid = cid.split(";", 1)[0] # split on ";" and take the first bit
|
|
pcaveid = pcaveid.strip().rstrip("\n")
|
|
if pcaveid =="":
|
|
continue
|
|
pending.add(pcaveid)
|
|
|
|
with transaction.atomic():
|
|
print(" - Deleting Caves and Entrances")
|
|
# attempting to avoid MariaDB crash when doing this
|
|
try:
|
|
Area.objects.all().delete()
|
|
except:
|
|
pass
|
|
try:
|
|
Cave.objects.all().delete()
|
|
except:
|
|
pass
|
|
try:
|
|
Entrance.objects.all().delete()
|
|
except:
|
|
pass
|
|
|
|
# Clear the cave data issues and the caves as we are reloading
|
|
DataIssue.objects.filter(parser="areas").delete()
|
|
DataIssue.objects.filter(parser="caves").delete()
|
|
DataIssue.objects.filter(parser="caves ok").delete()
|
|
DataIssue.objects.filter(parser="aliases").delete()
|
|
DataIssue.objects.filter(parser="aliases ok").delete()
|
|
|
|
#DataIssue.objects.filter(parser="entrances").delete()
|
|
#DataIssue.objects.filter(parser="xEntrances").delete()
|
|
|
|
with transaction.atomic():
|
|
print(" - Reading Entrances from entrance descriptions xml files")
|
|
for filename in next(os.walk(ENTRANCEDESCRIPTIONS))[2]: # Should be a better way of getting a list of files
|
|
read_entrance(filename)
|
|
|
|
# Why is this needed ? Without it, we lose these DataIssues!
|
|
ent_issues = DataIssue.objects.filter(parser="entrances")
|
|
print(f" _ We now have {len(ent_issues)} entrance DataIssues")
|
|
|
|
with transaction.atomic():
|
|
print(" - Reading Caves from cave descriptions xml files")
|
|
for filename in next(os.walk(CAVEDESCRIPTIONS))[2]: # Should be a better way of getting a list of files
|
|
if filename.endswith(".html"):
|
|
read_cave(filename)
|
|
|
|
print(" - Setting up all the variously useful alias names")
|
|
GetCaveLookup()
|
|
|
|
print(" - Setting pending caves")
|
|
# Do this last, so we can detect if they are created and no longer 'pending'
|
|
|
|
with transaction.atomic():
|
|
for k in pending:
|
|
|
|
if k[0:3] == "162":
|
|
areacode = k[0:4]
|
|
number = k[5:]
|
|
url = f"{areacode}/{k[5:]}.html" # Note we are appending the .html to allow for offline websites
|
|
else:
|
|
areacode = "1623"
|
|
number = k
|
|
url = f"1623/{k}"
|
|
|
|
try:
|
|
do_pending_cave(k, number, url, areacode)
|
|
except:
|
|
message = f" ! Error. Cannot create pending cave, pending-id:{k} in area {areacode}"
|
|
DataIssue.objects.create(parser="caves", message=message)
|
|
print(message)
|
|
raise
|