mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2025-12-15 12:37:10 +00:00
Removed class Area, use Cave.areacode Cave.subarea
This commit is contained in:
127
parsers/caves.py
127
parsers/caves.py
@@ -8,7 +8,7 @@ from pathlib import Path
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
|
||||
from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance, GetCaveLookup
|
||||
from troggle.core.models.caves import Cave, CaveAndEntrance, Entrance, GetCaveLookup
|
||||
from troggle.core.models.logbooks import CaveSlug
|
||||
from troggle.core.models.troggle import DataIssue
|
||||
from troggle.settings import CAVEDESCRIPTIONS, ENTRANCEDESCRIPTIONS, EXPOWEB, SURVEX_DATA
|
||||
@@ -99,36 +99,6 @@ def set_dummy_entrance(id, slug, cave, msg="DUMMY"):
|
||||
# DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.url}")
|
||||
# print(message)
|
||||
|
||||
def make_areas():
|
||||
print(" - Creating Areas 1623, 1624, 1627 and 1626")
|
||||
# This crashes on the server with MariaDB even though a null parent is explicitly allowed.
|
||||
area_1623 = Area.objects.create(short_name="1623", super=None)
|
||||
area_1623.save()
|
||||
area_1624 = Area.objects.create(short_name="1624", super=None)
|
||||
area_1624.save()
|
||||
area_1626 = Area.objects.create(short_name="1626", super=None)
|
||||
area_1626.save()
|
||||
area_1627 = Area.objects.create(short_name="1627", super=None)
|
||||
area_1627.save()
|
||||
|
||||
def get_area(areanum):
|
||||
"""Given the number as a string, return the area object
|
||||
"""
|
||||
a = Area.objects.all()
|
||||
if len(a) == 0:
|
||||
make_areas()
|
||||
|
||||
area = Area.objects.get(short_name="1623") # default
|
||||
|
||||
if areanum == "1623":
|
||||
area = Area.objects.get(short_name="1623")
|
||||
if areanum == "1624":
|
||||
area = Area.objects.get(short_name="1624")
|
||||
if areanum == "1626":
|
||||
area = Area.objects.get(short_name="1626")
|
||||
if areanum == "1627":
|
||||
area = Area.objects.get(short_name="1627")
|
||||
return area
|
||||
|
||||
def create_new_cave(svxpath, msg=None):
|
||||
"""This is called only when a new survex file is edited online which has a path on the
|
||||
@@ -145,29 +115,29 @@ def create_new_cave(svxpath, msg=None):
|
||||
print(f"parts {parts}, {a}, {caveid}")
|
||||
# double check
|
||||
if a[0:3] == "162":
|
||||
areanum = a[0:4]
|
||||
url = f"{areanum}/{caveid}.html" # Note we are appending the .html as we are believe in backwards compatability.
|
||||
#url = f"{areanum}/{a[5:]}.html" # This is original code, but a above is only defined as being 4 characters long, so it did not make sense and produced non unique urls
|
||||
areacode = a[0:4]
|
||||
url = f"{areacode}/{caveid}.html" # Note we are appending the .html as we are believe in backwards compatability.
|
||||
#url = f"{areacode}/{a[5:]}.html" # This is original code, but a above is only defined as being 4 characters long, so it did not make sense and produced non unique urls
|
||||
else:
|
||||
print(f"WARNING: parsers/caves/create_new_cave called with svxpath '{svxpath}'. Surely it should start 'caves-162*'? {msg}")
|
||||
areanum = "1623"
|
||||
areacode = "1623"
|
||||
url = f"1623/{caveid}.html"
|
||||
#url = f"1623/{k}.html" # This is original code, but a above is only defined as being 4 characters long, so it did not make sense and produced non unique urls
|
||||
|
||||
k = f"{areanum}-{caveid}"
|
||||
area = get_area(areanum)
|
||||
k = f"{areacode}-{caveid}"
|
||||
|
||||
caves = Cave.objects.filter(unofficial_number=caveid, area =areanum)
|
||||
|
||||
caves = Cave.objects.filter(unofficial_number=caveid, areacode =areacode)
|
||||
if caves:
|
||||
message = f" ! Already exists, caveid:{k} in area {areanum} {caves} - {msg}"
|
||||
message = f" ! Already exists, caveid:{k} in areacode {areacode} {caves} - {msg}"
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
return caves[0]
|
||||
|
||||
try:
|
||||
cave = do_pending_cave(k, caveid, url, area, msg)
|
||||
cave = do_pending_cave(k, caveid, url, areacode, msg)
|
||||
except:
|
||||
message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum} - {msg}"
|
||||
message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areacode} - {msg}"
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
raise
|
||||
@@ -175,14 +145,15 @@ def create_new_cave(svxpath, msg=None):
|
||||
# we know what the survex file is, we don't need to use the guess.
|
||||
# But this sets the survex file on he cave from the first one we find, not necessarily the best survex file for this cave
|
||||
cave.survex_file=survex_file
|
||||
cave.areacode=areacode
|
||||
cave.save()
|
||||
return cave
|
||||
|
||||
def do_ARGE_cave(slug, caveid, url, area, svxid):
|
||||
def do_ARGE_cave(slug, caveid, url, areacode, svxid):
|
||||
"""Only called by survex parser.
|
||||
Creates a new Cave object, but with abbreviated data as the survex file (from ARGE) is all we have.
|
||||
We already know the survex file.
|
||||
We already know that it doesn't exist.
|
||||
We already know that it doesn't exist... though there are bugs..
|
||||
"""
|
||||
|
||||
default_note = "This is an ARGE cave where we only have the survex file and no other information"
|
||||
@@ -203,35 +174,31 @@ def do_ARGE_cave(slug, caveid, url, area, svxid):
|
||||
|
||||
cave = Cave(
|
||||
unofficial_number=caveid.upper(),
|
||||
kataster_number=caveid.upper(), # should only set this if all digits
|
||||
underground_description="ARGE cave.",
|
||||
survex_file= f"{svxid}.svx",
|
||||
url=url,
|
||||
notes=default_note,
|
||||
areacode=areacode,
|
||||
)
|
||||
if cave:
|
||||
cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key.
|
||||
# cave.area.add(area)
|
||||
# cave.save() # crashes entire transaction with foreign key error.
|
||||
# The 'caves' list page uses the area__short_name to select for the area, so these ARGE caves do not appear.
|
||||
|
||||
# message = f" ! {slug:18} ARGE cave url: {url} "
|
||||
# DataIssue.objects.create(parser="caves", message=message, url=url)
|
||||
# print(message)
|
||||
|
||||
try: # Now create a cave slug ID
|
||||
CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False)
|
||||
except:
|
||||
message = f" ! {slug:11s} ARGE CaveSLUG create failure {caveid=} {url=} {area=} {svxid=}"
|
||||
message = f" ! {slug:11s} ARGE CaveSLUG create failure {caveid=} {url=} {areacode=} {svxid=}"
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
else:
|
||||
message = f" ! {slug:11s} ARGE cave create failure {caveid=} {url=} {area=} {svxid=}"
|
||||
message = f" ! {slug:11s} ARGE cave create failure {caveid=} {url=} {areacode=} {svxid=}"
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
return None
|
||||
|
||||
return cave
|
||||
|
||||
def do_pending_cave(slug, caveid, url, area, msg=None):
|
||||
def do_pending_cave(slug, caveid, url, areacode, msg=None):
|
||||
"""
|
||||
default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists
|
||||
in expoweb/cave_data/1623-"k".html
|
||||
@@ -250,16 +217,16 @@ def do_pending_cave(slug, caveid, url, area, msg=None):
|
||||
else:
|
||||
id = Path(k)
|
||||
|
||||
survex_file = f"caves-{area.short_name}/{id}/{id}.svx"
|
||||
survex_file = f"caves-{areacode}/{id}/{id}.svx"
|
||||
if Path(settings.SURVEX_DATA, survex_file).is_file():
|
||||
return survex_file
|
||||
else:
|
||||
survex_file = f"caves-{area.short_name}/{id}.svx"
|
||||
survex_file = f"caves-{areacode}/{id}.svx"
|
||||
if Path(settings.SURVEX_DATA, survex_file).is_file():
|
||||
return survex_file
|
||||
|
||||
survex_file = ""
|
||||
d = Path(settings.SURVEX_DATA, f"caves-{area.short_name}/{id}")
|
||||
d = Path(settings.SURVEX_DATA, f"caves-{areacode}/{id}")
|
||||
if d.is_dir():
|
||||
prime_suspect = ""
|
||||
dir = d.iterdir()
|
||||
@@ -334,11 +301,10 @@ def do_pending_cave(slug, caveid, url, area, msg=None):
|
||||
survex_file=survex_file,
|
||||
url=url,
|
||||
notes=default_note,
|
||||
areacode=areacode,
|
||||
)
|
||||
if cave:
|
||||
cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key.
|
||||
cave.area.add(area)
|
||||
cave.save()
|
||||
message = f" ! {slug:18} Pending cave write-up url: {url} - {msg}"
|
||||
DataIssue.objects.create(parser="caves", message=message, url=url)
|
||||
print(message)
|
||||
@@ -555,7 +521,7 @@ def read_cave(filename, cave=None):
|
||||
# print(f"! Entrance {eslug}")
|
||||
if eslug.endswith('a b'):
|
||||
message = f' - Entrance has weird name slug:"{eslug}" cave:"{cave}" caveslug:"{slug}" filename:"cave_data/{filename}"'
|
||||
DataIssue.objects.create(parser="xEntrances", message=message, url=f"{cave.area}/{cave.area}-{cave.url}_cave_edit/")
|
||||
DataIssue.objects.create(parser="xEntrances", message=message, url=f"{cave.areacode}/{cave.areacode}-{cave.url}_cave_edit/")
|
||||
# print(message)
|
||||
|
||||
letter = getXML(e, "letter", maxItems=1, context=context)[0]
|
||||
@@ -566,7 +532,7 @@ def read_cave(filename, cave=None):
|
||||
if letter.lower() not in list(string.ascii_lowercase):
|
||||
letter = "x"
|
||||
message = f"- Warning - Empty 'letter' field for '{eslug}' in multiple-entrance cave '{cave}', setting to {letter}."
|
||||
DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.area}/{cave.area}-{cave.url}_cave_edit/")
|
||||
DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.areacode}/{cave.areacode}-{cave.url}_cave_edit/")
|
||||
print(message)
|
||||
|
||||
if len(entrances) == 1 and not eslug: # may be empty: <entranceslug></entranceslug>
|
||||
@@ -584,13 +550,13 @@ def read_cave(filename, cave=None):
|
||||
entrances_xslug[eslug] = entrance
|
||||
except:
|
||||
message = f"! Fail entrance loading {eslug} /entrance_data/{eslug} file does not exist or loading it failed."
|
||||
DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.area}/{cave.area}-{cave.url}_cave_edit/")
|
||||
DataIssue.objects.create(parser="entrances", message=message, url=f"{cave.areacode}/{cave.areacode}-{cave.url}_cave_edit/")
|
||||
print(message)
|
||||
return
|
||||
|
||||
if eslug != f"{entrance}":
|
||||
message = f"eslug {eslug} using different entrance {entrance} to set CaveAndEntrance"
|
||||
DataIssue.objects.create(parser="xEntrances", message=message, url=f"{cave.area}/{cave.area}-{cave.url}_cave_edit/")
|
||||
DataIssue.objects.create(parser="xEntrances", message=message, url=f"{cave.areacode}/{cave.areacode}-{cave.url}_cave_edit/")
|
||||
print(message)
|
||||
try:
|
||||
# this fails if there is not an unambiguous letter set.
|
||||
@@ -728,27 +694,13 @@ def read_cave(filename, cave=None):
|
||||
cave.description_file=description_file[0]
|
||||
cave.url=url[0]
|
||||
|
||||
areas = getXML(cavecontents, "area", context=context)
|
||||
cave.area.clear() # Deletes all links to areas in db
|
||||
areas = getXML(cavecontents, "area", context=context) # can be multiple <area> tags
|
||||
for area_slug in areas:
|
||||
if area_slug in areas_xslug:
|
||||
newArea = areas_xslug[area_slug]
|
||||
if area_slug in ["1623", "1624", "1626", "1627"]: # ignore sub areas which are in another <area> tag
|
||||
cave.areacode = area_slug
|
||||
else:
|
||||
areas_new = Area.objects.filter(short_name=area_slug)
|
||||
if areas_new:
|
||||
newArea = areas_new[0] # just the first one we find, but we are going to clean up Areas anyway
|
||||
else:
|
||||
# Area not seen before. SHould not happen with manual edit
|
||||
if manual_edit:
|
||||
message = f" ! Cave edit failure due to unrecognised Area: {area_slug[0]}, skipping this field edit. "
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
# super value is highly dodgy
|
||||
newArea = Area(short_name=area_slug, super=Area.objects.get(short_name="1623"))
|
||||
newArea.save()
|
||||
areas_xslug[area_slug] = newArea
|
||||
cave.area.add(newArea)
|
||||
|
||||
cave.subarea = area_slug
|
||||
|
||||
entrances = getXML(cavecontents, "entrance", context=context)
|
||||
do_entrances()
|
||||
# print(f"- {entrances_xslug=}")
|
||||
@@ -834,8 +786,6 @@ def readcaves():
|
||||
#DataIssue.objects.filter(parser="xEntrances").delete()
|
||||
|
||||
with transaction.atomic():
|
||||
area = get_area("1623")
|
||||
|
||||
print(" - Reading Entrances from entrance descriptions xml files")
|
||||
for filename in next(os.walk(ENTRANCEDESCRIPTIONS))[2]: # Should be a better way of getting a list of files
|
||||
read_entrance(filename)
|
||||
@@ -860,19 +810,18 @@ def readcaves():
|
||||
for k in pending:
|
||||
|
||||
if k[0:3] == "162":
|
||||
areanum = k[0:4]
|
||||
areacode = k[0:4]
|
||||
number = k[5:]
|
||||
url = f"{areanum}/{k[5:]}.html" # Note we are appending the .htm to allow for offline websites
|
||||
url = f"{areacode}/{k[5:]}.html" # Note we are appending the .htm to allow for offline websites
|
||||
else:
|
||||
areanum = "1623"
|
||||
areacode = "1623"
|
||||
number = k
|
||||
url = f"1623/{k}"
|
||||
|
||||
area = get_area(areanum)
|
||||
try:
|
||||
do_pending_cave(k, number, url, area)
|
||||
do_pending_cave(k, number, url, areacode)
|
||||
except:
|
||||
message = f" ! Error. Cannot create pending cave, pending-id:{k} in area {areanum}"
|
||||
message = f" ! Error. Cannot create pending cave, pending-id:{k} in area {areacode}"
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
raise
|
||||
|
||||
Reference in New Issue
Block a user