mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2025-03-13 05:41:47 +00:00
Now creating ARGE caves properly
This commit is contained in:
parent
327b1923b0
commit
ad272fab3b
@ -178,13 +178,67 @@ def create_new_cave(svxpath, msg=None):
|
||||
cave.save()
|
||||
return cave
|
||||
|
||||
def do_pending_cave(k, caveid, url, area, msg=None):
|
||||
def do_ARGE_cave(slug, caveid, url, area, svxid):
|
||||
"""Only called by survex parser.
|
||||
Creates a new Cave object, but with abbreviated data as the survex file (from ARGE) is all we have.
|
||||
We already know the survex file.
|
||||
We already know that it doesn't exist.
|
||||
"""
|
||||
|
||||
default_note = "This is an ARGE cave where we only have the survex file and no other information"
|
||||
|
||||
urltest = Cave.objects.filter(url=url)
|
||||
if urltest:
|
||||
message = f" ! Cave {urltest[0]} already exists with this url {url}. Can't create new ARGE cave {slug}"
|
||||
DataIssue.objects.create(parser="caves", message=message, url=url)
|
||||
print(message)
|
||||
return urltest[0]
|
||||
|
||||
numtest = Cave.objects.filter(unofficial_number=caveid.upper())
|
||||
if numtest:
|
||||
message = f" ! Cave {numtest[0]} already exists with this unofficial_number {caveid.upper()}. Can't create new ARGE cave {slug}"
|
||||
DataIssue.objects.create(parser="caves", message=message, url=url)
|
||||
print(message)
|
||||
return numtest[0]
|
||||
|
||||
cave = Cave(
|
||||
unofficial_number=caveid.upper(),
|
||||
underground_description="ARGE cave.",
|
||||
survex_file= f"{svxid}.svx",
|
||||
url=url,
|
||||
notes=default_note,
|
||||
)
|
||||
if cave:
|
||||
cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key.
|
||||
# cave.area.add(area)
|
||||
# cave.save() # crashes entire transaction with foreign key error.
|
||||
# The 'caves' list page uses the area__short_name to select for the area, so these ARGE caves do not appear.
|
||||
|
||||
# message = f" ! {slug:18} ARGE cave url: {url} "
|
||||
# DataIssue.objects.create(parser="caves", message=message, url=url)
|
||||
# print(message)
|
||||
|
||||
try: # Now create a cave slug ID
|
||||
CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False)
|
||||
except:
|
||||
message = f" ! {slug:11s} ARGE CaveSLUG create failure {caveid=} {url=} {area=} {svxid=}"
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
else:
|
||||
message = f" ! {slug:11s} ARGE cave create failure {caveid=} {url=} {area=} {svxid=}"
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
|
||||
return cave
|
||||
|
||||
def do_pending_cave(slug, caveid, url, area, msg=None):
|
||||
"""
|
||||
default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists
|
||||
in expoweb/cave_data/1623-"k".html
|
||||
|
||||
Note that at this point in importing the data we have not yet seen the survex files, so we can't
|
||||
Note that at this point (parsing caves) in importing the data we have not yet seen the survex files, so we can't
|
||||
look inside the relevant survex file to find the year and so we con't provide helpful links.
|
||||
This also gets called when parsing survex files, when we do have this info.
|
||||
"""
|
||||
|
||||
def get_survex_file(k):
|
||||
@ -222,12 +276,10 @@ def do_pending_cave(k, caveid, url, area, msg=None):
|
||||
# print(message)
|
||||
return survex_file
|
||||
|
||||
slug = k
|
||||
|
||||
g = GetCaveLookup()
|
||||
with transaction.atomic():
|
||||
if slug in g:
|
||||
message = f" ! {k:18} cave listed in pendingcaves.txt already exists. - {msg}"
|
||||
message = f" ! {slug:18} cave listed in pendingcaves.txt already exists. - {msg}"
|
||||
DataIssue.objects.create(parser="caves", message=message, url=url)
|
||||
print(message)
|
||||
return
|
||||
@ -274,7 +326,7 @@ def do_pending_cave(k, caveid, url, area, msg=None):
|
||||
print(message)
|
||||
return urltest[0]
|
||||
|
||||
survex_file = get_survex_file(k)
|
||||
survex_file = get_survex_file(slug)
|
||||
|
||||
cave = Cave(
|
||||
unofficial_number=caveid.upper(),
|
||||
@ -287,18 +339,18 @@ def do_pending_cave(k, caveid, url, area, msg=None):
|
||||
cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key.
|
||||
cave.area.add(area)
|
||||
cave.save()
|
||||
message = f" ! {k:18} Pending cave write-up url: {url} - {msg}"
|
||||
message = f" ! {slug:18} Pending cave write-up url: {url} - {msg}"
|
||||
DataIssue.objects.create(parser="caves", message=message, url=url)
|
||||
print(message)
|
||||
|
||||
try: # Now create a cave slug ID
|
||||
CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False)
|
||||
except:
|
||||
message = f" ! {k:11s} PENDING CaveSLUG {slug} create failure - {msg}"
|
||||
message = f" ! {slug:11s} PENDING CaveSLUG {slug} create failure - {msg}"
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
else:
|
||||
message = f" ! {k:11s} PENDING cave create failure - {msg}"
|
||||
message = f" ! {slug:11s} PENDING cave create failure - {msg}"
|
||||
DataIssue.objects.create(parser="caves", message=message)
|
||||
print(message)
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
|
||||
from troggle.core.models.wallets import Wallet
|
||||
from troggle.core.models.troggle import DataIssue, Expedition
|
||||
from troggle.core.utils import chaosmonkey, get_process_memory
|
||||
from troggle.parsers.caves import create_new_cave
|
||||
from troggle.parsers.caves import create_new_cave, do_ARGE_cave
|
||||
from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner
|
||||
|
||||
"""Imports the tree of survex files following from a defined root .svx file
|
||||
@ -1175,10 +1175,10 @@ class LoadingSurvex:
|
||||
for key in self.caveslist:
|
||||
cave = self.caveslist[key]
|
||||
if type(cave) != Cave:
|
||||
print(f"BAD CAVE TYPE '{key}' -- {type(cave)}'{cave}'")
|
||||
print(f"BAD CAVE TYPE '{type(cave)}'{cave}' -- {key}'")
|
||||
for key in self.caveslist:
|
||||
cave = self.caveslist[key]
|
||||
print(f"{key} -- Cave<{cave}>")
|
||||
print(f"Cave<{cave}> -- {key}")
|
||||
|
||||
for i in self.ignoreprefix:
|
||||
if cavepath.lower().startswith(i):
|
||||
@ -1192,19 +1192,23 @@ class LoadingSurvex:
|
||||
# rx_cave = re.compile(r"(?i)caves-(\d\d\d\d)/([-\d\w]+|\d\d\d\d-?\w+-\d+)")
|
||||
path_match = self.rx_cave.search(cavepath)
|
||||
if path_match:
|
||||
sluggy = f"{path_match.group(1)}-{path_match.group(2)}"
|
||||
area = path_match.group(1)
|
||||
caveid = path_match.group(2)
|
||||
sluggy = f"{area}-{caveid}"
|
||||
seek = [sluggy, sluggy.replace("1623-","")] # to catch '2023-kt-02' etc
|
||||
for s in seek:
|
||||
if s in self.caveslist:
|
||||
self.caveslist[cavepath] = self.caveslist[s] # set "caves-1626/107/107" as index to cave 1626-107
|
||||
return self.caveslist[s]
|
||||
|
||||
if (cavepath.startswith("caves-1624") or cavepath.startswith("caves-1626")):
|
||||
url = f"/survexfile/{svxid}.svx"
|
||||
return do_ARGE_cave(sluggy, caveid, url, area, svxid)
|
||||
|
||||
cave = create_new_cave(cavepath, f"Make cave found in survex file {svxid}") # uses the pending code to create pending cave descriptions
|
||||
self.caveslist[cavepath] = cave
|
||||
message = f"\n ! MAKING cave {sluggy} for {cavepath=} {svxid=} (not reporting this for 1624 or 1626)"
|
||||
message = f"\n ! MAKING cave {sluggy} for {cavepath=} {svxid=}"
|
||||
# stash_data_issue(parser="survex", message=message, url="/survexfile/{svxid}.svx", sb=(svxid))
|
||||
if not (cavepath.startswith("caves-1624") or cavepath.startswith("caves-1626")):
|
||||
print(message, file=sys.stderr)
|
||||
return cave
|
||||
else:
|
||||
path_match = rx_svxcollection.search(svxid)
|
||||
@ -1219,39 +1223,6 @@ class LoadingSurvex:
|
||||
print("\n" + message, file=sys.stderr)
|
||||
stash_data_issue(parser="survex", message=message, url="{svxid}.svx", sb=(svxid))
|
||||
return None
|
||||
|
||||
def is_it_already_pending(self, headpath, includelabel, depth):
|
||||
"""Ignore surface, kataser and gpx *include survex files"""
|
||||
if not self.pending:
|
||||
self.pending = set()
|
||||
fpending = Path(settings.CAVEDESCRIPTIONS, "pendingcaves.txt")
|
||||
if fpending.is_file():
|
||||
with open(fpending, "r") as fo:
|
||||
cids = fo.readlines()
|
||||
for cid in cids:
|
||||
id = cid.strip().rstrip("\n").upper()
|
||||
if cid.startswith("162"):
|
||||
self.pending.add(id)
|
||||
else:
|
||||
self.pending.add("1623-" + id)
|
||||
|
||||
if headpath in self.ignorenoncave:
|
||||
message = f" - {headpath} is <ignorenoncave> (while creating '{includelabel}' sfile & sdirectory)"
|
||||
# print("\n"+message)
|
||||
# print("\n"+message,file=sys.stderr)
|
||||
return True
|
||||
|
||||
caveid = f"{headpath[6:10]}-{headpath[11:]}".upper()
|
||||
if caveid in self.pending:
|
||||
# Yes we didn't find this cave, but we know it is a pending one. So not an error.
|
||||
print(f'! ALREADY PENDING caveid {caveid}',file=sys.stderr)
|
||||
return True
|
||||
id = caveid[5:]
|
||||
if id in self.pending:
|
||||
print(f"! ALREADY PENDING id {id}", file=sys.stderr)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def LoadSurvexFile(self, svxid):
|
||||
"""Creates SurvexFile in the database, and SurvexDirectory if needed
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user