forked from expo/troggle
Creating forgotten caves & better GetCaveLookup()
This commit is contained in:
Philip Sargent
parent
459ed11b58
commit
a60a495c83
5
.gitignore
vendored
5
.gitignore
vendored
@ -30,3 +30,8 @@ posnotfound
|
||||
troggle.sqlite-journal
|
||||
loadsurvexblks.log
|
||||
logbktrips.shelve
|
||||
cave-lookup.json
|
||||
svxblks.log
|
||||
svxlinear.log
|
||||
loadlogbk.log
|
||||
_1623.svx
|
||||
|
||||
@ -3,6 +3,7 @@ import os
|
||||
import datetime
|
||||
import logging
|
||||
import re
|
||||
import json
|
||||
from subprocess import call
|
||||
|
||||
from urllib.parse import urljoin
|
||||
@ -574,3 +575,169 @@ class PersonTrip(TroggleModel):
|
||||
|
||||
def __str__(self):
|
||||
return "%s (%s)" % (self.personexpedition, self.logbook_entry.date)
|
||||
|
||||
# lookup function modelled on GetPersonExpeditionNameLookup
|
||||
# repeated assignment each call, needs refactoring
|
||||
Gcavelookup = None
|
||||
def GetCaveLookup():
|
||||
global Gcavelookup
|
||||
if Gcavelookup:
|
||||
return Gcavelookup
|
||||
Gcavelookup = {"NONEPLACEHOLDER":None}
|
||||
for cave in Cave.objects.all():
|
||||
Gcavelookup[cave.official_name.lower()] = cave
|
||||
if cave.kataster_number:
|
||||
Gcavelookup[cave.kataster_number] = cave
|
||||
if cave.unofficial_number:
|
||||
Gcavelookup[cave.unofficial_number.lower()] = cave
|
||||
if cave.filename:
|
||||
# this is the slug - usually..
|
||||
Gcavelookup[cave.filename.replace(".html","").lower()] = cave
|
||||
if cave.slug():
|
||||
slug = cave.slug()
|
||||
Gcavelookup[slug.lower()] = cave
|
||||
# These are exact matches! edit to check for prefix only!
|
||||
# mostly taken from expoweb/noinfo/cave-number-index
|
||||
# and Becka's email of 25 may 2020 on new kataster numbers
|
||||
# this should be re-done as a JSON file upload
|
||||
Gcavelookup["1987-02"] = Gcavelookup["267"]
|
||||
Gcavelookup["1990-01"] = Gcavelookup["171"]
|
||||
Gcavelookup["1990-02"] = Gcavelookup["172"]
|
||||
Gcavelookup["1990-03"] = Gcavelookup["173"]
|
||||
Gcavelookup["1990-04"] = Gcavelookup["174"]
|
||||
Gcavelookup["1990-05"] = Gcavelookup["175"]
|
||||
Gcavelookup["1990-06"] = Gcavelookup["176"]
|
||||
Gcavelookup["1990-07"] = Gcavelookup["177"]
|
||||
Gcavelookup["1990-08"] = Gcavelookup["178"]
|
||||
Gcavelookup["1990-09"] = Gcavelookup["179"]
|
||||
Gcavelookup["1990-10"] = Gcavelookup["180"]
|
||||
Gcavelookup["1990-11"] = Gcavelookup["181"]
|
||||
Gcavelookup["1990-12"] = Gcavelookup["182"]
|
||||
Gcavelookup["1990-13"] = Gcavelookup["183"]
|
||||
Gcavelookup["1990-14"] = Gcavelookup["184"]
|
||||
Gcavelookup["1990-18"] = Gcavelookup["188"]
|
||||
Gcavelookup["1990-adam"] = Gcavelookup["225"]
|
||||
Gcavelookup["1993-01"] = Gcavelookup["200"]
|
||||
Gcavelookup["1996-02"] = Gcavelookup["224"]
|
||||
Gcavelookup["1996-03"] = Gcavelookup["223"]
|
||||
Gcavelookup["1996-04"] = Gcavelookup["222"]
|
||||
Gcavelookup["1996wk2"] = Gcavelookup["207"]
|
||||
Gcavelookup["1996wk3"] = Gcavelookup["208"]
|
||||
Gcavelookup["1996wk5"] = Gcavelookup["219"]
|
||||
Gcavelookup["1996wk6"] = Gcavelookup["218"]
|
||||
Gcavelookup["1996wk8"] = Gcavelookup["209"]
|
||||
Gcavelookup["1996wk11"] = Gcavelookup["268"]
|
||||
Gcavelookup["96wk11"] = Gcavelookup["268"]
|
||||
Gcavelookup["1998-01"] = Gcavelookup["201"]
|
||||
Gcavelookup["1998-03"] = Gcavelookup["210"]
|
||||
Gcavelookup["1999-03"] = Gcavelookup["204"]
|
||||
Gcavelookup["1999-04"] = Gcavelookup["230"]
|
||||
Gcavelookup["1999-10"] = Gcavelookup["162"]
|
||||
Gcavelookup["1999-bo-01"] = Gcavelookup["205"]
|
||||
Gcavelookup["1999-ob-01"] = Gcavelookup["205"]
|
||||
Gcavelookup["1999-ob-03"] = Gcavelookup["226"]
|
||||
Gcavelookup["1999-ob-04"] = Gcavelookup["227"]
|
||||
Gcavelookup["2000-01"] = Gcavelookup["231"]
|
||||
Gcavelookup["2000-03"] = Gcavelookup["214"]
|
||||
Gcavelookup["2000-04"] = Gcavelookup["220"]
|
||||
Gcavelookup["2000-05"] = Gcavelookup["215"]
|
||||
Gcavelookup["2000-06"] = Gcavelookup["216"]
|
||||
Gcavelookup["2000-07"] = Gcavelookup["217"]
|
||||
Gcavelookup["2000-09"] = Gcavelookup["234"]
|
||||
Gcavelookup["2000-aa-01"] = Gcavelookup["250"]
|
||||
Gcavelookup["2001-04"] = Gcavelookup["239"]
|
||||
Gcavelookup["2001-05"] = Gcavelookup["243"]
|
||||
Gcavelookup["2002-01"] = Gcavelookup["249"]
|
||||
Gcavelookup["2002-02"] = Gcavelookup["234"]
|
||||
Gcavelookup["2002-04"] = Gcavelookup["242"]
|
||||
Gcavelookup["2002-05"] = Gcavelookup["294"]
|
||||
Gcavelookup["2003-01"] = Gcavelookup["256"]
|
||||
Gcavelookup["2003-02"] = Gcavelookup["248"]
|
||||
Gcavelookup["2003-03"] = Gcavelookup["247"]
|
||||
Gcavelookup["2003-04"] = Gcavelookup["241"]
|
||||
Gcavelookup["2003-05"] = Gcavelookup["246"]
|
||||
Gcavelookup["2003-06"] = Gcavelookup["161"]
|
||||
Gcavelookup["2003-08"] = Gcavelookup["240"]
|
||||
Gcavelookup["2003-09"] = Gcavelookup["245"]
|
||||
Gcavelookup["2003-10"] = Gcavelookup["244"]
|
||||
Gcavelookup["2004-01"] = Gcavelookup["269"]
|
||||
Gcavelookup["2004-03"] = Gcavelookup["270"]
|
||||
Gcavelookup["2004-11"] = Gcavelookup["251"]
|
||||
Gcavelookup["2004-12"] = Gcavelookup["161"]
|
||||
Gcavelookup["2004-15"] = Gcavelookup["253"]
|
||||
Gcavelookup["2004-19"] = Gcavelookup["254"]
|
||||
Gcavelookup["2004-20"] = Gcavelookup["255"]
|
||||
Gcavelookup["2005-04"] = Gcavelookup["204"]
|
||||
Gcavelookup["2005-05"] = Gcavelookup["264"]
|
||||
Gcavelookup["2005-07"] = Gcavelookup["257"]
|
||||
Gcavelookup["2006-08"] = Gcavelookup["285"]
|
||||
Gcavelookup["2006-09"] = Gcavelookup["298"]
|
||||
Gcavelookup["2007-71"] = Gcavelookup["271"]
|
||||
Gcavelookup["2010-01"] = Gcavelookup["263"]
|
||||
Gcavelookup["2010-03"] = Gcavelookup["293"]
|
||||
Gcavelookup["2011-01"] = Gcavelookup["292"]
|
||||
Gcavelookup["2012-dd-05"] = Gcavelookup["286"]
|
||||
Gcavelookup["2012-ns-13"] = Gcavelookup["292"]
|
||||
Gcavelookup["2014-neo-01"] = Gcavelookup["273"]
|
||||
Gcavelookup["2014-sd-01"] = Gcavelookup["274"]
|
||||
Gcavelookup["2014-ms-14"] = Gcavelookup["287"]
|
||||
Gcavelookup["2015-mf-06"] = Gcavelookup["288"]
|
||||
Gcavelookup["2016-jb-01"] = Gcavelookup["289"]
|
||||
Gcavelookup["2017-pw-01"] = Gcavelookup["277"]
|
||||
Gcavelookup["2018-dm-07"] = Gcavelookup["359"]
|
||||
Gcavelookup["2017_cucc_24"] = Gcavelookup["291"]
|
||||
Gcavelookup["2017_cucc_23"] = Gcavelookup["295"]
|
||||
Gcavelookup["2017_cucc_28"] = Gcavelookup["290"]
|
||||
Gcavelookup["bs17"] = Gcavelookup["283"]
|
||||
|
||||
Gcavelookup["1976/b11"] = Gcavelookup["198"]
|
||||
Gcavelookup["1976/b8"] = Gcavelookup["197"]
|
||||
Gcavelookup["1976/b9"] = Gcavelookup["190"]
|
||||
Gcavelookup["b11"] = Gcavelookup["1976/b11"]
|
||||
Gcavelookup["b8"] = Gcavelookup["1976/b8"]
|
||||
Gcavelookup["b9"] = Gcavelookup["1976/b9"]
|
||||
|
||||
Gcavelookup["2011-01-bs30"] = Gcavelookup["190"]
|
||||
Gcavelookup["bs30"] = Gcavelookup["190"]
|
||||
Gcavelookup["2011-01"] = Gcavelookup["190"]
|
||||
|
||||
Gcavelookup["2002-x11"] = Gcavelookup["2005-08"]
|
||||
Gcavelookup["2002-x12"] = Gcavelookup["2005-07"]
|
||||
Gcavelookup["2002-x13"] = Gcavelookup["2005-06"]
|
||||
Gcavelookup["2002-x14"] = Gcavelookup["2005-05"]
|
||||
|
||||
Gcavelookup["kh"] = Gcavelookup["161"]
|
||||
Gcavelookup["161-kh"] = Gcavelookup["161"]
|
||||
Gcavelookup["204-steinBH"] = Gcavelookup["204"]
|
||||
Gcavelookup["stonebridge"] = Gcavelookup["204"]
|
||||
Gcavelookup["hauchhole"] = Gcavelookup["234"]
|
||||
Gcavelookup["hauch"] = Gcavelookup["234"]
|
||||
Gcavelookup["234-hauch"] = Gcavelookup["234"]
|
||||
Gcavelookup["tunnocks"] = Gcavelookup["258"]
|
||||
Gcavelookup["balcony"] = Gcavelookup["264"]
|
||||
Gcavelookup["balkon"] = Gcavelookup["264"]
|
||||
Gcavelookup["fgh"] = Gcavelookup["290"]
|
||||
Gcavelookup["gsh"] = Gcavelookup["291"]
|
||||
|
||||
Gcavelookup["homecoming"] = Gcavelookup["2018-dm-07"]
|
||||
Gcavelookup["99ob02"] = Gcavelookup["1999-ob-02"]
|
||||
|
||||
addmore = {}
|
||||
for id in Gcavelookup:
|
||||
addmore[id.replace("-","_")] = Gcavelookup[id]
|
||||
addmore[id.replace("_","-")] = Gcavelookup[id]
|
||||
Gcavelookup = {**addmore, **Gcavelookup}
|
||||
|
||||
addmore ={}
|
||||
for id in Gcavelookup:
|
||||
if not Gcavelookup[id]:
|
||||
pass
|
||||
elif Gcavelookup[id].kataster_number:
|
||||
#print(Gcavelookup[id], file=sys.stderr)
|
||||
addmore[id] = Gcavelookup[id].kataster_number
|
||||
elif Gcavelookup[id].unofficial_number:
|
||||
addmore[id] = Gcavelookup[id].unofficial_number.lower()
|
||||
with open("cave-lookup.json", 'w') as f:
|
||||
json.dump(addmore, f)
|
||||
|
||||
return Gcavelookup
|
||||
|
||||
@ -8,21 +8,44 @@ import troggle.core.models as models
|
||||
import troggle.core.models_caves as models_caves
|
||||
|
||||
def readcaves():
|
||||
# Clear the cave data issues as we are reloading
|
||||
# Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
|
||||
models.DataIssue.objects.filter(parser='caves').delete()
|
||||
# Clear the cave data issues as we are reloading
|
||||
models.DataIssue.objects.filter(parser='caves').delete()
|
||||
|
||||
# Do this first, so that these empty entries are overwritten as they get properly created.
|
||||
# For those caves which do not have XML files even though they exist and have surveys
|
||||
forgotten = ["2007-04", "2007-05", "2007-06", "2007-07", "2007-12", "2009-01", "2009-02",
|
||||
"2010-06", "2010-07", "2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06",
|
||||
"2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888",
|
||||
"2018-pf-01", "2018-pf-02", "haldenloch", "gruenstein"]
|
||||
for k in forgotten:
|
||||
try:
|
||||
cave = models_caves.Cave(
|
||||
unofficial_number = k,
|
||||
official_name = "Mislaid cave - created as empty object. No XML available at this time.",
|
||||
notes="_Survex file found in loser repo but no description in expoweb")
|
||||
if cave:
|
||||
print("{} {}".format(cave.unofficial_number, cave.official_name))
|
||||
cave.save()
|
||||
else:
|
||||
print("Failed to create cave {} ".format(k))
|
||||
except:
|
||||
message = " ! Forgotten cave error, slug: %s forgotten-id: %s" % (slug, k)
|
||||
models.DataIssue.objects.create(parser='caves', message=message)
|
||||
print(message)
|
||||
|
||||
# Note: these are HTML files in the EXPOWEB repo, not from the loser repo.
|
||||
area_1623 = models_caves.Area.objects.update_or_create(short_name = "1623", parent = None)
|
||||
area_1626 = models_caves.Area.objects.update_or_create(short_name = "1626", parent = None)
|
||||
print(" - Reading Entrances")
|
||||
for filename in next(os.walk(settings.ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
|
||||
if filename.endswith('.html'):
|
||||
readentrance(filename)
|
||||
print (" - Reading Caves")
|
||||
for filename in next(os.walk(settings.CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
|
||||
if filename.endswith('.html'):
|
||||
readcave(filename)
|
||||
|
||||
|
||||
area_1623 = models_caves.Area.objects.update_or_create(short_name = "1623", parent = None)
|
||||
area_1626 = models_caves.Area.objects.update_or_create(short_name = "1626", parent = None)
|
||||
print(" - Reading Entrances")
|
||||
#print "list of <Slug> <Filename>"
|
||||
for filename in next(os.walk(settings.ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
|
||||
if filename.endswith('.html'):
|
||||
readentrance(filename)
|
||||
print (" - Reading Caves")
|
||||
for filename in next(os.walk(settings.CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
|
||||
if filename.endswith('.html'):
|
||||
readcave(filename)
|
||||
|
||||
|
||||
def readentrance(filename):
|
||||
@ -204,7 +227,7 @@ def readcave(filename):
|
||||
message = " ! Entrance setting failure, slug: %s letter: %s" % (slug, letter)
|
||||
models.DataIssue.objects.create(parser='caves', message=message)
|
||||
print(message)
|
||||
|
||||
|
||||
|
||||
def getXML(text, itemname, minItems = 1, maxItems = None, printwarnings = True, context = ""):
|
||||
items = re.findall("<%(itemname)s>(.*?)</%(itemname)s>" % {"itemname": itemname}, text, re.S)
|
||||
|
||||
@ -12,7 +12,7 @@ from django.template.defaultfilters import slugify
|
||||
from django.utils.timezone import get_current_timezone, make_aware
|
||||
|
||||
from troggle.core.models import DataIssue, Expedition
|
||||
from troggle.core.models_caves import Cave, OtherCaveName, LogbookEntry, PersonTrip
|
||||
from troggle.core.models_caves import Cave, OtherCaveName, LogbookEntry, PersonTrip, GetCaveLookup
|
||||
from parsers.people import GetPersonExpeditionNameLookup
|
||||
from utils import save_carefully
|
||||
|
||||
@ -78,33 +78,6 @@ def GetTripCave(place):
|
||||
print(("No cave found for place " , place))
|
||||
return None
|
||||
|
||||
# lookup function modelled on GetPersonExpeditionNameLookup
|
||||
# repeated assignment each call, needs refactoring
|
||||
Gcavelookup = None
|
||||
def GetCaveLookup():
|
||||
global Gcavelookup
|
||||
if Gcavelookup:
|
||||
return Gcavelookup
|
||||
Gcavelookup = {"NONEPLACEHOLDER":None}
|
||||
for cave in Cave.objects.all():
|
||||
Gcavelookup[cave.official_name.lower()] = cave
|
||||
if cave.kataster_number:
|
||||
Gcavelookup[cave.kataster_number] = cave
|
||||
if cave.unofficial_number:
|
||||
Gcavelookup[cave.unofficial_number.lower()] = cave
|
||||
if cave.filename:
|
||||
# this is the slug - usually..
|
||||
Gcavelookup[cave.filename.replace(".html","").lower()] = cave
|
||||
# These are exact matches! edit to check for prefix only!
|
||||
Gcavelookup["tunnocks"] = Gcavelookup["258"]
|
||||
Gcavelookup["hauchhole"] = Gcavelookup["234"]
|
||||
Gcavelookup["KH"] = Gcavelookup["161"]
|
||||
Gcavelookup["Balcony"] = Gcavelookup["264"]
|
||||
Gcavelookup["Balkon"] = Gcavelookup["264"]
|
||||
Gcavelookup["FGH"] = Gcavelookup["290"]
|
||||
Gcavelookup["GSH"] = Gcavelookup["291"]
|
||||
Gcavelookup["Homecoming"] = Gcavelookup["2018-dm-07"]
|
||||
return Gcavelookup
|
||||
|
||||
|
||||
logentries = [] # the entire logbook for one year is a single object: a list of entries
|
||||
|
||||
Loading…
Reference in New Issue
Block a user