xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Fixing multiple caves with same kataser no

Browse Source
This commit is contained in:
Philip Sargent 2021-03-28 23:47:47 +01:00
parent 0ecaa9b8ee
commit 623483f3b1
5 changed files with 97 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
core/models_caves.py
View File

@ -5,6 +5,7 @@ import logging
import re
import json
from subprocess import call
from collections import defaultdict
from urllib.parse import urljoin
@ -21,6 +22,7 @@ from django.template import Context, loader
from troggle.core.models import TroggleModel, Person, Expedition
from troggle.core.models_survex import SurvexStation
from troggle.core.models import DataIssue
class Area(TroggleModel):
short_name = models.CharField(max_length=100)
@ -35,7 +37,7 @@ class Area(TroggleModel):
return str(self.short_name)
def kat_area(self):
if self.short_name in ["1623", "1626"]:
if self.short_name in ["1623", "1626", "1624", "1627"]:
return self.short_name
elif self.parent:
return self.parent.kat_area()
@ -515,30 +517,47 @@ def get_scan_path(instance, filename):
return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg')
Gcavelookup = None
Gcave_count = None
def GetCaveLookup():
"""lookup function modelled on GetPersonExpeditionNameLookup
repeated assignment each call, needs refactoring
Does NOT detect duplicates! Needs fixing.
Needs to be a proper funciton that raises an exception if there is a duplicate.
OR we could set it to return None if there are duplictes, and require the caller to
fall back on doing the actual database query it wants rathe rthna using this cache shortcut
"""
global Gcavelookup
if Gcavelookup:
return Gcavelookup
Gcavelookup = {"NONEPLACEHOLDER":None}
Gcavelookup = {"NONEPLACEHOLDER": None}
global Gcave_count
Gcave_count = defaultdict(int) # sets default value to int(0)
for cave in Cave.objects.all():
Gcavelookup[cave.official_name.lower()] = cave
key = cave.official_name.lower()
Gcavelookup[key] = cave
Gcave_count[key] += 1
if cave.kataster_number:
Gcavelookup[cave.kataster_number] = cave
Gcavelookup[cave.kataster_number] = cave # DUPLICATE as we have 1623-55 and 1626-55
Gcave_count[cave.kataster_number] += 1
if cave.unofficial_number:
Gcavelookup[cave.unofficial_number.lower()] = cave
Gcave_count[cave.unofficial_number.lower()] += 1
if cave.filename:
# this is the slug - usually..
Gcavelookup[cave.filename.replace(".html","").lower()] = cave
Gcave_count[cave.filename.replace(".html","").lower()] += 1
if cave.slug():
slug = cave.slug()
Gcavelookup[slug.lower()] = cave
Gcave_count[slug.lower()] += 1
# These are exact matches! edit to check for prefix only!
# mostly taken from expoweb/noinfo/cave-number-index
# and Becka's email of 25 may 2020 on new kataster numbers
# this should be re-done as a JSON file upload
# These might alse create more duplicate entries, so re-write it to check
Gcavelookup["1987-02"] = Gcavelookup["267"]
Gcavelookup["1990-01"] = Gcavelookup["171"]
Gcavelookup["1990-02"] = Gcavelookup["172"]
@ -624,7 +643,7 @@ def GetCaveLookup():
Gcavelookup["2016-jb-01"] = Gcavelookup["289"]
Gcavelookup["2017-pw-01"] = Gcavelookup["277"]
Gcavelookup["2018-dm-07"] = Gcavelookup["359"]
Gcavelookup["2017_cucc_24"] = Gcavelookup["291"]
Gcavelookup["2017_cucc_24"] = Gcavelookup["291"] # note _ not - here
Gcavelookup["2017_cucc_23"] = Gcavelookup["295"]
Gcavelookup["2017_cucc_28"] = Gcavelookup["290"]
Gcavelookup["bs17"] = Gcavelookup["283"]
@ -679,5 +698,12 @@ def GetCaveLookup():
addmore[id] = Gcavelookup[id].unofficial_number.lower()
with open("cave-lookup.json", 'w') as f:
json.dump(addmore, f)
for c in Gcave_count:
if Gcave_count[c] > 1:
message = " ** Duplicate cave id: {}:{}:{}".format(Gcave_count[c], Gcavelookup[c], c)
#print(message)
#DataIssue.objects.create(parser='caves', message=message)
# logdataissues[Gcavelookup[c]]=message # pending troggle-wide issues logging system
return Gcavelookup

11
core/view_surveys.py
View File

@ -15,17 +15,20 @@ import parsers.surveys
'''Some of these views serve files as binary blobs, and simply set the mime type based on the file extension,
as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked
by looking inside the file before being served.
need to check if inavlid query string is invalid, or produces multiple replies
and render a user-friendly error page.
'''
def surveyscansfolder(request, path):
#print [ s.walletname for s in ScansFolder.objects.all() ]
scansfolder = ScansFolder.objects.get(walletname=urlunquote(path))
scansfolder = ScansFolder.objects.get(walletname=urlunquote(path)) # need to check if inavlid query string and produce friendly error
return render(request, 'scansfolder.html', { 'scansfolder':scansfolder, 'settings': settings })
def surveyscansingle(request, path, file):
'''sends a single binary file to the user,
'''
scansfolder = ScansFolder.objects.get(walletname=urlunquote(path))
scansfolder = ScansFolder.objects.get(walletname=urlunquote(path)) # need to check if inavlid query string and produce friendly error
singlescan = SingleScan.objects.get(scansfolder=scansfolder, name=file)
# print(" - surveyscansingle {}:{}:{}:".format(path, file, flatviews.getmimetype(file)))
return HttpResponse(content=open(singlescan.ffile,"rb"), content_type=flatviews.getmimetype(file)) # any type of image
@ -44,12 +47,12 @@ def tunneldata(request):
def tunnelfilesingle(request, path):
'''sends a single binary file to the user, We should have a renderer that syntax-colours this Tunnel xml
'''
tunnelfile = TunnelFile.objects.get(tunnelpath=urlunquote(path))
tunnelfile = TunnelFile.objects.get(tunnelpath=urlunquote(path)) # need to check if inavlid query string and produce friendly error
tfile = Path(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
return HttpResponse(content=open(tfile), content_type="text/xhtml") # for display not download
def tunnelfileupload(request, path):
tunnelfile = TunnelFile.objects.get(tunnelpath=urlunquote(path))
tunnelfile = TunnelFile.objects.get(tunnelpath=urlunquote(path)) # need to check if inavlid query string and produce friendly error
tfile = Path(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"]

54
core/views_caves.py
View File

@ -11,11 +11,12 @@ from django.conf import settings
from django.urls import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
import troggle.settings as settings
import troggle.core.models as models
from troggle.core.models import Expedition
from troggle.core.models_caves import CaveSlug, Cave, CaveAndEntrance, QM, EntranceSlug, Entrance, Area, SurvexStation
#import troggle.core.models as models
from troggle.core.models import Expedition, DataIssue
from troggle.core.models_caves import CaveSlug, Cave, CaveAndEntrance, QM, EntranceSlug, Entrance, Area, SurvexStation, GetCaveLookup
from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
from troggle.helper import login_required_if_public
@ -41,14 +42,14 @@ class MapLocations(object):
k = ent.caveandentrance_set.all()[0].cave
except:
message = " ! Failed to get Cave linked to Entrance:{} from:{} best:{}".format(ent.name, ent.filename, ent.best_station())
models.DataIssue.objects.create(parser='entrances', message=message)
DataIssue.objects.create(parser='entrances', message=message)
print(message)
raise
try:
areaName = k.getArea().short_name
except:
message = " ! Failed to get Area on cave '{}' linked to Entrance:{} from:{} best:{}".format(cave, ent.name, ent.filename, ent.best_station())
models.DataIssue.objects.create(parser='entrances', message=message)
DataIssue.objects.create(parser='entrances', message=message)
print(message)
raise
self.p.append((ent.best_station(), "%s-%s" % (areaName, str(ent)[5:]), ent.needs_surface_work(), str(ent)))
@ -58,13 +59,23 @@ class MapLocations(object):
return "{} map locations".format(len(self.p))
def getCave(cave_id):
"""Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm."""
# REPLACE OR MERGE this with other functions that do exactly th same thing!
'''Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm.
TO DO: search GCavelookup first, which should raise a MultpleObjectsReturned exception if there
are duplicates'''
try:
cave = Cave.objects.get(kataster_number=cave_id)
except Cave.MultipleObjectsReturned as ex:
raise MultipleObjectsReturned("Duplicate kataster number") from ex # propagate this up
except Cave.DoesNotExist:
cave = Cave.objects.get(unofficial_number=cave_id)
return cave
Gcavelookup = GetCaveLookup() # dictionary makes strings to Cave objects
if cave_id in Gcavelookup:
return Gcavelookup[cave_id]
else:
raise ObjectDoesNotExist("No cave found with this identifier in any id field")
except:
raise ObjectDoesNotExist("No cave found with this identifier in any id field")
def pad5(x):
return "0" * (5 -len(x.group(0))) + x.group(0)
@ -75,7 +86,7 @@ def numericalcmp(x, y):
def caveKey(x):
"""python3 function for sort. Done in a hurry.
Note that cave kataster numbers are not generally integers.
Note that cave kataster numbers are not always integers.
This needs to be fixed make a decent sort order.
"""
if not x.kataster_number:
@ -107,7 +118,12 @@ def caveindex(request):
return render(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':getnotablecaves(), 'cavepage': True})
def cave3d(request, cave_id=''):
cave = getCave(cave_id)
try:
cave = getCave(cave_id)
except Cave.MultipleObjectsReturned: # entirely the wrong action, REPLACE with the right display
caves = Cave.objects.filter(kataster_number=cave_id)
return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves })
survexfilename = settings.SURVEX_DATA + cave.survex_file
threedfilename = settings.THREEDCACHEDIR + '%s.3d' % cave_id
if True or os.path.getmtime(survexfilename) > os.path.getmtime(threedfilename):
@ -121,9 +137,16 @@ def cave3d(request, cave_id=''):
return response
def cave(request, cave_id='', offical_name=''):
cave=getCave(cave_id)
try:
cave=getCave(cave_id)
except MultipleObjectsReturned:
caves = Cave.objects.filter(kataster_number=cave_id)
return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves }) # not the right template, needs a specific one
except ObjectDoesNotExist:
return render(request, 'svxcavesingle404.html', {'settings': settings, "cave":cave_id })
if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
return render(request,'nonpublic.html', {'instance': cave, 'cavepage': True, 'cave_id': cave_id})
return render(request, 'nonpublic.html', {'instance': cave, 'cavepage': True, 'cave_id': cave_id})
else:
return render(request,'cave.html', {'settings': settings, 'cave': cave, 'cavepage': True, 'cave_id': cave_id})
@ -254,8 +277,11 @@ def editEntrance(request, caveslug, slug=None):
def qm(request,cave_id,qm_id,year,grade=None):
year=int(year)
try:
qm=getCave(cave_id).get_QMs().get(number=qm_id,found_by__date__year=year)
qm=getCave(cave_id).get_QMs().get(number=qm_id,found_by__date__year=year)
return render(request,'qm.html',locals())
except Cave.MultipleObjectsReturned: # entirely the wrong action, REPLACE with the right display
caves = Cave.objects.filter(kataster_number=cave_id)
return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves })
except QM.DoesNotExist:
url=urllib.parse.urljoin(settings.URL_ROOT, r'/admin/core/qm/add/'+'?'+ r'number=' + qm_id)

2
core/views_survex.py
View File

@ -377,6 +377,8 @@ def survexcavesingle(request, survex_cave):
'''parsing all the survex files of a single cave and showing that it's consistent and can find all
the files and people. Should explicity fix the kataster number thing.
kataster numbers are not unique across areas. Fix this.
Should use getCave() from models_caves
'''
sc = survex_cave
try:

47
flatpages/views.py
View File

@ -13,7 +13,6 @@ from django.views.decorators.csrf import ensure_csrf_cookie
import django.forms as forms
from troggle.helper import login_required_if_public
#from troggle.flatpages.models import Redirect, EntranceRedirect
from troggle.core.models_caves import Cave
import troggle.core.views_caves
import troggle.settings as settings
@ -30,14 +29,14 @@ def expofilessingle(request, filepath):
fn = Path(settings.EXPOFILES,filepath)
if fn.is_dir():
return expofilesdir(request, Path(fn), Path(filepath))
print(" - expofilessingle {}:{}:{}:".format(filepath, fn, getmimetype(fn)))
# print(" - expofilessingle {}:{}:{}:".format(filepath, fn, getmimetype(fn)))
return HttpResponse(content=open(fn, "rb"),content_type=getmimetype(filepath)) # any file
def expofilesdir(request, dirpath, filepath):
'''does a directory display. If there is an index.html file we should display that.
- dirpath is a Path() and it does not have /expofiles/ in it
'''
print(" - expofilesdir {}".format(dirpath))
# print(" - expofilesdir {}".format(dirpath))
urlpath = 'expofiles' / Path(filepath)
fileitems = []
diritems = []
@ -53,31 +52,19 @@ def expofilesdir(request, dirpath, filepath):
def flatpage(request, path):
'''Either renders an HTML page from expoweb with all the menus,
or serves an unadorned binary file with mime type'''
print(" - FLATPAGES delivering the file: {} as MIME type: {}".format(path,getmimetype(path)),flush=True)
or serves an unadorned binary file with mime type
This is a horrible mess and some code is redundant and unreachable because of urls.py setup
'''
# print(" - FLATPAGES delivering the file: {} as MIME type: {}".format(path,getmimetype(path)),flush=True)
if path.startswith("noinfo") and settings.PUBLIC_SITE and not request.user.is_authenticated():
print((" - FLATPAGES redirect to logon: flat path noinfo", path))
# print((" - FLATPAGES redirect to logon: flat path noinfo", path))
return HttpResponseRedirect(urljoin(reverse("auth_login"),'?next={}'.format(request.path)))
try:
r = Cave.objects.get(url = path)
return troggle.core.views_caves.caveSlug(request, r.slug())
except Cave.DoesNotExist:
pass
except:
#print(" ! FAILED to get only one cave per slug for: "+path)
# we should do this proper;ly, not this hack that returns the first cave that matches
caves = Cave.objects.all().filter(url = path)
for c in caves:
print(path, c.slug())
if c.slug() != None:
return troggle.core.views_caves.caveSlug(request, c.slug())
pass
expowebpath = Path(settings.EXPOWEB)
if path.endswith("/") or path == "":
print(" - FLATPAGES the file: {} ENDSWITH ...".format(path))
# print(" - FLATPAGES the file: {} ENDSWITH ...".format(path))
try:
o = open(os.path.normpath(expowebpath / path / "index.html"), "rb")
@ -89,25 +76,25 @@ def flatpage(request, path):
except IOError:
return render(request, 'pagenotfound.html', {'path': path})
else:
print(" - FLATPAGES the file: '{}' ...".format(path))
# print(" - FLATPAGES the file: '{}' ...".format(path))
if path.startswith('site_media'):
print(" - MEDIA_ROOT: {} ...{}".format(settings.MEDIA_ROOT, path))
# print(" - MEDIA_ROOT: {} ...{}".format(settings.MEDIA_ROOT, path))
path = path.replace("site_media", settings.MEDIA_ROOT)
filetobeopened = os.path.normpath(path)
elif path.startswith("static"):
print(" - STATIC_ROOT: {} ...{}".format(settings.MEDIA_ROOT, path))
# print(" - STATIC_ROOT: {} ...{}".format(settings.MEDIA_ROOT, path))
path = path.replace("static", settings.MEDIA_ROOT)
filetobeopened = os.path.normpath(path)
else:
print(" - NO _ROOT: {} ...".format(expowebpath))
# print(" - NO _ROOT: {} ...".format(expowebpath))
filetobeopened = os.path.normpath(expowebpath / path)
print(" - FLATPAGES full path : {} ...".format(filetobeopened))
# print(" - FLATPAGES full path : {} ...".format(filetobeopened))
try:
o = open(filetobeopened, "rb")
print(" - FLATPAGES full path no error: {} ...".format(filetobeopened))
#print(" - FLATPAGES full path no error: {} ...".format(filetobeopened))
except IOError:
print(" - FLATPAGES ERROR: {} ...".format(filetobeopened))
#print(" - FLATPAGES ERROR: {} ...".format(filetobeopened))
#o.close()
return render(request, 'pagenotfound.html', {'path': path})
@ -147,7 +134,7 @@ def flatpage(request, path):
return render(request, 'flatpage.html', {'editable': editable, 'path': path, 'title': title,
'body': body, 'homepage': (path == "index.htm"), 'has_menu': has_menu})
else:
print(" - FLATPAGES delivering the file: {} as MIME type: {}".format(path,getmimetype(path)))
# print(" - FLATPAGES delivering the file: {} as MIME type: {}".format(path,getmimetype(path)))
return HttpResponse(content=open(filetobeopened, "rb"), content_type=getmimetype(path))
#return HttpResponse(content=open(singlescan.ffile,"rb"), content_type=getmimetype(path))