2
0
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-01-19 09:22:32 +00:00

rebuild descriptions database, some visuals

This commit is contained in:
Rad 2019-02-28 12:36:49 +00:00
parent ce268ec306
commit 6b59e3a689
10 changed files with 236 additions and 152 deletions

View File

@ -1,2 +1,24 @@
def emptyfun():
return
import utm
import math
from django.conf import settings
def lat_lon_entrance(utmstring):
try:
x = float(utmstring.split()[0])
y = float(utmstring.split()[1])
#return ' '+str(x+y)+' '+str(y)
q = utm.to_latlon(x, y, 33, 'U')
return "{:.5f} {:.5f}".format(q[0],q[1])
except:
return 'Not found'
def top_camp_distance(utmstring):
try:
x = float(utmstring.split()[0])
y = float(utmstring.split()[1])
tx = settings.TOPCAMPX
ty = settings.TOPCAMPY
dist = math.sqrt( (tx-x)*(tx-x) + (ty-y)*(ty-y) )
return "{:.1f}".format(dist)
except:
return 'Not found'

View File

@ -1,7 +1,7 @@
from django.db import models
from django.conf import settings
from troggle.core.methods_millenial import *
import troggle.core.methods_millenial as methods_millenial
#
# This file was created in 2019
@ -37,15 +37,32 @@ class CaveM(models.Model): #instance of this class corresponds to one 'thing' th
survex_file = models.TextField() #gives path to top level survex file
total_length = models.FloatField() #holds total length of this cave (as given by cavern)
total_depth = models.FloatField() #holds total depth of this cave (as given by cavern)
description = models.TextField() #holds decription of the cave
description = models.TextField() #holds link to description
date = models.TextField() #holds date of last visit
def top_camp_distance(self): #returns distance of this cave from topcamp
return 0
return methods_millenial.top_camp_distance(self.entrance)
def top_camp_bearing(self): #returns bearing to this cave from topcamp in format 235.5 (float north-based azimuth)
return 0
return methods_millenial.top_camp_bearing(self.entrance)
def top_camp_bearing_letter(self): #returns bearing to this cave from topcamp in format e.g. 'NE'
return 0
def last_visit(self): #returns Survey class instance of the most recent visit
return 0
return methods_millenial.top_camp_bearing_letter(self.entrance)
def lat_lon_entrance(self): #lat_lon entrance location
return methods_millenial.lat_lon_entrance(self.entrance)
class Cave_descriptionM(models.Model): #instance of this class corresponds to each of the .html files in descriptions
#each of those holds one XML field
slug = models.TextField()
explorers = models.TextField()
underground_description = models.TextField()
equipment = models.TextField()
references = models.TextField()
survey = models.TextField()
kataster_status = models.TextField()
underground_centre_line = models.TextField()
survex_file = models.TextField() #as given in .html file
notes = models.TextField()
class ExpeditionM(models.Model): #instance of this class corresponds to one expo (usually one year)
date = models.CharField(max_length=100) #date in format YYYY.MM.DD-YYYY.MM.DD
@ -57,7 +74,7 @@ class SurveyM(models.Model): #instance of this class corresponds to one .svx fil
class Logbook_entryM(models.Model): #instance of this class corresponds to one bit of logbook (c.f. expo.survex.com/years/2015/logbook.html or simil)
date = models.CharField(max_length=100) #date as typed into logbook
title = models.TextField() #contents of the logbook chunk
contents = models.TextField() #contents of the logbook chunk
class Parser_messageM(models.Model): #instance of this class contains one error or warining message produce by any of the parsers
parsername = models.CharField(max_length = 20) #name of parser

View File

@ -1,7 +1,7 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation
from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation, CaveM, Cave_descriptionM
from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
import troggle.core.models as models
import troggle.settings as settings
@ -21,6 +21,17 @@ import settings
from PIL import Image, ImageDraw, ImageFont
import string, os, sys, subprocess
def millenialcaves(request):
#RW messing around area
caves = CaveM.objects.all()
descr = Cave_descriptionM.objects.all()
return render_with_context(request,'millenialcaves.html',{'caves': caves,'descriptions' : descr})
def millenialdescription(request, slug):
desc = Cave_descriptionM.objects.get(slug=slug)
return render_with_context(request,'cave_uground_description.html', {'cave': desc})
def getCave(cave_id):
"""Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm."""
try:
@ -54,16 +65,16 @@ def caveindex(request):
caves = Cave.objects.all()
notablecavehrefs = settings.NOTABLECAVESHREFS
notablecaves = [Cave.objects.get(kataster_number=kataster_number) for kataster_number in notablecavehrefs ]
caves1623 = list(Cave.objects.filter(area__short_name = "1623"))
#caves1623 = list(Cave.objects.filter(area__short_name = "1623"))
caves1623 = list(Cave.objects.all())
caves1626 = list(Cave.objects.filter(area__short_name = "1626"))
caves1623.sort(caveCmp)
caves1626.sort(caveCmp)
return render_with_context(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':notablecaves, 'cavepage': True})
def millenialcaves(request):
#RW messing around area
return HttpResponse("Test text", content_type="text/plain")
def cave3d(request, cave_id=''):

View File

@ -9,7 +9,7 @@ from django.contrib.auth.models import User
from django.http import HttpResponse
from django.core.urlresolvers import reverse
from troggle.core.models import Cave, Entrance
from troggle.core.models import PersonM, SurveyM, CaveM, ExpeditionM, Logbook_entryM
from troggle.core.models import PersonM, SurveyM, CaveM, ExpeditionM, Logbook_entryM, Cave_descriptionM
import troggle.flatpages.models
databasename=settings.DATABASES['default']['NAME']
@ -42,6 +42,7 @@ def gracefull_flush():
SurveyM.objects.all().delete()
ExpeditionM.objects.all().delete()
Logbook_entryM.objects.all().delete()
Cave_descriptionM.objects.all().delete()
print('Deleted contents of the database, ready to load new stuff :)')
def load_redirects():

View File

@ -34,17 +34,18 @@ function filterTable(tablename)
rows = table.rows;
for(i=1; i< rows.length; i++)
{
name = (rows[i].getElementsByTagName("TD")[0]).innerHTML.toLowerCase();
name = (rows[i].getElementsByTagName("TD")[1]).innerHTML.toLowerCase();
depth = (rows[i].getElementsByTagName("TD")[1]).innerHTML.toLowerCase();
depth = (rows[i].getElementsByTagName("TD")[2]).innerHTML.toLowerCase();
depth = Number(depth.replace(/[^0-9.]/g,''));
length = (rows[i].getElementsByTagName("TD")[2]).innerHTML.toLowerCase();
length = (rows[i].getElementsByTagName("TD")[3]).innerHTML.toLowerCase();
length = Number(length.replace(/[^0-9.]/g,''));
date = (rows[i].getElementsByTagName("TD")[3]).innerHTML.toLowerCase();
date = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase();
recentvisitor = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase();
//recentvisitor = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase();
recentvisitor = ""
if(cavename != "" && !name.includes(cavename))
{

View File

@ -19,16 +19,23 @@ def load():
print('Hi! I\'m caves parser. Ready to work')
print('Loading caves of 1623 area')
loadarea('caves-1623/')
loadarea('1623')
def loadarea(areacode):
if not file_exists(settings.SURVEX_DATA+'1623-and-1626.3d'):
print('Computing master .3d file')
bash('cavern -o'+settings.SURVEX_DATA+' '+settings.SURVEX_DATA+'1623-and-1626.svx')
else:
print('Loading from existing master .3d file')
master3d = bash('dump3d -d '+settings.SURVEX_DATA+'1623-and-1626.3d').splitlines()
master3dN = [x for x in master3d if ('NODE' in x)] #list of nodes of master survex file
master3dL = [x for x in master3d if ('LINE' in x)] #list of nodes of master survex file
print('Searching all cave dirs files')
basedir = settings.SURVEX_DATA+areacode
bash('cavern -o'+settings.SURVEX_DATA+' '+settings.SURVEX_DATA+'1623-and-1626.svx')
basedir = settings.SURVEX_DATA+'caves-'+areacode+'/'
cavedirs = bash("find "+basedir+" -maxdepth 1 -type d").splitlines() #this command finds all directories
print('Obtained list of directories! (#dirs='+str(len(cavedirs))+')')
@ -40,14 +47,14 @@ def loadarea(areacode):
cavename = bash('echo '+cavedir+' | rev | cut -f1 -d \'/\' | rev').splitlines()[0] #get final bit of the directory
test = bash('if [ ! -f '+cavedir+'/'+cavename+'.svx ] ; then echo MISSING; fi')#test for file exisence
if 'MISSING' in test: #send error message to the database
if not file_exists(cavedir+'/'+cavename+'.svx'):
msg = models.Parser_messageM(parsername='caves',content=cavedir+'/'+cavename+' MISSING!',message_type='warn')
print('Cave missing'+cavename+' :(')
msg.save()
continue
fullname=cavedir+'/'+cavename+'.svx'
print('Found cave:'+cavename)
cavernout = bash('cavern -q '+fullname) #make cavern process the thing
cavernout = bash('cavern -o '+cavedir+' '+fullname) #make cavern process the thing
if 'cavern: error:' in cavernout:
msg = models.Parser_messageM(parsername='caves',content=cavedir+'/'+cavename+' Survex file messed up!',message_type='warn')
print('Fucked svx'+cavename+' :(')
@ -57,28 +64,71 @@ def loadarea(areacode):
cavernout = cavernout.splitlines()
depth = float(([x for x in cavernout if ('Total vertical length' in x)][0].split()[-1])[:-2])
length = float(([x for x in cavernout if ('Total length' in x)][0].split()[6])[:-1])
surveyname = bash('cat '+fullname+' | grep \'\*begin\' | head -n1 | cut -f2 -d \' \' ').splitlines().pop()
title = (bash('cat '+fullname+' | grep \'\*title\' | head -n1 | cut -f2 -d \' \' ').splitlines() or ["Not found"])[0]
print((('depth','length','surv name'),(depth,length,surveyname)))
print('dump3d '+settings.SURVEX_DATA+'1623-and-1626.3d | grep NODE | grep \'\\[\\.'+surveyname+'.*\\]\'')
nodes = bash('dump3d '+settings.SURVEX_DATA+'1623-and-1626.3d | grep NODE | grep \'\\[.*\\.'+surveyname+'.*\\]\'').splitlines()
entran = [x for x in nodes if ('ENTRANCE' in x) ]
print(nodes)
cavefile = open(fullname,'r')
cavefilecontents = cavefile.read().splitlines()
surveyname = [x for x in cavefilecontents if ('*begin ') in x][0].split()[1].lower()
try:
title = [x for x in cavefilecontents if ('*title ') in x][0].split()[1]
except:
syrveyname = "Untitled"
relevant_nodes = [x for x in master3dN if (('['+areacode+'.'+surveyname+'.' in x) or ('['+areacode+'.'+surveyname+']' in x))]
entrance_nodes = [x for x in relevant_nodes if 'ENTRANCE' in x]
surface_nodes = [x for x in relevant_nodes if 'SURFACE' in x]
location_nodes = []
print('rel_nodes'+str(len(relevant_nodes)))
if len(entrance_nodes) > 0:
location_nodes = entrance_nodes
elif len(surface_nodes) > 0:
location_nodes = surface_nodes
elif len(relevant_nodes) > 0:
location_nodes = relevant_nodes
newcave = models.CaveM(survex_file = fullname, total_length = length, name=title, total_depth = depth)
try:
location = sorted(location_nodes, key = lambda y : float(y.split()[3])).pop()
except:
print(location_nodes)
location = 'Not found'
relevant_lines = [x for x in master3dL if (('['+areacode+'.'+surveyname+'.' in x) or ('['+areacode+'.'+surveyname+']' in x))]
try:
lastleg = sorted(relevant_lines, key = lambda y : y.split().pop()).pop()
except:
lastleg = ['LINE 1900.01.01']
try:
lastdate = lastleg.split().pop()
if 'STYLE' in lastdate:
lastdate = lastleg.split().pop().pop()
except:
lastdate = '1900.01.01'
entrance = ' '.join(location.split()[1:3])
print((('depth','length','surv name','entr','date'),(depth,length,surveyname,entrance,lastdate))) #sanity check print
newcave = models.CaveM(
survex_file = fullname,
total_length = length,
name=areacode+'.'+surveyname,
total_depth = depth,
date = lastdate,
entrance = entrance)
newcave.save()
#end of reading survex masterfiles
print ("Reading cave descriptions")
cavefiles = bash('find '+settings.CAVEDESCRIPTIONS+' -name \'*.html\'').splitlines()
for fn in cavefiles:
f = open(fn, "r")
print(fn)
contents = f.read()
slug = re.sub(r"\s+", "", extractXML(contents,'caveslug'))
desc = extractXML(contents,'underground_description')
name = re.search(r'>.*<',extractXML(contents,'caveslug')).group()[6:-1]
name = slug[5:] #get survex compatible name
area = slug[0:4]
print([area,name])
if desc==None or name==None:
msg = models.Parser_messageM(parsername='caves',content=fn+' Description meesed up!',message_type='warn')
@ -86,26 +136,56 @@ def loadarea(areacode):
msg.save()
continue
print(area+'/'+name+'/'+name+'.svx')
updatecave = models.CaveM.objects.filter(survex_file__icontains='/'+name+'.svx')
updatecave = models.CaveM.objects.filter(survex_file__icontains=area+'/'+name+'/'+name+'.svx')
if len(updatecave)>1:
print('Non unique solution - skipping. Name:'+name)
elif len(updatecave)==0:
print('Cave with no survex data'+name)
newcave = models.CaveM(description = desc, name = name)
newcave.save()
print('Cave with no survex data:'+name)
continue
else: #exaclty one match
print('Adding desc:'+name)
updatecave = updatecave[0]
updatecave.description = desc
if updatecave.name=="Not found":
updatecave.name=name
updatecave.description = '/cave/descriptionM/'+slug #area-name
updatecave.title=name
updatecave.save()
slugS = slug
explorersS = extractXML(contents,'explorers')
underground_descriptionS = extractXML(contents,'underground_description')
equipmentS = extractXML(contents,'equipment')
referencesS = extractXML(contents,'references')
surveyS = extractXML(contents,'survey')
kataster_statusS = extractXML(contents,'kataster_status')
underground_centre_lineS = extractXML(contents,'underground_centre_line')
survex_fileS = extractXML(contents,'survex_file')
notesS = extractXML(contents,'notes')
newcavedesc = models.Cave_descriptionM(
slug = slugS,
explorers = explorersS,
underground_description = underground_descriptionS,
equipment = equipmentS,
references = referencesS,
survey = surveyS,
kataster_status = kataster_statusS,
underground_centre_line = underground_centre_lineS,
survex_file = survex_fileS,
notes = notesS)
newcavedesc.save()
#end of reading cave descriptions
def file_exists(filename):
test = bash('if [ ! -f '+filename+' ] ; then echo MISSING; fi')#test for file exisence
if 'MISSING' in test: #send error message to the database
return False
return True
def extractXML(contents,tag):
#find correct lines
@ -119,9 +199,10 @@ def extractXML(contents,tag):
if endi!=begi:
segment = '\n'.join(lines[begi:endi+1])
else:
segment = lines[begi:endi+1]
return segment[0]
segment = lines[begi:endi+1][0]
hit = re.findall('<'+tag+'>(.*)</'+tag+'>', segment, re.S)[0]
return hit
def bash(cmd): #calls command in bash shell, returns output
process = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)

View File

@ -71,6 +71,9 @@ if django.VERSION[0] == 1 and django.VERSION[1] < 4:
else:
authmodule = 'django.contrib.auth.context_processors.auth'
TOPCAMPX=411571.00
TOPCAMPY=5282639.00
TEMPLATE_CONTEXT_PROCESSORS = ( authmodule, "core.context.troggle_context", )
LOGIN_REDIRECT_URL = '/'

View File

@ -20,7 +20,7 @@
<table class="searchable">
{% for cave in caves1623 %}
<tr><td> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> </td></tr>
<tr><td> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> {{ cave.slug }}</td></tr>
{% endfor %}
</table>

View File

@ -8,116 +8,62 @@
<body>
<table>
<h2>Caves of loserplateau (locations acording to all.3d)</h2>
<span style="font-size:70%">
Name contains:<br>
<div id="inputf"><input type="text" name="CaveName" id="CaveName" style="width:100%"></div><br>
Depth between (min, max) in meters (0 disables filter):<br>
<div id="inputf"><input type="number" name="CaveDepthMin" id="CaveDepthMin" style="width:45%"> - <input type="number" name="CaveDepthMax" id="CaveDepthMax" style="width:45%"></div><br>
Length between (min, max) in meters (0 disables filter):<br>
<div id="inputf"><input type="number" name="CaveLengthMin" id="CaveLengthMin" style="width:45%"> - <input type="number" name="CaveLengthMax" id="CaveLengthMax" style="width:45%"></div><br>
Last visit after (date in YYYY.MM.DD format works best):<br>
<div id="inputf"><input type="text" name="VisitDate" id="VisitDate" style="width:100%"></div><br>
Last visited by (single word or regular expression, search is not case sensitive):<br>
(e.g. <span id="mono">/da.e/</span> matches both Dave and Dane, <span id="mono">/w..k|ol{2}y/</span> matches either Wook and Olly)<br>
<div id="inputf"><input type="text" name="Visitor" id="Visitor" style="width:100%"></div><br>
Hide incomplete entries:<br>
<div id="inputf"><input type="checkbox" name="Incomplete" id="Incomplete" style="width:100%"></div><br><br>
<button onclick="filterTable('caves_table')">Filter</button><br>
<button onclick="filterTableReset('caves_table')">Reset filters</button><br>
Click on column headers to sort/reverse sort<br><br><br>
</span>
<table id="caves_table">
<tr>
<th>Cave</th>
<th>Components</th>
<th>aaa</th>
<th>bbb</th>
<th onclick="sortTable(0,'caves_table',0)">Cave survex id</th>
<th onclick="sortTable(1,'caves_table',0)">Cave name</th>
<th onclick="sortTable(2,'caves_table',1)">Cave depth</th>
<th onclick="sortTable(3,'caves_table',1)">Cave length</th>
<th onclick="sortTable(4,'caves_table',0)">Last leg date</th>
<th onclick="sortTable(5,'caves_table',0)">Cave location (UTM)</th>
<th onclick="sortTable(6,'caves_table',0)">Cave location (lat/lon)</th>
<th onclick="sortTable(7,'caves_table',1)">Top camp distance [m]</th>
</tr>
{% for subdircave, cavefiles, subsurvdirs in subdircaves %}
{% for cave in caves %}
<tr>
<td><b><a href="{% url "svx" cavefiles.0.0 %}">{{cavefiles.0.1}}</a></b></td>
<td>
root: {% for cavepath, cavename in cavefiles.1 %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
<br>
{% for primarycavefile, subcavefiles in subsurvdirs %}
{{ primarycavefile.1 }}:
{% for cavepath, cavename in subcavefiles %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
<br>
{% endfor %}
</td>
<td><a href={{cave.description}}>{{ cave.name }}</a></td>
<td>{{ cave.title }}</td>
<td>{{ cave.total_depth }}</td>
<td>{{ cave.total_length }}</td>
<td>{{ cave.date }}</td>
<td>33U {{ cave.entrance }}</td>
<td>{{ cave.lat_lon_entrance }}</td>
<td>{{ cave.top_camp_distance}}</td>
</tr>
{% endfor %}
{% for primarycavefile, subcavefiles in multifilecaves %}
<tr>
<td>
<a href="{% url "survexcavessingle" primarycavefile.1 %}">{{primarycavefile.1}}</a>
</td>
<td>
{% for cavepath, cavename in subcavefiles %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
</td>
</tr>
{% endfor %}
</table>
<script type="text/javascript" src="{{ settings.MEDIA_URL }}/scripts/TableSort.js"></script>
</body>
======================= OLD STUFF ==============================
<td><a href="{% url "svx" cavefiles.0.0 %}">{{cavefiles.0.1}}</a></td>
<p><a href="#cdir">caves with subdirectories</a> | <a href="#cmult">caves with multiple files</a> | <a href="#csing">caves with single files</a></p>
<h3><a href="/survexfile/all.svx">Link to all.svx for processing</a></h3>
<h2 id="cdir">Caves with subdirectories</h2>
{% for subdircave, cavefiles, subsurvdirs in subdircaves %}
<h3>{{cavefiles.0.1}} - <a href="{% url "survexcavessingle" cavefiles.0.1 %}">dates and explorers</a></h3>
<table>
<tr>
<td><b><a href="{% url "svx" cavefiles.0.0 %}">{{cavefiles.0.1}}</a></b></td>
<td>
{% for cavepath, cavename in cavefiles.1 %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
</td>
</tr>
{% for primarycavefile, subcavefiles in subsurvdirs %}
<tr>
<td><a href="{% url "svx" primarycavefile.0 %}">{{primarycavefile.1}}</a></td>
<td>
{% for cavepath, cavename in subcavefiles %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
</td>
</tr>
{% endfor %}
</table>
{% endfor %}
<h2 id="cmult">Caves of multiple files</h2>
<table>
<tr><th>Dates and explorers</th><th>Survex files</th></tr>
{% for primarycavefile, subcavefiles in multifilecaves %}
<tr>
<td>
<a href="{% url "survexcavessingle" primarycavefile.1 %}">{{primarycavefile.1}}</a>
</td>
<td>
<a href="{% url "svx" primarycavefile.0 %}">{{primarycavefile.1}}</a> -
{% for cavepath, cavename in subcavefiles %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
</td>
</tr>
{% endfor %}
</table>
<h2 id="csing">Caves of one file</h2>
<p>
{% for cavepath, cavename in onefilecaves %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
</p>
<body>
<script type="text/javascript" src="scripts/TableSort.js"></script>
</html>

View File

@ -23,7 +23,9 @@ admin.autodiscover()
actualurlpatterns = patterns('',
url(r'^millenialcaves/?$', views_survex.millenialcaves, name="millenialcaves"),
url(r'^millenialcaves/?$', views_caves.millenialcaves, name="millenialcaves"),
url(r'^cave/descriptionM/([^/]+)/?$', views_caves.millenialdescription),
#url(r'^cave/description/([^/]+)/?$', views_caves.caveDescription),
url(r'^troggle$', views_other.frontpage, name="frontpage"),
url(r'^todo/$', views_other.todo, name="todo"),