Backed out changeset: f23440eb11a3

This commit is contained in:
Sam Wenham 2019-03-04 19:39:43 +00:00
parent b505a26ce4
commit e01507d541
10 changed files with 152 additions and 236 deletions

View File

@ -1,24 +1,2 @@
import utm
import math
from django.conf import settings
def lat_lon_entrance(utmstring):
try:
x = float(utmstring.split()[0])
y = float(utmstring.split()[1])
#return ' '+str(x+y)+' '+str(y)
q = utm.to_latlon(x, y, 33, 'U')
return "{:.5f} {:.5f}".format(q[0],q[1])
except:
return 'Not found'
def top_camp_distance(utmstring):
try:
x = float(utmstring.split()[0])
y = float(utmstring.split()[1])
tx = settings.TOPCAMPX
ty = settings.TOPCAMPY
dist = math.sqrt( (tx-x)*(tx-x) + (ty-y)*(ty-y) )
return "{:.1f}".format(dist)
except:
return 'Not found'
def emptyfun():
return

View File

@ -1,7 +1,7 @@
from django.db import models
from django.conf import settings
import troggle.core.methods_millenial as methods_millenial
from troggle.core.methods_millenial import *
#
# This file was created in 2019
@ -37,32 +37,15 @@ class CaveM(models.Model): #instance of this class corresponds to one 'thing' th
survex_file = models.TextField() #gives path to top level survex file
total_length = models.FloatField() #holds total length of this cave (as given by cavern)
total_depth = models.FloatField() #holds total depth of this cave (as given by cavern)
description = models.TextField() #holds link to description
date = models.TextField() #holds date of last visit
description = models.TextField() #holds decription of the cave
def top_camp_distance(self): #returns distance of this cave from topcamp
return methods_millenial.top_camp_distance(self.entrance)
return 0
def top_camp_bearing(self): #returns bearing to this cave from topcamp in format 235.5 (float north-based azimuth)
return methods_millenial.top_camp_bearing(self.entrance)
return 0
def top_camp_bearing_letter(self): #returns bearing to this cave from topcamp in format e.g. 'NE'
return methods_millenial.top_camp_bearing_letter(self.entrance)
def lat_lon_entrance(self): #lat_lon entrance location
return methods_millenial.lat_lon_entrance(self.entrance)
class Cave_descriptionM(models.Model): #instance of this class corresponds to each of the .html files in descriptions
#each of those holds one XML field
slug = models.TextField()
explorers = models.TextField()
underground_description = models.TextField()
equipment = models.TextField()
references = models.TextField()
survey = models.TextField()
kataster_status = models.TextField()
underground_centre_line = models.TextField()
survex_file = models.TextField() #as given in .html file
notes = models.TextField()
return 0
def last_visit(self): #returns Survey class instance of the most recent visit
return 0
class ExpeditionM(models.Model): #instance of this class corresponds to one expo (usually one year)
date = models.CharField(max_length=100) #date in format YYYY.MM.DD-YYYY.MM.DD
@ -74,7 +57,7 @@ class SurveyM(models.Model): #instance of this class corresponds to one .svx fil
class Logbook_entryM(models.Model): #instance of this class corresponds to one bit of logbook (c.f. expo.survex.com/years/2015/logbook.html or simil)
date = models.CharField(max_length=100) #date as typed into logbook
contents = models.TextField() #contents of the logbook chunk
title = models.TextField() #contents of the logbook chunk
class Parser_messageM(models.Model): #instance of this class contains one error or warining message produce by any of the parsers
parsername = models.CharField(max_length = 20) #name of parser

View File

@ -1,7 +1,7 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation, CaveM, Cave_descriptionM
from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation
from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
import troggle.core.models as models
import troggle.settings as settings
@ -21,17 +21,6 @@ import settings
from PIL import Image, ImageDraw, ImageFont
import string, os, sys, subprocess
def millenialcaves(request):
#RW messing around area
caves = CaveM.objects.all()
descr = Cave_descriptionM.objects.all()
return render_with_context(request,'millenialcaves.html',{'caves': caves,'descriptions' : descr})
def millenialdescription(request, slug):
desc = Cave_descriptionM.objects.get(slug=slug)
return render_with_context(request,'cave_uground_description.html', {'cave': desc})
def getCave(cave_id):
"""Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm."""
try:
@ -65,15 +54,15 @@ def caveindex(request):
caves = Cave.objects.all()
notablecavehrefs = settings.NOTABLECAVESHREFS
notablecaves = [Cave.objects.get(kataster_number=kataster_number) for kataster_number in notablecavehrefs ]
#caves1623 = list(Cave.objects.filter(area__short_name = "1623"))
caves1623 = list(Cave.objects.all())
caves1623 = list(Cave.objects.filter(area__short_name = "1623"))
caves1626 = list(Cave.objects.filter(area__short_name = "1626"))
caves1623.sort(caveCmp)
caves1626.sort(caveCmp)
return render_with_context(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':notablecaves, 'cavepage': True})
def millenialcaves(request):
#RW messing around area
return HttpResponse("Test text", content_type="text/plain")

View File

@ -9,7 +9,7 @@ from django.contrib.auth.models import User
from django.http import HttpResponse
from django.core.urlresolvers import reverse
from troggle.core.models import Cave, Entrance
from troggle.core.models import PersonM, SurveyM, CaveM, ExpeditionM, Logbook_entryM, Cave_descriptionM
from troggle.core.models import PersonM, SurveyM, CaveM, ExpeditionM, Logbook_entryM
import troggle.flatpages.models
databasename=settings.DATABASES['default']['NAME']
@ -42,7 +42,6 @@ def gracefull_flush():
SurveyM.objects.all().delete()
ExpeditionM.objects.all().delete()
Logbook_entryM.objects.all().delete()
Cave_descriptionM.objects.all().delete()
print('Deleted contents of the database, ready to load new stuff :)')
def load_redirects():

View File

@ -34,18 +34,17 @@ function filterTable(tablename)
rows = table.rows;
for(i=1; i< rows.length; i++)
{
name = (rows[i].getElementsByTagName("TD")[1]).innerHTML.toLowerCase();
name = (rows[i].getElementsByTagName("TD")[0]).innerHTML.toLowerCase();
depth = (rows[i].getElementsByTagName("TD")[2]).innerHTML.toLowerCase();
depth = (rows[i].getElementsByTagName("TD")[1]).innerHTML.toLowerCase();
depth = Number(depth.replace(/[^0-9.]/g,''));
length = (rows[i].getElementsByTagName("TD")[3]).innerHTML.toLowerCase();
length = (rows[i].getElementsByTagName("TD")[2]).innerHTML.toLowerCase();
length = Number(length.replace(/[^0-9.]/g,''));
date = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase();
date = (rows[i].getElementsByTagName("TD")[3]).innerHTML.toLowerCase();
//recentvisitor = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase();
recentvisitor = ""
recentvisitor = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase();
if(cavename != "" && !name.includes(cavename))
{

View File

@ -19,23 +19,16 @@ def load():
print('Hi! I\'m caves parser. Ready to work')
print('Loading caves of 1623 area')
loadarea('1623')
loadarea('caves-1623/')
def loadarea(areacode):
if not file_exists(settings.SURVEX_DATA+'1623-and-1626.3d'):
print('Computing master .3d file')
bash('cavern -o'+settings.SURVEX_DATA+' '+settings.SURVEX_DATA+'1623-and-1626.svx')
else:
print('Loading from existing master .3d file')
master3d = bash('dump3d -d '+settings.SURVEX_DATA+'1623-and-1626.3d').splitlines()
master3dN = [x for x in master3d if ('NODE' in x)] #list of nodes of master survex file
master3dL = [x for x in master3d if ('LINE' in x)] #list of nodes of master survex file
print('Searching all cave dirs files')
basedir = settings.SURVEX_DATA+'caves-'+areacode+'/'
basedir = settings.SURVEX_DATA+areacode
bash('cavern -o'+settings.SURVEX_DATA+' '+settings.SURVEX_DATA+'1623-and-1626.svx')
cavedirs = bash("find "+basedir+" -maxdepth 1 -type d").splitlines() #this command finds all directories
print('Obtained list of directories! (#dirs='+str(len(cavedirs))+')')
@ -47,14 +40,14 @@ def loadarea(areacode):
cavename = bash('echo '+cavedir+' | rev | cut -f1 -d \'/\' | rev').splitlines()[0] #get final bit of the directory
test = bash('if [ ! -f '+cavedir+'/'+cavename+'.svx ] ; then echo MISSING; fi')#test for file exisence
if not file_exists(cavedir+'/'+cavename+'.svx'):
if 'MISSING' in test: #send error message to the database
msg = models.Parser_messageM(parsername='caves',content=cavedir+'/'+cavename+' MISSING!',message_type='warn')
print('Cave missing'+cavename+' :(')
msg.save()
continue
fullname=cavedir+'/'+cavename+'.svx'
print('Found cave:'+cavename)
cavernout = bash('cavern -o '+cavedir+' '+fullname) #make cavern process the thing
cavernout = bash('cavern -q '+fullname) #make cavern process the thing
if 'cavern: error:' in cavernout:
msg = models.Parser_messageM(parsername='caves',content=cavedir+'/'+cavename+' Survex file messed up!',message_type='warn')
print('Fucked svx'+cavename+' :(')
@ -64,55 +57,16 @@ def loadarea(areacode):
cavernout = cavernout.splitlines()
depth = float(([x for x in cavernout if ('Total vertical length' in x)][0].split()[-1])[:-2])
length = float(([x for x in cavernout if ('Total length' in x)][0].split()[6])[:-1])
cavefile = open(fullname,'r')
cavefilecontents = cavefile.read().splitlines()
surveyname = [x for x in cavefilecontents if ('*begin ') in x][0].split()[1].lower()
try:
title = [x for x in cavefilecontents if ('*title ') in x][0].split()[1]
except:
syrveyname = "Untitled"
relevant_nodes = [x for x in master3dN if (('['+areacode+'.'+surveyname+'.' in x) or ('['+areacode+'.'+surveyname+']' in x))]
entrance_nodes = [x for x in relevant_nodes if 'ENTRANCE' in x]
surface_nodes = [x for x in relevant_nodes if 'SURFACE' in x]
location_nodes = []
print('rel_nodes'+str(len(relevant_nodes)))
if len(entrance_nodes) > 0:
location_nodes = entrance_nodes
elif len(surface_nodes) > 0:
location_nodes = surface_nodes
elif len(relevant_nodes) > 0:
location_nodes = relevant_nodes
try:
location = sorted(location_nodes, key = lambda y : float(y.split()[3])).pop()
except:
print(location_nodes)
location = 'Not found'
relevant_lines = [x for x in master3dL if (('['+areacode+'.'+surveyname+'.' in x) or ('['+areacode+'.'+surveyname+']' in x))]
try:
lastleg = sorted(relevant_lines, key = lambda y : y.split().pop()).pop()
except:
lastleg = ['LINE 1900.01.01']
try:
lastdate = lastleg.split().pop()
if 'STYLE' in lastdate:
lastdate = lastleg.split().pop().pop()
except:
lastdate = '1900.01.01'
entrance = ' '.join(location.split()[1:3])
print((('depth','length','surv name','entr','date'),(depth,length,surveyname,entrance,lastdate))) #sanity check print
surveyname = bash('cat '+fullname+' | grep \'\*begin\' | head -n1 | cut -f2 -d \' \' ').splitlines().pop()
title = (bash('cat '+fullname+' | grep \'\*title\' | head -n1 | cut -f2 -d \' \' ').splitlines() or ["Not found"])[0]
print((('depth','length','surv name'),(depth,length,surveyname)))
print('dump3d '+settings.SURVEX_DATA+'1623-and-1626.3d | grep NODE | grep \'\\[\\.'+surveyname+'.*\\]\'')
nodes = bash('dump3d '+settings.SURVEX_DATA+'1623-and-1626.3d | grep NODE | grep \'\\[.*\\.'+surveyname+'.*\\]\'').splitlines()
entran = [x for x in nodes if ('ENTRANCE' in x) ]
print(nodes)
newcave = models.CaveM(
survex_file = fullname,
total_length = length,
name=areacode+'.'+surveyname,
total_depth = depth,
date = lastdate,
entrance = entrance)
newcave = models.CaveM(survex_file = fullname, total_length = length, name=title, total_depth = depth)
newcave.save()
#end of reading survex masterfiles
@ -123,12 +77,8 @@ def loadarea(areacode):
print(fn)
contents = f.read()
slug = re.sub(r"\s+", "", extractXML(contents,'caveslug'))
desc = extractXML(contents,'underground_description')
name = slug[5:] #get survex compatible name
area = slug[0:4]
print([area,name])
name = re.search(r'>.*<',extractXML(contents,'caveslug')).group()[6:-1]
if desc==None or name==None:
msg = models.Parser_messageM(parsername='caves',content=fn+' Description meesed up!',message_type='warn')
@ -136,56 +86,26 @@ def loadarea(areacode):
msg.save()
continue
print(area+'/'+name+'/'+name+'.svx')
updatecave = models.CaveM.objects.filter(survex_file__icontains=area+'/'+name+'/'+name+'.svx')
updatecave = models.CaveM.objects.filter(survex_file__icontains='/'+name+'.svx')
if len(updatecave)>1:
print('Non unique solution - skipping. Name:'+name)
elif len(updatecave)==0:
print('Cave with no survex data:'+name)
continue
print('Cave with no survex data'+name)
newcave = models.CaveM(description = desc, name = name)
newcave.save()
else: #exaclty one match
print('Adding desc:'+name)
updatecave = updatecave[0]
updatecave.description = '/cave/descriptionM/'+slug #area-name
updatecave.description = desc
if updatecave.name=="Not found":
updatecave.name=name
updatecave.title=name
updatecave.save()
slugS = slug
explorersS = extractXML(contents,'explorers')
underground_descriptionS = extractXML(contents,'underground_description')
equipmentS = extractXML(contents,'equipment')
referencesS = extractXML(contents,'references')
surveyS = extractXML(contents,'survey')
kataster_statusS = extractXML(contents,'kataster_status')
underground_centre_lineS = extractXML(contents,'underground_centre_line')
survex_fileS = extractXML(contents,'survex_file')
notesS = extractXML(contents,'notes')
newcavedesc = models.Cave_descriptionM(
slug = slugS,
explorers = explorersS,
underground_description = underground_descriptionS,
equipment = equipmentS,
references = referencesS,
survey = surveyS,
kataster_status = kataster_statusS,
underground_centre_line = underground_centre_lineS,
survex_file = survex_fileS,
notes = notesS)
newcavedesc.save()
#end of reading cave descriptions
def file_exists(filename):
test = bash('if [ ! -f '+filename+' ] ; then echo MISSING; fi')#test for file exisence
if 'MISSING' in test: #send error message to the database
return False
return True
def extractXML(contents,tag):
#find correct lines
@ -199,10 +119,9 @@ def extractXML(contents,tag):
if endi!=begi:
segment = '\n'.join(lines[begi:endi+1])
else:
segment = lines[begi:endi+1][0]
segment = lines[begi:endi+1]
return segment[0]
hit = re.findall('<'+tag+'>(.*)</'+tag+'>', segment, re.S)[0]
return hit
def bash(cmd): #calls command in bash shell, returns output
process = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)

View File

@ -71,9 +71,6 @@ if django.VERSION[0] == 1 and django.VERSION[1] < 4:
else:
authmodule = 'django.contrib.auth.context_processors.auth'
TOPCAMPX=411571.00
TOPCAMPY=5282639.00
TEMPLATE_CONTEXT_PROCESSORS = ( authmodule, "core.context.troggle_context", )
LOGIN_REDIRECT_URL = '/'

View File

@ -20,7 +20,7 @@
<table class="searchable">
{% for cave in caves1623 %}
<tr><td> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> {{ cave.slug }}</td></tr>
<tr><td> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> </td></tr>
{% endfor %}
</table>

View File

@ -8,62 +8,116 @@
<body>
<h2>Caves of loserplateau (locations acording to all.3d)</h2>
<span style="font-size:70%">
Name contains:<br>
<div id="inputf"><input type="text" name="CaveName" id="CaveName" style="width:100%"></div><br>
Depth between (min, max) in meters (0 disables filter):<br>
<div id="inputf"><input type="number" name="CaveDepthMin" id="CaveDepthMin" style="width:45%"> - <input type="number" name="CaveDepthMax" id="CaveDepthMax" style="width:45%"></div><br>
Length between (min, max) in meters (0 disables filter):<br>
<div id="inputf"><input type="number" name="CaveLengthMin" id="CaveLengthMin" style="width:45%"> - <input type="number" name="CaveLengthMax" id="CaveLengthMax" style="width:45%"></div><br>
Last visit after (date in YYYY.MM.DD format works best):<br>
<div id="inputf"><input type="text" name="VisitDate" id="VisitDate" style="width:100%"></div><br>
Last visited by (single word or regular expression, search is not case sensitive):<br>
(e.g. <span id="mono">/da.e/</span> matches both Dave and Dane, <span id="mono">/w..k|ol{2}y/</span> matches either Wook and Olly)<br>
<div id="inputf"><input type="text" name="Visitor" id="Visitor" style="width:100%"></div><br>
Hide incomplete entries:<br>
<div id="inputf"><input type="checkbox" name="Incomplete" id="Incomplete" style="width:100%"></div><br><br>
<button onclick="filterTable('caves_table')">Filter</button><br>
<button onclick="filterTableReset('caves_table')">Reset filters</button><br>
Click on column headers to sort/reverse sort<br><br><br>
</span>
<table id="caves_table">
<table>
<tr>
<th onclick="sortTable(0,'caves_table',0)">Cave survex id</th>
<th onclick="sortTable(1,'caves_table',0)">Cave name</th>
<th onclick="sortTable(2,'caves_table',1)">Cave depth</th>
<th onclick="sortTable(3,'caves_table',1)">Cave length</th>
<th onclick="sortTable(4,'caves_table',0)">Last leg date</th>
<th onclick="sortTable(5,'caves_table',0)">Cave location (UTM)</th>
<th onclick="sortTable(6,'caves_table',0)">Cave location (lat/lon)</th>
<th onclick="sortTable(7,'caves_table',1)">Top camp distance [m]</th>
<th>Cave</th>
<th>Components</th>
<th>aaa</th>
<th>bbb</th>
</tr>
{% for cave in caves %}
{% for subdircave, cavefiles, subsurvdirs in subdircaves %}
<tr>
<td><a href={{cave.description}}>{{ cave.name }}</a></td>
<td>{{ cave.title }}</td>
<td>{{ cave.total_depth }}</td>
<td>{{ cave.total_length }}</td>
<td>{{ cave.date }}</td>
<td>33U {{ cave.entrance }}</td>
<td>{{ cave.lat_lon_entrance }}</td>
<td>{{ cave.top_camp_distance}}</td>
<td><b><a href="{% url "svx" cavefiles.0.0 %}">{{cavefiles.0.1}}</a></b></td>
<td>
root: {% for cavepath, cavename in cavefiles.1 %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
<br>
{% for primarycavefile, subcavefiles in subsurvdirs %}
{{ primarycavefile.1 }}:
{% for cavepath, cavename in subcavefiles %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
<br>
{% endfor %}
</td>
</tr>
{% endfor %}
{% for primarycavefile, subcavefiles in multifilecaves %}
<tr>
<td>
<a href="{% url "survexcavessingle" primarycavefile.1 %}">{{primarycavefile.1}}</a>
</td>
<td>
{% for cavepath, cavename in subcavefiles %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
</td>
</tr>
{% endfor %}
</table>
<script type="text/javascript" src="{{ settings.MEDIA_URL }}/scripts/TableSort.js"></script>
</body>
======================= OLD STUFF ==============================
<td><a href="{% url "svx" cavefiles.0.0 %}">{{cavefiles.0.1}}</a></td>
<p><a href="#cdir">caves with subdirectories</a> | <a href="#cmult">caves with multiple files</a> | <a href="#csing">caves with single files</a></p>
<h3><a href="/survexfile/all.svx">Link to all.svx for processing</a></h3>
<h2 id="cdir">Caves with subdirectories</h2>
{% for subdircave, cavefiles, subsurvdirs in subdircaves %}
<h3>{{cavefiles.0.1}} - <a href="{% url "survexcavessingle" cavefiles.0.1 %}">dates and explorers</a></h3>
<table>
<tr>
<td><b><a href="{% url "svx" cavefiles.0.0 %}">{{cavefiles.0.1}}</a></b></td>
<td>
{% for cavepath, cavename in cavefiles.1 %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
</td>
</tr>
{% for primarycavefile, subcavefiles in subsurvdirs %}
<tr>
<td><a href="{% url "svx" primarycavefile.0 %}">{{primarycavefile.1}}</a></td>
<td>
{% for cavepath, cavename in subcavefiles %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
</td>
</tr>
{% endfor %}
</table>
{% endfor %}
<h2 id="cmult">Caves of multiple files</h2>
<table>
<tr><th>Dates and explorers</th><th>Survex files</th></tr>
{% for primarycavefile, subcavefiles in multifilecaves %}
<tr>
<td>
<a href="{% url "survexcavessingle" primarycavefile.1 %}">{{primarycavefile.1}}</a>
</td>
<td>
<a href="{% url "svx" primarycavefile.0 %}">{{primarycavefile.1}}</a> -
{% for cavepath, cavename in subcavefiles %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
</td>
</tr>
{% endfor %}
</table>
<h2 id="csing">Caves of one file</h2>
<p>
{% for cavepath, cavename in onefilecaves %}
<a href="{% url "svx" cavepath %}">{{cavename}}</a>
{% endfor %}
</p>
<body>
<script type="text/javascript" src="scripts/TableSort.js"></script>
</html>

View File

@ -23,9 +23,7 @@ admin.autodiscover()
actualurlpatterns = patterns('',
url(r'^millenialcaves/?$', views_caves.millenialcaves, name="millenialcaves"),
url(r'^cave/descriptionM/([^/]+)/?$', views_caves.millenialdescription),
#url(r'^cave/description/([^/]+)/?$', views_caves.caveDescription),
url(r'^millenialcaves/?$', views_survex.millenialcaves, name="millenialcaves"),
url(r'^troggle$', views_other.frontpage, name="frontpage"),
url(r'^todo/$', views_other.todo, name="todo"),