diff --git a/core/methods_millenial.py b/core/methods_millenial.py index 5d2483a..ce7ce11 100644 --- a/core/methods_millenial.py +++ b/core/methods_millenial.py @@ -1,2 +1,24 @@ -def emptyfun(): - return +import utm +import math +from django.conf import settings + +def lat_lon_entrance(utmstring): + try: + x = float(utmstring.split()[0]) + y = float(utmstring.split()[1]) + #return ' '+str(x+y)+' '+str(y) + q = utm.to_latlon(x, y, 33, 'U') + return "{:.5f} {:.5f}".format(q[0],q[1]) + except: + return 'Not found' + +def top_camp_distance(utmstring): + try: + x = float(utmstring.split()[0]) + y = float(utmstring.split()[1]) + tx = settings.TOPCAMPX + ty = settings.TOPCAMPY + dist = math.sqrt( (tx-x)*(tx-x) + (ty-y)*(ty-y) ) + return "{:.1f}".format(dist) + except: + return 'Not found' diff --git a/core/models_millenial.py b/core/models_millenial.py index db893bc..c97be36 100644 --- a/core/models_millenial.py +++ b/core/models_millenial.py @@ -1,7 +1,7 @@ from django.db import models from django.conf import settings -from troggle.core.methods_millenial import * +import troggle.core.methods_millenial as methods_millenial # # This file was created in 2019 @@ -37,15 +37,32 @@ class CaveM(models.Model): #instance of this class corresponds to one 'thing' th survex_file = models.TextField() #gives path to top level survex file total_length = models.FloatField() #holds total length of this cave (as given by cavern) total_depth = models.FloatField() #holds total depth of this cave (as given by cavern) - description = models.TextField() #holds decription of the cave + description = models.TextField() #holds link to description + date = models.TextField() #holds date of last visit def top_camp_distance(self): #returns distance of this cave from topcamp - return 0 + return methods_millenial.top_camp_distance(self.entrance) def top_camp_bearing(self): #returns bearing to this cave from topcamp in format 235.5 (float north-based azimuth) - return 0 + return methods_millenial.top_camp_bearing(self.entrance) def top_camp_bearing_letter(self): #returns bearing to this cave from topcamp in format e.g. 'NE' - return 0 - def last_visit(self): #returns Survey class instance of the most recent visit - return 0 + return methods_millenial.top_camp_bearing_letter(self.entrance) + def lat_lon_entrance(self): #lat_lon entrance location + return methods_millenial.lat_lon_entrance(self.entrance) + + +class Cave_descriptionM(models.Model): #instance of this class corresponds to each of the .html files in descriptions + #each of those holds one XML field + slug = models.TextField() + explorers = models.TextField() + underground_description = models.TextField() + equipment = models.TextField() + references = models.TextField() + survey = models.TextField() + kataster_status = models.TextField() + underground_centre_line = models.TextField() + survex_file = models.TextField() #as given in .html file + notes = models.TextField() + + class ExpeditionM(models.Model): #instance of this class corresponds to one expo (usually one year) date = models.CharField(max_length=100) #date in format YYYY.MM.DD-YYYY.MM.DD @@ -57,7 +74,7 @@ class SurveyM(models.Model): #instance of this class corresponds to one .svx fil class Logbook_entryM(models.Model): #instance of this class corresponds to one bit of logbook (c.f. expo.survex.com/years/2015/logbook.html or simil) date = models.CharField(max_length=100) #date as typed into logbook - title = models.TextField() #contents of the logbook chunk + contents = models.TextField() #contents of the logbook chunk class Parser_messageM(models.Model): #instance of this class contains one error or warining message produce by any of the parsers parsername = models.CharField(max_length = 20) #name of parser diff --git a/core/views_caves.py b/core/views_caves.py index a785170..302003d 100644 --- a/core/views_caves.py +++ b/core/views_caves.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation +from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation, CaveM, Cave_descriptionM from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm import troggle.core.models as models import troggle.settings as settings @@ -21,6 +21,17 @@ import settings from PIL import Image, ImageDraw, ImageFont import string, os, sys, subprocess +def millenialcaves(request): + #RW messing around area + caves = CaveM.objects.all() + descr = Cave_descriptionM.objects.all() + return render_with_context(request,'millenialcaves.html',{'caves': caves,'descriptions' : descr}) + +def millenialdescription(request, slug): + desc = Cave_descriptionM.objects.get(slug=slug) + return render_with_context(request,'cave_uground_description.html', {'cave': desc}) + + def getCave(cave_id): """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm.""" try: @@ -54,16 +65,16 @@ def caveindex(request): caves = Cave.objects.all() notablecavehrefs = settings.NOTABLECAVESHREFS notablecaves = [Cave.objects.get(kataster_number=kataster_number) for kataster_number in notablecavehrefs ] - caves1623 = list(Cave.objects.filter(area__short_name = "1623")) + #caves1623 = list(Cave.objects.filter(area__short_name = "1623")) + caves1623 = list(Cave.objects.all()) caves1626 = list(Cave.objects.filter(area__short_name = "1626")) caves1623.sort(caveCmp) caves1626.sort(caveCmp) return render_with_context(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':notablecaves, 'cavepage': True}) -def millenialcaves(request): - #RW messing around area - return HttpResponse("Test text", content_type="text/plain") - + + + def cave3d(request, cave_id=''): diff --git a/databaseResetM.py b/databaseResetM.py index 07e4702..4530763 100644 --- a/databaseResetM.py +++ b/databaseResetM.py @@ -9,7 +9,7 @@ from django.contrib.auth.models import User from django.http import HttpResponse from django.core.urlresolvers import reverse from troggle.core.models import Cave, Entrance -from troggle.core.models import PersonM, SurveyM, CaveM, ExpeditionM, Logbook_entryM +from troggle.core.models import PersonM, SurveyM, CaveM, ExpeditionM, Logbook_entryM, Cave_descriptionM import troggle.flatpages.models databasename=settings.DATABASES['default']['NAME'] @@ -42,6 +42,7 @@ def gracefull_flush(): SurveyM.objects.all().delete() ExpeditionM.objects.all().delete() Logbook_entryM.objects.all().delete() + Cave_descriptionM.objects.all().delete() print('Deleted contents of the database, ready to load new stuff :)') def load_redirects(): diff --git a/media/scripts/TableSort.js b/media/scripts/TableSort.js index 39f5e92..1ab80fd 100644 --- a/media/scripts/TableSort.js +++ b/media/scripts/TableSort.js @@ -34,17 +34,18 @@ function filterTable(tablename) rows = table.rows; for(i=1; i< rows.length; i++) { - name = (rows[i].getElementsByTagName("TD")[0]).innerHTML.toLowerCase(); + name = (rows[i].getElementsByTagName("TD")[1]).innerHTML.toLowerCase(); - depth = (rows[i].getElementsByTagName("TD")[1]).innerHTML.toLowerCase(); + depth = (rows[i].getElementsByTagName("TD")[2]).innerHTML.toLowerCase(); depth = Number(depth.replace(/[^0-9.]/g,'')); - length = (rows[i].getElementsByTagName("TD")[2]).innerHTML.toLowerCase(); + length = (rows[i].getElementsByTagName("TD")[3]).innerHTML.toLowerCase(); length = Number(length.replace(/[^0-9.]/g,'')); - date = (rows[i].getElementsByTagName("TD")[3]).innerHTML.toLowerCase(); + date = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase(); - recentvisitor = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase(); + //recentvisitor = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase(); + recentvisitor = "" if(cavename != "" && !name.includes(cavename)) { diff --git a/parsers/cavesM.py b/parsers/cavesM.py index 13cd5d5..71bda0d 100644 --- a/parsers/cavesM.py +++ b/parsers/cavesM.py @@ -19,16 +19,23 @@ def load(): print('Hi! I\'m caves parser. Ready to work') print('Loading caves of 1623 area') - loadarea('caves-1623/') + loadarea('1623') def loadarea(areacode): + if not file_exists(settings.SURVEX_DATA+'1623-and-1626.3d'): + print('Computing master .3d file') + bash('cavern -o'+settings.SURVEX_DATA+' '+settings.SURVEX_DATA+'1623-and-1626.svx') + else: + print('Loading from existing master .3d file') + + master3d = bash('dump3d -d '+settings.SURVEX_DATA+'1623-and-1626.3d').splitlines() + master3dN = [x for x in master3d if ('NODE' in x)] #list of nodes of master survex file + master3dL = [x for x in master3d if ('LINE' in x)] #list of nodes of master survex file print('Searching all cave dirs files') - basedir = settings.SURVEX_DATA+areacode - - bash('cavern -o'+settings.SURVEX_DATA+' '+settings.SURVEX_DATA+'1623-and-1626.svx') + basedir = settings.SURVEX_DATA+'caves-'+areacode+'/' cavedirs = bash("find "+basedir+" -maxdepth 1 -type d").splitlines() #this command finds all directories print('Obtained list of directories! (#dirs='+str(len(cavedirs))+')') @@ -40,14 +47,14 @@ def loadarea(areacode): cavename = bash('echo '+cavedir+' | rev | cut -f1 -d \'/\' | rev').splitlines()[0] #get final bit of the directory test = bash('if [ ! -f '+cavedir+'/'+cavename+'.svx ] ; then echo MISSING; fi')#test for file exisence - if 'MISSING' in test: #send error message to the database + if not file_exists(cavedir+'/'+cavename+'.svx'): msg = models.Parser_messageM(parsername='caves',content=cavedir+'/'+cavename+' MISSING!',message_type='warn') print('Cave missing'+cavename+' :(') msg.save() continue fullname=cavedir+'/'+cavename+'.svx' print('Found cave:'+cavename) - cavernout = bash('cavern -q '+fullname) #make cavern process the thing + cavernout = bash('cavern -o '+cavedir+' '+fullname) #make cavern process the thing if 'cavern: error:' in cavernout: msg = models.Parser_messageM(parsername='caves',content=cavedir+'/'+cavename+' Survex file messed up!',message_type='warn') print('Fucked svx'+cavename+' :(') @@ -57,28 +64,71 @@ def loadarea(areacode): cavernout = cavernout.splitlines() depth = float(([x for x in cavernout if ('Total vertical length' in x)][0].split()[-1])[:-2]) length = float(([x for x in cavernout if ('Total length' in x)][0].split()[6])[:-1]) - surveyname = bash('cat '+fullname+' | grep \'\*begin\' | head -n1 | cut -f2 -d \' \' ').splitlines().pop() - title = (bash('cat '+fullname+' | grep \'\*title\' | head -n1 | cut -f2 -d \' \' ').splitlines() or ["Not found"])[0] - print((('depth','length','surv name'),(depth,length,surveyname))) - print('dump3d '+settings.SURVEX_DATA+'1623-and-1626.3d | grep NODE | grep \'\\[\\.'+surveyname+'.*\\]\'') - nodes = bash('dump3d '+settings.SURVEX_DATA+'1623-and-1626.3d | grep NODE | grep \'\\[.*\\.'+surveyname+'.*\\]\'').splitlines() - entran = [x for x in nodes if ('ENTRANCE' in x) ] - print(nodes) + cavefile = open(fullname,'r') + cavefilecontents = cavefile.read().splitlines() + surveyname = [x for x in cavefilecontents if ('*begin ') in x][0].split()[1].lower() + try: + title = [x for x in cavefilecontents if ('*title ') in x][0].split()[1] + except: + syrveyname = "Untitled" + relevant_nodes = [x for x in master3dN if (('['+areacode+'.'+surveyname+'.' in x) or ('['+areacode+'.'+surveyname+']' in x))] + entrance_nodes = [x for x in relevant_nodes if 'ENTRANCE' in x] + surface_nodes = [x for x in relevant_nodes if 'SURFACE' in x] + location_nodes = [] + print('rel_nodes'+str(len(relevant_nodes))) + if len(entrance_nodes) > 0: + location_nodes = entrance_nodes + elif len(surface_nodes) > 0: + location_nodes = surface_nodes + elif len(relevant_nodes) > 0: + location_nodes = relevant_nodes - newcave = models.CaveM(survex_file = fullname, total_length = length, name=title, total_depth = depth) + try: + location = sorted(location_nodes, key = lambda y : float(y.split()[3])).pop() + except: + print(location_nodes) + location = 'Not found' + + relevant_lines = [x for x in master3dL if (('['+areacode+'.'+surveyname+'.' in x) or ('['+areacode+'.'+surveyname+']' in x))] + try: + lastleg = sorted(relevant_lines, key = lambda y : y.split().pop()).pop() + except: + lastleg = ['LINE 1900.01.01'] + try: + lastdate = lastleg.split().pop() + if 'STYLE' in lastdate: + lastdate = lastleg.split().pop().pop() + except: + lastdate = '1900.01.01' + + entrance = ' '.join(location.split()[1:3]) + print((('depth','length','surv name','entr','date'),(depth,length,surveyname,entrance,lastdate))) #sanity check print + + + newcave = models.CaveM( + survex_file = fullname, + total_length = length, + name=areacode+'.'+surveyname, + total_depth = depth, + date = lastdate, + entrance = entrance) newcave.save() #end of reading survex masterfiles - + print ("Reading cave descriptions") cavefiles = bash('find '+settings.CAVEDESCRIPTIONS+' -name \'*.html\'').splitlines() for fn in cavefiles: f = open(fn, "r") print(fn) contents = f.read() - + + slug = re.sub(r"\s+", "", extractXML(contents,'caveslug')) desc = extractXML(contents,'underground_description') - name = re.search(r'>.*<',extractXML(contents,'caveslug')).group()[6:-1] + name = slug[5:] #get survex compatible name + area = slug[0:4] + + print([area,name]) if desc==None or name==None: msg = models.Parser_messageM(parsername='caves',content=fn+' Description meesed up!',message_type='warn') @@ -86,26 +136,56 @@ def loadarea(areacode): msg.save() continue + print(area+'/'+name+'/'+name+'.svx') - updatecave = models.CaveM.objects.filter(survex_file__icontains='/'+name+'.svx') + updatecave = models.CaveM.objects.filter(survex_file__icontains=area+'/'+name+'/'+name+'.svx') if len(updatecave)>1: print('Non unique solution - skipping. Name:'+name) elif len(updatecave)==0: - print('Cave with no survex data'+name) - newcave = models.CaveM(description = desc, name = name) - newcave.save() + print('Cave with no survex data:'+name) + continue else: #exaclty one match + print('Adding desc:'+name) updatecave = updatecave[0] - updatecave.description = desc - if updatecave.name=="Not found": - updatecave.name=name + updatecave.description = '/cave/descriptionM/'+slug #area-name updatecave.title=name updatecave.save() - + slugS = slug + explorersS = extractXML(contents,'explorers') + underground_descriptionS = extractXML(contents,'underground_description') + equipmentS = extractXML(contents,'equipment') + referencesS = extractXML(contents,'references') + surveyS = extractXML(contents,'survey') + kataster_statusS = extractXML(contents,'kataster_status') + underground_centre_lineS = extractXML(contents,'underground_centre_line') + survex_fileS = extractXML(contents,'survex_file') + notesS = extractXML(contents,'notes') + + + newcavedesc = models.Cave_descriptionM( + slug = slugS, + explorers = explorersS, + underground_description = underground_descriptionS, + equipment = equipmentS, + references = referencesS, + survey = surveyS, + kataster_status = kataster_statusS, + underground_centre_line = underground_centre_lineS, + survex_file = survex_fileS, + notes = notesS) + newcavedesc.save() + + + + #end of reading cave descriptions - +def file_exists(filename): + test = bash('if [ ! -f '+filename+' ] ; then echo MISSING; fi')#test for file exisence + if 'MISSING' in test: #send error message to the database + return False + return True def extractXML(contents,tag): #find correct lines @@ -119,9 +199,10 @@ def extractXML(contents,tag): if endi!=begi: segment = '\n'.join(lines[begi:endi+1]) else: - segment = lines[begi:endi+1] - return segment[0] - + segment = lines[begi:endi+1][0] + + hit = re.findall('<'+tag+'>(.*)', segment, re.S)[0] + return hit def bash(cmd): #calls command in bash shell, returns output process = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) diff --git a/settings.py b/settings.py index 3c3b8d2..364b45d 100644 --- a/settings.py +++ b/settings.py @@ -71,6 +71,9 @@ if django.VERSION[0] == 1 and django.VERSION[1] < 4: else: authmodule = 'django.contrib.auth.context_processors.auth' +TOPCAMPX=411571.00 +TOPCAMPY=5282639.00 + TEMPLATE_CONTEXT_PROCESSORS = ( authmodule, "core.context.troggle_context", ) LOGIN_REDIRECT_URL = '/' diff --git a/templates/caveindex.html b/templates/caveindex.html index be53ae3..53f3dbe 100644 --- a/templates/caveindex.html +++ b/templates/caveindex.html @@ -20,7 +20,7 @@ {% for cave in caves1623 %} - + {% endfor %}
{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}
{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}} {{ cave.slug }}
diff --git a/templates/millenialcaves.html b/templates/millenialcaves.html index 8e8c5c6..73372f8 100644 --- a/templates/millenialcaves.html +++ b/templates/millenialcaves.html @@ -8,116 +8,62 @@ - +

Caves of loserplateau (locations acording to all.3d)

+ +Name contains:
+

+Depth between (min, max) in meters (0 disables filter):
+
-

+Length between (min, max) in meters (0 disables filter):
+
-

+Last visit after (date in YYYY.MM.DD format works best):
+

+Last visited by (single word or regular expression, search is not case sensitive):
+(e.g. /da.e/ matches both Dave and Dane, /w..k|ol{2}y/ matches either Wook and Olly)
+

+Hide incomplete entries:
+


+ +
+
+ +Click on column headers to sort/reverse sort


+ + + + + +
- - - - + + + + + + + + - -{% for subdircave, cavefiles, subsurvdirs in subdircaves %} +{% for cave in caves %} - - + + + + + + + + {% endfor %} -{% for primarycavefile, subcavefiles in multifilecaves %} - - - - -{% endfor %} - -
CaveComponentsaaabbbCave survex idCave nameCave depthCave lengthLast leg dateCave location (UTM)Cave location (lat/lon)Top camp distance [m]
{{cavefiles.0.1}} - root: {% for cavepath, cavename in cavefiles.1 %} - {{cavename}} - {% endfor %} -
- {% for primarycavefile, subcavefiles in subsurvdirs %} - {{ primarycavefile.1 }}: - {% for cavepath, cavename in subcavefiles %} - {{cavename}} - {% endfor %} -
- {% endfor %} - 		{{ cave.name }}{{ cave.title }}{{ cave.total_depth }}{{ cave.total_length }}{{ cave.date }}33U {{ cave.entrance }}{{ cave.lat_lon_entrance }}{{ cave.top_camp_distance}}
- {{primarycavefile.1}} - - {% for cavepath, cavename in subcavefiles %} - {{cavename}} - {% endfor %} -
+ + -======================= OLD STUFF ============================== - -{{cavefiles.0.1}} - -

caves with subdirectories | caves with multiple files | caves with single files

- -

Link to all.svx for processing

- -

Caves with subdirectories

- -{% for subdircave, cavefiles, subsurvdirs in subdircaves %} -

{{cavefiles.0.1}} - dates and explorers

- - - - - - -{% for primarycavefile, subcavefiles in subsurvdirs %} - - - - -{% endfor %} -
{{cavefiles.0.1}} - {% for cavepath, cavename in cavefiles.1 %} - {{cavename}} - {% endfor %} -
{{primarycavefile.1}} - {% for cavepath, cavename in subcavefiles %} - {{cavename}} - {% endfor %} -
- -{% endfor %} - - -

Caves of multiple files

- - -{% for primarycavefile, subcavefiles in multifilecaves %} - - - - -{% endfor %} -
Dates and explorersSurvex files
- {{primarycavefile.1}} - - {{primarycavefile.1}} - - {% for cavepath, cavename in subcavefiles %} - {{cavename}} - {% endfor %} -
- -

Caves of one file

-

-{% for cavepath, cavename in onefilecaves %} - {{cavename}} -{% endfor %} -

- - - diff --git a/urls.py b/urls.py index 7022d61..a38b7a4 100644 --- a/urls.py +++ b/urls.py @@ -23,7 +23,9 @@ admin.autodiscover() actualurlpatterns = patterns('', - url(r'^millenialcaves/?$', views_survex.millenialcaves, name="millenialcaves"), + url(r'^millenialcaves/?$', views_caves.millenialcaves, name="millenialcaves"), + url(r'^cave/descriptionM/([^/]+)/?$', views_caves.millenialdescription), + #url(r'^cave/description/([^/]+)/?$', views_caves.caveDescription), url(r'^troggle$', views_other.frontpage, name="frontpage"), url(r'^todo/$', views_other.todo, name="todo"),