From e01507d54112cfc61748eeff27a3a264e1ad7516 Mon Sep 17 00:00:00 2001 From: Sam Wenham Date: Mon, 4 Mar 2019 19:39:43 +0000 Subject: [PATCH] Backed out changeset: f23440eb11a3 --- core/methods_millenial.py | 26 +----- core/models_millenial.py | 33 ++------ core/views_caves.py | 23 ++---- databaseResetM.py | 3 +- media/scripts/TableSort.js | 11 ++- parsers/cavesM.py | 139 +++++++------------------------- settings.py | 3 - templates/caveindex.html | 2 +- templates/millenialcaves.html | 144 +++++++++++++++++++++++----------- urls.py | 4 +- 10 files changed, 152 insertions(+), 236 deletions(-) diff --git a/core/methods_millenial.py b/core/methods_millenial.py index ce7ce11..5d2483a 100644 --- a/core/methods_millenial.py +++ b/core/methods_millenial.py @@ -1,24 +1,2 @@ -import utm -import math -from django.conf import settings - -def lat_lon_entrance(utmstring): - try: - x = float(utmstring.split()[0]) - y = float(utmstring.split()[1]) - #return ' '+str(x+y)+' '+str(y) - q = utm.to_latlon(x, y, 33, 'U') - return "{:.5f} {:.5f}".format(q[0],q[1]) - except: - return 'Not found' - -def top_camp_distance(utmstring): - try: - x = float(utmstring.split()[0]) - y = float(utmstring.split()[1]) - tx = settings.TOPCAMPX - ty = settings.TOPCAMPY - dist = math.sqrt( (tx-x)*(tx-x) + (ty-y)*(ty-y) ) - return "{:.1f}".format(dist) - except: - return 'Not found' +def emptyfun(): + return diff --git a/core/models_millenial.py b/core/models_millenial.py index c97be36..db893bc 100644 --- a/core/models_millenial.py +++ b/core/models_millenial.py @@ -1,7 +1,7 @@ from django.db import models from django.conf import settings -import troggle.core.methods_millenial as methods_millenial +from troggle.core.methods_millenial import * # # This file was created in 2019 @@ -37,32 +37,15 @@ class CaveM(models.Model): #instance of this class corresponds to one 'thing' th survex_file = models.TextField() #gives path to top level survex file total_length = models.FloatField() #holds total length of this cave (as given by cavern) total_depth = models.FloatField() #holds total depth of this cave (as given by cavern) - description = models.TextField() #holds link to description - date = models.TextField() #holds date of last visit + description = models.TextField() #holds decription of the cave def top_camp_distance(self): #returns distance of this cave from topcamp - return methods_millenial.top_camp_distance(self.entrance) + return 0 def top_camp_bearing(self): #returns bearing to this cave from topcamp in format 235.5 (float north-based azimuth) - return methods_millenial.top_camp_bearing(self.entrance) + return 0 def top_camp_bearing_letter(self): #returns bearing to this cave from topcamp in format e.g. 'NE' - return methods_millenial.top_camp_bearing_letter(self.entrance) - def lat_lon_entrance(self): #lat_lon entrance location - return methods_millenial.lat_lon_entrance(self.entrance) - - -class Cave_descriptionM(models.Model): #instance of this class corresponds to each of the .html files in descriptions - #each of those holds one XML field - slug = models.TextField() - explorers = models.TextField() - underground_description = models.TextField() - equipment = models.TextField() - references = models.TextField() - survey = models.TextField() - kataster_status = models.TextField() - underground_centre_line = models.TextField() - survex_file = models.TextField() #as given in .html file - notes = models.TextField() - - + return 0 + def last_visit(self): #returns Survey class instance of the most recent visit + return 0 class ExpeditionM(models.Model): #instance of this class corresponds to one expo (usually one year) date = models.CharField(max_length=100) #date in format YYYY.MM.DD-YYYY.MM.DD @@ -74,7 +57,7 @@ class SurveyM(models.Model): #instance of this class corresponds to one .svx fil class Logbook_entryM(models.Model): #instance of this class corresponds to one bit of logbook (c.f. expo.survex.com/years/2015/logbook.html or simil) date = models.CharField(max_length=100) #date as typed into logbook - contents = models.TextField() #contents of the logbook chunk + title = models.TextField() #contents of the logbook chunk class Parser_messageM(models.Model): #instance of this class contains one error or warining message produce by any of the parsers parsername = models.CharField(max_length = 20) #name of parser diff --git a/core/views_caves.py b/core/views_caves.py index 302003d..a785170 100644 --- a/core/views_caves.py +++ b/core/views_caves.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation, CaveM, Cave_descriptionM +from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm import troggle.core.models as models import troggle.settings as settings @@ -21,17 +21,6 @@ import settings from PIL import Image, ImageDraw, ImageFont import string, os, sys, subprocess -def millenialcaves(request): - #RW messing around area - caves = CaveM.objects.all() - descr = Cave_descriptionM.objects.all() - return render_with_context(request,'millenialcaves.html',{'caves': caves,'descriptions' : descr}) - -def millenialdescription(request, slug): - desc = Cave_descriptionM.objects.get(slug=slug) - return render_with_context(request,'cave_uground_description.html', {'cave': desc}) - - def getCave(cave_id): """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm.""" try: @@ -65,16 +54,16 @@ def caveindex(request): caves = Cave.objects.all() notablecavehrefs = settings.NOTABLECAVESHREFS notablecaves = [Cave.objects.get(kataster_number=kataster_number) for kataster_number in notablecavehrefs ] - #caves1623 = list(Cave.objects.filter(area__short_name = "1623")) - caves1623 = list(Cave.objects.all()) + caves1623 = list(Cave.objects.filter(area__short_name = "1623")) caves1626 = list(Cave.objects.filter(area__short_name = "1626")) caves1623.sort(caveCmp) caves1626.sort(caveCmp) return render_with_context(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':notablecaves, 'cavepage': True}) - - - +def millenialcaves(request): + #RW messing around area + return HttpResponse("Test text", content_type="text/plain") + def cave3d(request, cave_id=''): diff --git a/databaseResetM.py b/databaseResetM.py index 4530763..07e4702 100644 --- a/databaseResetM.py +++ b/databaseResetM.py @@ -9,7 +9,7 @@ from django.contrib.auth.models import User from django.http import HttpResponse from django.core.urlresolvers import reverse from troggle.core.models import Cave, Entrance -from troggle.core.models import PersonM, SurveyM, CaveM, ExpeditionM, Logbook_entryM, Cave_descriptionM +from troggle.core.models import PersonM, SurveyM, CaveM, ExpeditionM, Logbook_entryM import troggle.flatpages.models databasename=settings.DATABASES['default']['NAME'] @@ -42,7 +42,6 @@ def gracefull_flush(): SurveyM.objects.all().delete() ExpeditionM.objects.all().delete() Logbook_entryM.objects.all().delete() - Cave_descriptionM.objects.all().delete() print('Deleted contents of the database, ready to load new stuff :)') def load_redirects(): diff --git a/media/scripts/TableSort.js b/media/scripts/TableSort.js index 1ab80fd..39f5e92 100644 --- a/media/scripts/TableSort.js +++ b/media/scripts/TableSort.js @@ -34,18 +34,17 @@ function filterTable(tablename) rows = table.rows; for(i=1; i< rows.length; i++) { - name = (rows[i].getElementsByTagName("TD")[1]).innerHTML.toLowerCase(); + name = (rows[i].getElementsByTagName("TD")[0]).innerHTML.toLowerCase(); - depth = (rows[i].getElementsByTagName("TD")[2]).innerHTML.toLowerCase(); + depth = (rows[i].getElementsByTagName("TD")[1]).innerHTML.toLowerCase(); depth = Number(depth.replace(/[^0-9.]/g,'')); - length = (rows[i].getElementsByTagName("TD")[3]).innerHTML.toLowerCase(); + length = (rows[i].getElementsByTagName("TD")[2]).innerHTML.toLowerCase(); length = Number(length.replace(/[^0-9.]/g,'')); - date = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase(); + date = (rows[i].getElementsByTagName("TD")[3]).innerHTML.toLowerCase(); - //recentvisitor = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase(); - recentvisitor = "" + recentvisitor = (rows[i].getElementsByTagName("TD")[4]).innerHTML.toLowerCase(); if(cavename != "" && !name.includes(cavename)) { diff --git a/parsers/cavesM.py b/parsers/cavesM.py index 71bda0d..13cd5d5 100644 --- a/parsers/cavesM.py +++ b/parsers/cavesM.py @@ -19,23 +19,16 @@ def load(): print('Hi! I\'m caves parser. Ready to work') print('Loading caves of 1623 area') - loadarea('1623') + loadarea('caves-1623/') def loadarea(areacode): - if not file_exists(settings.SURVEX_DATA+'1623-and-1626.3d'): - print('Computing master .3d file') - bash('cavern -o'+settings.SURVEX_DATA+' '+settings.SURVEX_DATA+'1623-and-1626.svx') - else: - print('Loading from existing master .3d file') - - master3d = bash('dump3d -d '+settings.SURVEX_DATA+'1623-and-1626.3d').splitlines() - master3dN = [x for x in master3d if ('NODE' in x)] #list of nodes of master survex file - master3dL = [x for x in master3d if ('LINE' in x)] #list of nodes of master survex file print('Searching all cave dirs files') - basedir = settings.SURVEX_DATA+'caves-'+areacode+'/' + basedir = settings.SURVEX_DATA+areacode + + bash('cavern -o'+settings.SURVEX_DATA+' '+settings.SURVEX_DATA+'1623-and-1626.svx') cavedirs = bash("find "+basedir+" -maxdepth 1 -type d").splitlines() #this command finds all directories print('Obtained list of directories! (#dirs='+str(len(cavedirs))+')') @@ -47,14 +40,14 @@ def loadarea(areacode): cavename = bash('echo '+cavedir+' | rev | cut -f1 -d \'/\' | rev').splitlines()[0] #get final bit of the directory test = bash('if [ ! -f '+cavedir+'/'+cavename+'.svx ] ; then echo MISSING; fi')#test for file exisence - if not file_exists(cavedir+'/'+cavename+'.svx'): + if 'MISSING' in test: #send error message to the database msg = models.Parser_messageM(parsername='caves',content=cavedir+'/'+cavename+' MISSING!',message_type='warn') print('Cave missing'+cavename+' :(') msg.save() continue fullname=cavedir+'/'+cavename+'.svx' print('Found cave:'+cavename) - cavernout = bash('cavern -o '+cavedir+' '+fullname) #make cavern process the thing + cavernout = bash('cavern -q '+fullname) #make cavern process the thing if 'cavern: error:' in cavernout: msg = models.Parser_messageM(parsername='caves',content=cavedir+'/'+cavename+' Survex file messed up!',message_type='warn') print('Fucked svx'+cavename+' :(') @@ -64,71 +57,28 @@ def loadarea(areacode): cavernout = cavernout.splitlines() depth = float(([x for x in cavernout if ('Total vertical length' in x)][0].split()[-1])[:-2]) length = float(([x for x in cavernout if ('Total length' in x)][0].split()[6])[:-1]) - cavefile = open(fullname,'r') - cavefilecontents = cavefile.read().splitlines() - surveyname = [x for x in cavefilecontents if ('*begin ') in x][0].split()[1].lower() - try: - title = [x for x in cavefilecontents if ('*title ') in x][0].split()[1] - except: - syrveyname = "Untitled" + surveyname = bash('cat '+fullname+' | grep \'\*begin\' | head -n1 | cut -f2 -d \' \' ').splitlines().pop() + title = (bash('cat '+fullname+' | grep \'\*title\' | head -n1 | cut -f2 -d \' \' ').splitlines() or ["Not found"])[0] + print((('depth','length','surv name'),(depth,length,surveyname))) + print('dump3d '+settings.SURVEX_DATA+'1623-and-1626.3d | grep NODE | grep \'\\[\\.'+surveyname+'.*\\]\'') + nodes = bash('dump3d '+settings.SURVEX_DATA+'1623-and-1626.3d | grep NODE | grep \'\\[.*\\.'+surveyname+'.*\\]\'').splitlines() + entran = [x for x in nodes if ('ENTRANCE' in x) ] + print(nodes) - relevant_nodes = [x for x in master3dN if (('['+areacode+'.'+surveyname+'.' in x) or ('['+areacode+'.'+surveyname+']' in x))] - entrance_nodes = [x for x in relevant_nodes if 'ENTRANCE' in x] - surface_nodes = [x for x in relevant_nodes if 'SURFACE' in x] - location_nodes = [] - print('rel_nodes'+str(len(relevant_nodes))) - if len(entrance_nodes) > 0: - location_nodes = entrance_nodes - elif len(surface_nodes) > 0: - location_nodes = surface_nodes - elif len(relevant_nodes) > 0: - location_nodes = relevant_nodes - try: - location = sorted(location_nodes, key = lambda y : float(y.split()[3])).pop() - except: - print(location_nodes) - location = 'Not found' - - relevant_lines = [x for x in master3dL if (('['+areacode+'.'+surveyname+'.' in x) or ('['+areacode+'.'+surveyname+']' in x))] - try: - lastleg = sorted(relevant_lines, key = lambda y : y.split().pop()).pop() - except: - lastleg = ['LINE 1900.01.01'] - try: - lastdate = lastleg.split().pop() - if 'STYLE' in lastdate: - lastdate = lastleg.split().pop().pop() - except: - lastdate = '1900.01.01' - - entrance = ' '.join(location.split()[1:3]) - print((('depth','length','surv name','entr','date'),(depth,length,surveyname,entrance,lastdate))) #sanity check print - - - newcave = models.CaveM( - survex_file = fullname, - total_length = length, - name=areacode+'.'+surveyname, - total_depth = depth, - date = lastdate, - entrance = entrance) + newcave = models.CaveM(survex_file = fullname, total_length = length, name=title, total_depth = depth) newcave.save() #end of reading survex masterfiles - + print ("Reading cave descriptions") cavefiles = bash('find '+settings.CAVEDESCRIPTIONS+' -name \'*.html\'').splitlines() for fn in cavefiles: f = open(fn, "r") print(fn) contents = f.read() - - slug = re.sub(r"\s+", "", extractXML(contents,'caveslug')) + desc = extractXML(contents,'underground_description') - name = slug[5:] #get survex compatible name - area = slug[0:4] - - print([area,name]) + name = re.search(r'>.*<',extractXML(contents,'caveslug')).group()[6:-1] if desc==None or name==None: msg = models.Parser_messageM(parsername='caves',content=fn+' Description meesed up!',message_type='warn') @@ -136,56 +86,26 @@ def loadarea(areacode): msg.save() continue - print(area+'/'+name+'/'+name+'.svx') - updatecave = models.CaveM.objects.filter(survex_file__icontains=area+'/'+name+'/'+name+'.svx') + updatecave = models.CaveM.objects.filter(survex_file__icontains='/'+name+'.svx') if len(updatecave)>1: print('Non unique solution - skipping. Name:'+name) elif len(updatecave)==0: - print('Cave with no survex data:'+name) - continue + print('Cave with no survex data'+name) + newcave = models.CaveM(description = desc, name = name) + newcave.save() else: #exaclty one match - print('Adding desc:'+name) updatecave = updatecave[0] - updatecave.description = '/cave/descriptionM/'+slug #area-name + updatecave.description = desc + if updatecave.name=="Not found": + updatecave.name=name updatecave.title=name updatecave.save() + - slugS = slug - explorersS = extractXML(contents,'explorers') - underground_descriptionS = extractXML(contents,'underground_description') - equipmentS = extractXML(contents,'equipment') - referencesS = extractXML(contents,'references') - surveyS = extractXML(contents,'survey') - kataster_statusS = extractXML(contents,'kataster_status') - underground_centre_lineS = extractXML(contents,'underground_centre_line') - survex_fileS = extractXML(contents,'survex_file') - notesS = extractXML(contents,'notes') - - - newcavedesc = models.Cave_descriptionM( - slug = slugS, - explorers = explorersS, - underground_description = underground_descriptionS, - equipment = equipmentS, - references = referencesS, - survey = surveyS, - kataster_status = kataster_statusS, - underground_centre_line = underground_centre_lineS, - survex_file = survex_fileS, - notes = notesS) - newcavedesc.save() - - - - #end of reading cave descriptions -def file_exists(filename): - test = bash('if [ ! -f '+filename+' ] ; then echo MISSING; fi')#test for file exisence - if 'MISSING' in test: #send error message to the database - return False - return True + def extractXML(contents,tag): #find correct lines @@ -199,10 +119,9 @@ def extractXML(contents,tag): if endi!=begi: segment = '\n'.join(lines[begi:endi+1]) else: - segment = lines[begi:endi+1][0] - - hit = re.findall('<'+tag+'>(.*)', segment, re.S)[0] - return hit + segment = lines[begi:endi+1] + return segment[0] + def bash(cmd): #calls command in bash shell, returns output process = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) diff --git a/settings.py b/settings.py index 364b45d..3c3b8d2 100644 --- a/settings.py +++ b/settings.py @@ -71,9 +71,6 @@ if django.VERSION[0] == 1 and django.VERSION[1] < 4: else: authmodule = 'django.contrib.auth.context_processors.auth' -TOPCAMPX=411571.00 -TOPCAMPY=5282639.00 - TEMPLATE_CONTEXT_PROCESSORS = ( authmodule, "core.context.troggle_context", ) LOGIN_REDIRECT_URL = '/' diff --git a/templates/caveindex.html b/templates/caveindex.html index 53f3dbe..be53ae3 100644 --- a/templates/caveindex.html +++ b/templates/caveindex.html @@ -20,7 +20,7 @@ {% for cave in caves1623 %} - + {% endfor %}
{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}} {{ cave.slug }}
{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}
diff --git a/templates/millenialcaves.html b/templates/millenialcaves.html index 73372f8..8e8c5c6 100644 --- a/templates/millenialcaves.html +++ b/templates/millenialcaves.html @@ -8,62 +8,116 @@ -

Caves of loserplateau (locations acording to all.3d)

- -Name contains:
-

-Depth between (min, max) in meters (0 disables filter):
-
-

-Length between (min, max) in meters (0 disables filter):
-
-

-Last visit after (date in YYYY.MM.DD format works best):
-

-Last visited by (single word or regular expression, search is not case sensitive):
-(e.g. /da.e/ matches both Dave and Dane, /w..k|ol{2}y/ matches either Wook and Olly)
-

-Hide incomplete entries:
-


- -
-
- -Click on column headers to sort/reverse sort


- - - - - - +
- - - - - - - - + + + + -{% for cave in caves %} + +{% for subdircave, cavefiles, subsurvdirs in subdircaves %} - - - - - - - - + + {% endfor %} +{% for primarycavefile, subcavefiles in multifilecaves %} + + + + +{% endfor %} + +
Cave survex idCave nameCave depthCave lengthLast leg dateCave location (UTM)Cave location (lat/lon)Top camp distance [m]CaveComponentsaaabbb
{{ cave.name }}{{ cave.title }}{{ cave.total_depth }}{{ cave.total_length }}{{ cave.date }}33U {{ cave.entrance }}{{ cave.lat_lon_entrance }}{{ cave.top_camp_distance}}{{cavefiles.0.1}} + root: {% for cavepath, cavename in cavefiles.1 %} + {{cavename}} + {% endfor %} +
+ {% for primarycavefile, subcavefiles in subsurvdirs %} + {{ primarycavefile.1 }}: + {% for cavepath, cavename in subcavefiles %} + {{cavename}} + {% endfor %} +
+ {% endfor %} +
+ {{primarycavefile.1}} + + {% for cavepath, cavename in subcavefiles %} + {{cavename}} + {% endfor %} +
- - +======================= OLD STUFF ============================== + +{{cavefiles.0.1}} + +

caves with subdirectories | caves with multiple files | caves with single files

+ +

Link to all.svx for processing

+ +

Caves with subdirectories

+ +{% for subdircave, cavefiles, subsurvdirs in subdircaves %} +

{{cavefiles.0.1}} - dates and explorers

+ + + + + + +{% for primarycavefile, subcavefiles in subsurvdirs %} + + + + +{% endfor %} +
{{cavefiles.0.1}} + {% for cavepath, cavename in cavefiles.1 %} + {{cavename}} + {% endfor %} +
{{primarycavefile.1}} + {% for cavepath, cavename in subcavefiles %} + {{cavename}} + {% endfor %} +
+ +{% endfor %} + + +

Caves of multiple files

+ + +{% for primarycavefile, subcavefiles in multifilecaves %} + + + + +{% endfor %} +
Dates and explorersSurvex files
+ {{primarycavefile.1}} + + {{primarycavefile.1}} - + {% for cavepath, cavename in subcavefiles %} + {{cavename}} + {% endfor %} +
+ +

Caves of one file

+

+{% for cavepath, cavename in onefilecaves %} + {{cavename}} +{% endfor %} +

+ + + diff --git a/urls.py b/urls.py index a38b7a4..7022d61 100644 --- a/urls.py +++ b/urls.py @@ -23,9 +23,7 @@ admin.autodiscover() actualurlpatterns = patterns('', - url(r'^millenialcaves/?$', views_caves.millenialcaves, name="millenialcaves"), - url(r'^cave/descriptionM/([^/]+)/?$', views_caves.millenialdescription), - #url(r'^cave/description/([^/]+)/?$', views_caves.caveDescription), + url(r'^millenialcaves/?$', views_survex.millenialcaves, name="millenialcaves"), url(r'^troggle$', views_other.frontpage, name="frontpage"), url(r'^todo/$', views_other.todo, name="todo"),