expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-22 07:11:52 +00:00
Code Issues 19 Projects 3 Activity

Refactor to avoid loading unused surveystations

Browse Source
This commit is contained in:
Philip Sargent 2020-05-28 01:16:45 +01:00
parent 73637ba53d
commit 6cc578435c
6 changed files with 103 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.gitignore vendored
View File

@ -3,6 +3,8 @@ syntax: glob
*.pyc
db*
*.sqlite
*.sql
localsettings.py
*~
parsing_log.txt
@ -13,22 +15,15 @@ troggle_log.txt
media/images/*
.vscode/*
.swp
imagekit-off/
localsettings-expo-live.py
.gitignore
desktop.ini
troggle-reset.log
troggle-reset0.log
troggle-surveys.log
troggle.log
troggle.sqlite
troggle.sqlite.0
troggle.sqlite.1
my_project.dot
memdump.sql
troggle-sqlite.sql
import_profile.json
import_times.json
ignored-files.log
tunnel-import.log
posnotfound

5
core/models.py
View File

@ -112,7 +112,6 @@ class Expedition(TroggleModel):
res = self.expeditionday_set.all()
return res and res[len(res) - 1] or None
class ExpeditionDay(TroggleModel):
expedition = models.ForeignKey("Expedition")
date = models.DateField()
@ -183,7 +182,6 @@ class Person(TroggleModel):
#self.orderref = self.first_name
#self.notability = 0.0 # set temporarily
class PersonExpedition(TroggleModel):
"""Person's attendance to one Expo
"""
@ -320,7 +318,6 @@ class LogbookEntry(TroggleModel):
def DayIndex(self):
return list(self.expeditionday.logbookentry_set.all()).index(self)
#
# Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry)
#
@ -359,7 +356,6 @@ class PersonTrip(TroggleModel):
return "%s (%s)" % (self.personexpedition, self.logbook_entry.date)
##########################################
# move following classes into models_cave
##########################################
@ -392,7 +388,6 @@ class CaveSlug(models.Model):
slug = models.SlugField(max_length=50, unique = True)
primary = models.BooleanField(default=False)
class Cave(TroggleModel):
# too much here perhaps,
official_name = models.CharField(max_length=160)

12
core/models_survex.py
View File

@ -1,8 +1,9 @@
from django.db import models
from django.conf import settings
import os
import urllib.parse
import re
from django.db import models
from django.conf import settings
from django.core.urlresolvers import reverse
@ -22,6 +23,7 @@ class SurvexDirectory(models.Model):
class Meta:
ordering = ('id',)
class SurvexFile(models.Model):
path = models.CharField(max_length=200)
survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True)
@ -49,6 +51,7 @@ class SurvexFile(models.Model):
self.survexdirectory = survexdirectory
self.save()
class SurvexEquate(models.Model):
cave = models.ForeignKey('Cave', blank=True, null=True)
@ -87,7 +90,6 @@ class SurvexLeg(models.Model):
compass = models.FloatField()
clino = models.FloatField()
#
# Single SurvexBlock
#
@ -153,7 +155,6 @@ class SurvexBlock(models.Model):
def DayIndex(self):
return list(self.expeditionday.survexblock_set.all()).index(self)
class SurvexTitle(models.Model):
survexblock = models.ForeignKey('SurvexBlock')
title = models.CharField(max_length=200)
@ -187,7 +188,6 @@ class SurvexPersonRole(models.Model):
def __str__(self):
return str(self.person) + " - " + str(self.survexblock) + " - " + str(self.nrole)
class SurvexScansFolder(models.Model):
fpath = models.CharField(max_length=200)
walletname = models.CharField(max_length=200)
@ -215,7 +215,6 @@ class SurvexScanSingle(models.Model):
def __str__(self):
return "Survey Scan Image: " + str(self.name) + " in " + str(self.survexscansfolder)
class TunnelFile(models.Model):
tunnelpath = models.CharField(max_length=200)
tunnelname = models.CharField(max_length=200)
@ -231,4 +230,3 @@ class TunnelFile(models.Model):
class Meta:
ordering = ('tunnelpath',)

103
core/views_caves.py
View File

@ -3,24 +3,52 @@
from __future__ import (absolute_import, division,
print_function, unicode_literals)
from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation
from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
import troggle.core.models as models
import troggle.settings as settings
from troggle.helper import login_required_if_public
from django.forms.models import modelformset_factory
from django import forms
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.conf import settings
import re, urllib.parse
from django.shortcuts import get_object_or_404, render
import sys
import os
import string
import subprocess
import re
import settings
import urllib.parse
from PIL import Image, ImageDraw, ImageFont
import string, os, sys, subprocess
from django import forms
from django.conf import settings
from django.forms.models import modelformset_factory
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
import troggle.settings as settings
import troggle.core.models as models
from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation
from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
from troggle.helper import login_required_if_public
class MapLocations(object):
p = [
("laser.0_7", "BNase", "Reference", "Bräuning Nase laser point"),
("226-96", "BZkn", "Reference", "Bräuning Zinken trig point"),
("vd1","VD1","Reference", "VD1 survey point"),
("laser.kt114_96","HSK","Reference", "Hinterer Schwarzmooskogel trig point"),
("2000","Nipple","Reference", "Nipple (Weiße Warze)"),
("3000","VSK","Reference", "Vorderer Schwarzmooskogel summit"),
("oldtopcamp", "OTC", "Reference", "Old Top Camp"),
("laser.0", "LSR0", "Reference", "Laser Point 0"),
("laser.0_1", "LSR1", "Reference", "Laser Point 0/1"),
("laser.0_3", "LSR3", "Reference", "Laser Point 0/3"),
("laser.0_5", "LSR5", "Reference", "Laser Point 0/5"),
("225-96", "BAlm", "Reference", "Bräuning Alm trig point")
]
def points(self):
for ent in Entrance.objects.all():
if ent.best_station():
areaName = ent.caveandentrance_set.all()[0].cave.getArea().short_name
self.p.append((ent.best_station(), "%s-%s" % (areaName, str(ent)[5:]), ent.needs_surface_work(), str(ent)))
return self.p
def __str__(self):
return "{} map locations".format(len(self.p))
def getCave(cave_id):
"""Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm."""
@ -58,8 +86,6 @@ def millenialcaves(request):
#RW messing around area
return HttpResponse("Test text", content_type="text/plain")
def cave3d(request, cave_id=''):
cave = getCave(cave_id)
survexfilename = settings.SURVEX_DATA + cave.survex_file
@ -217,8 +243,6 @@ def qm(request,cave_id,qm_id,year,grade=None):
url += r'&grade=' + grade
return HttpResponseRedirect(url)
def ent(request, cave_id, ent_letter):
cave = Cave.objects.filter(kataster_number = cave_id)[0]
cave_and_ent = CaveAndEntrance.objects.filter(cave = cave).filter(entrance_letter = ent_letter)[0]
@ -362,7 +386,6 @@ areacolours = {
'7' : '#808080'
}
for FONT in [
"/usr/share/fonts/truetype/freefont/FreeSans.ttf",
"/usr/X11R6/lib/X11/fonts/truetype/arial.ttf",
@ -460,26 +483,10 @@ def prospecting_image(request, name):
draw.line([10+m100, TEXTSIZE * 3, 10+m100, TEXTSIZE*2], fill='#000000', width=LINEWIDTH)
label = "100m"
draw.text([10 + (m100 - draw.textsize(label)[0]) / 2, TEXTSIZE/2], label, fill='#000000')
plot("laser.0_7", "BNase", "Reference", "Bräuning Nase laser point", name, draw, img)
plot("226-96", "BZkn", "Reference", "Bräuning Zinken trig point", name, draw, img)
plot("vd1","VD1","Reference", "VD1 survey point", name, draw, img)
plot("laser.kt114_96","HSK","Reference", "Hinterer Schwarzmooskogel trig point", name, draw, img)
plot("2000","Nipple","Reference", "Nipple (Weiße Warze)", name, draw, img)
plot("3000","VSK","Reference", "Vorderer Schwarzmooskogel summit", name, draw, img)
plot("topcamp", "TC", "Reference", "Top Camp", name, draw, img)
plot("laser.0", "LSR0", "Reference", "Laser Point 0", name, draw, img)
plot("laser.0_1", "LSR1", "Reference", "Laser Point 0/1", name, draw, img)
plot("laser.0_3", "LSR3", "Reference", "Laser Point 0/3", name, draw, img)
plot("laser.0_5", "LSR5", "Reference", "Laser Point 0/5", name, draw, img)
plot("225-96", "BAlm", "Reference", "Bräuning Alm trig point", name, draw, img)
for entrance in Entrance.objects.all():
station = entrance.best_station()
if station:
#try:
areaName = entrance.caveandentrance_set.all()[0].cave.getArea().short_name
plot(station, "%s-%s" % (areaName, str(entrance)[5:]), entrance.needs_surface_work(), str(entrance), name, draw, img)
#except:
# pass
for p in MapLocations.points():
surveypoint, number, point_type, label = p
plot(surveypoint, number, point_type, label, name, draw, img)
for (N, E, D, num) in [(35975.37, 83018.21, 100,"177"), # Calculated from bearings
(35350.00, 81630.00, 50, "71"), # From Auer map
@ -504,19 +511,3 @@ def prospecting_image(request, name):
del draw
img.save(response, "PNG")
return response
# Not used.
# All imported using parsers.survex.LoadPos() now
# STATIONS = {}
# poslineregex = re.compile("^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
# def LoadPos():
# call([settings.CAVERN, "--output=%s/all.3d" % settings.SURVEX_DATA, "%s/all.svx" % settings.SURVEX_DATA])
# call([settings.THREEDTOPOS, '%sall.3d' % settings.SURVEX_DATA], cwd = settings.SURVEX_DATA)
# posfile = open("%sall.pos" % settings.SURVEX_DATA)
# posfile.readline()#Drop header
# for line in posfile.readlines():
# r = poslineregex.match(line)
# if r:
# x, y, z, name = r.groups()
# STATIONS[name] = (x, y, z)

15
core/views_survex.py
View File

@ -23,13 +23,13 @@ survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau
*export [connecting stations]
*title "area title"
*date 2999.99.99
*date 2099.99.99
*team Insts [Caver]
*team Insts [Caver]
*team Notes [Caver]
*instrument [set number]
;ref.: 2009#NN
*ref: 2099#NN
*calibrate tape +0.0 ; +ve if tape was too short, -ve if too long
@ -41,7 +41,6 @@ survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau
*end [surveyname]"""
def ReplaceTabs(stext):
res = [ ]
nsl = 0
@ -254,7 +253,6 @@ def identifycavedircontents(gcavedir):
return subdirs, subsvx
# direct local non-database browsing through the svx file repositories
# perhaps should use the database and have a reload button for it
def survexcaveslist(request):
@ -307,10 +305,6 @@ def survexcaveslist(request):
return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves })
# parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people
# doesn't use recursion. just writes it twice
def survexcavesingle(request, survex_cave):
@ -319,8 +313,3 @@ def survexcavesingle(request, survex_cave):
if breload:
parsers.survex.ReloadSurvexCave(survex_cave)
return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })

47
parsers/survex.py
View File

@ -1,18 +1,18 @@
import troggle.settings as settings
import troggle.core.models as models
import troggle.settings as settings
import sys
import os
import re
import time
from datetime import datetime, timedelta
from subprocess import call, Popen, PIPE
from troggle.parsers.people import GetPersonExpeditionNameLookup
from django.utils.timezone import get_current_timezone
from django.utils.timezone import make_aware
import re
import os
import time
from datetime import datetime, timedelta
import sys
import troggle.settings as settings
import troggle.core.models as models
from troggle.parsers.people import GetPersonExpeditionNameLookup
from troggle.core.views_caves import MapLocations
"""A 'survex block' is a *begin...*end set of cave data.
A 'survexscansfolder' is what we today call a "survey scans folder" or a "wallet".
@ -227,7 +227,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
# print(insp+'QM notes %s' % qm_notes)
# If the QM isn't resolved (has a resolving station) then load it
if not qm_resolve_section or qm_resolve_section != '-' or qm_resolve_section is not 'None':
if not qm_resolve_section or qm_resolve_section != '-' or qm_resolve_section != 'None':
from_section = models.SurvexBlock.objects.filter(name=qm_from_section)
# If we can find a section (survex note chunck, named)
if len(from_section) > 0:
@ -474,7 +474,6 @@ def LoadAllSurvexBlocks():
poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
def LoadPos():
"""Run cavern to produce a complete .3d file, then run 3dtopos to produce a table of
all survey point positions. Then lookup each position by name to see if we have it in the database
@ -486,6 +485,8 @@ def LoadPos():
topdata = settings.SURVEX_DATA + settings.SURVEX_TOPNAME
print((' - Generating a list of Pos from %s.svx and then loading...' % (topdata)))
# TO DO - remove the cache file apparatus. Not needed. Only laser points and entrances loaded now.
# Be careful with the cache file.
# If LoadPos has been run before,
# but without cave import being run before,
@ -532,27 +533,34 @@ def LoadPos():
# cavern defaults to using same cwd as supplied input file
call([settings.CAVERN, "--output=%s.3d" % (topdata), "%s.svx" % (topdata)])
call([settings.THREEDTOPOS, '%s.3d' % (topdata)], cwd = settings.SURVEX_DATA)
print(" - This next bit takes a while. Matching ~32,000 survey positions. Be patient...")
#print(" - This next bit takes a while. Matching ~32,000 survey positions. Be patient...")
mappoints = {}
for pt in MapLocations().points():
svxid, number, point_type, label = pt
mappoints[svxid]=True
posfile = open("%s.pos" % (topdata))
posfile.readline() #Drop header
for line in posfile.readlines():
r = poslineregex.match(line)
if r:
x, y, z, name = r.groups() # easting, northing, altitude
if name in notfoundbefore:
skip[name] = 1
x, y, z, id = r.groups() # easting, northing, altitude, survexstation
if id in notfoundbefore:
skip[id] = 1
else:
for sid in mappoints:
if id.endswith(sid):
try:
ss = models.SurvexStation.objects.lookup(name)
ss = models.SurvexStation.objects.lookup(id)
ss.x = float(x)
ss.y = float(y)
ss.z = float(z)
ss.save()
found += 1
except:
notfoundnow.append(name)
print(" - %s stations not found in lookup of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip)))
notfoundnow.append(id)
print(" - %s failed lookups of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip)))
if found > 10: # i.e. a previous cave import has been done
try:
@ -566,4 +574,3 @@ def LoadPos():
except:
print(" FAILURE WRITE opening cache file %s" % (cachefile))
raise