expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-22 15:21:52 +00:00
Code Issues 19 Projects 3 Activity

Refactor to avoid loading unused surveystations

Browse Source
This commit is contained in:
Philip Sargent 2020-05-28 01:16:45 +01:00
parent 73637ba53d
commit 6cc578435c
6 changed files with 103 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.gitignore vendored
View File

@ -3,6 +3,8 @@ syntax: glob
*.pyc *.pyc
db* db*
*.sqlite
*.sql
localsettings.py localsettings.py
*~ *~
parsing_log.txt parsing_log.txt
@ -13,22 +15,15 @@ troggle_log.txt
media/images/* media/images/*
.vscode/* .vscode/*
.swp .swp
imagekit-off/
localsettings-expo-live.py localsettings-expo-live.py
.gitignore .gitignore
desktop.ini desktop.ini
troggle-reset.log
troggle-reset0.log
troggle-surveys.log
troggle.log troggle.log
troggle.sqlite troggle.sqlite
troggle.sqlite.0
troggle.sqlite.1
my_project.dot my_project.dot
memdump.sql memdump.sql
troggle-sqlite.sql troggle-sqlite.sql
import_profile.json import_profile.json
import_times.json
ignored-files.log ignored-files.log
tunnel-import.log tunnel-import.log
posnotfound posnotfound

5
core/models.py
View File

@ -112,7 +112,6 @@ class Expedition(TroggleModel):
res = self.expeditionday_set.all() res = self.expeditionday_set.all()
return res and res[len(res) - 1] or None return res and res[len(res) - 1] or None
class ExpeditionDay(TroggleModel): class ExpeditionDay(TroggleModel):
expedition = models.ForeignKey("Expedition") expedition = models.ForeignKey("Expedition")
date = models.DateField() date = models.DateField()
@ -183,7 +182,6 @@ class Person(TroggleModel):
#self.orderref = self.first_name #self.orderref = self.first_name
#self.notability = 0.0 # set temporarily #self.notability = 0.0 # set temporarily
class PersonExpedition(TroggleModel): class PersonExpedition(TroggleModel):
"""Person's attendance to one Expo """Person's attendance to one Expo
""" """
@ -320,7 +318,6 @@ class LogbookEntry(TroggleModel):
def DayIndex(self): def DayIndex(self):
return list(self.expeditionday.logbookentry_set.all()).index(self) return list(self.expeditionday.logbookentry_set.all()).index(self)
# #
# Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry) # Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry)
# #
@ -359,7 +356,6 @@ class PersonTrip(TroggleModel):
return "%s (%s)" % (self.personexpedition, self.logbook_entry.date) return "%s (%s)" % (self.personexpedition, self.logbook_entry.date)
########################################## ##########################################
# move following classes into models_cave # move following classes into models_cave
########################################## ##########################################
@ -392,7 +388,6 @@ class CaveSlug(models.Model):
slug = models.SlugField(max_length=50, unique = True) slug = models.SlugField(max_length=50, unique = True)
primary = models.BooleanField(default=False) primary = models.BooleanField(default=False)
class Cave(TroggleModel): class Cave(TroggleModel):
# too much here perhaps, # too much here perhaps,
official_name = models.CharField(max_length=160) official_name = models.CharField(max_length=160)

12
core/models_survex.py
View File

@ -1,8 +1,9 @@
from django.db import models
from django.conf import settings
import os import os
import urllib.parse import urllib.parse
import re import re
from django.db import models
from django.conf import settings
from django.core.urlresolvers import reverse from django.core.urlresolvers import reverse
@ -22,6 +23,7 @@ class SurvexDirectory(models.Model):
class Meta: class Meta:
ordering = ('id',) ordering = ('id',)
class SurvexFile(models.Model): class SurvexFile(models.Model):
path = models.CharField(max_length=200) path = models.CharField(max_length=200)
survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True) survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True)
@ -49,6 +51,7 @@ class SurvexFile(models.Model):
self.survexdirectory = survexdirectory self.survexdirectory = survexdirectory
self.save() self.save()
class SurvexEquate(models.Model): class SurvexEquate(models.Model):
cave = models.ForeignKey('Cave', blank=True, null=True) cave = models.ForeignKey('Cave', blank=True, null=True)
@ -87,7 +90,6 @@ class SurvexLeg(models.Model):
compass = models.FloatField() compass = models.FloatField()
clino = models.FloatField() clino = models.FloatField()
# #
# Single SurvexBlock # Single SurvexBlock
# #
@ -153,7 +155,6 @@ class SurvexBlock(models.Model):
def DayIndex(self): def DayIndex(self):
return list(self.expeditionday.survexblock_set.all()).index(self) return list(self.expeditionday.survexblock_set.all()).index(self)
class SurvexTitle(models.Model): class SurvexTitle(models.Model):
survexblock = models.ForeignKey('SurvexBlock') survexblock = models.ForeignKey('SurvexBlock')
title = models.CharField(max_length=200) title = models.CharField(max_length=200)
@ -187,7 +188,6 @@ class SurvexPersonRole(models.Model):
def __str__(self): def __str__(self):
return str(self.person) + " - " + str(self.survexblock) + " - " + str(self.nrole) return str(self.person) + " - " + str(self.survexblock) + " - " + str(self.nrole)
class SurvexScansFolder(models.Model): class SurvexScansFolder(models.Model):
fpath = models.CharField(max_length=200) fpath = models.CharField(max_length=200)
walletname = models.CharField(max_length=200) walletname = models.CharField(max_length=200)
@ -215,7 +215,6 @@ class SurvexScanSingle(models.Model):
def __str__(self): def __str__(self):
return "Survey Scan Image: " + str(self.name) + " in " + str(self.survexscansfolder) return "Survey Scan Image: " + str(self.name) + " in " + str(self.survexscansfolder)
class TunnelFile(models.Model): class TunnelFile(models.Model):
tunnelpath = models.CharField(max_length=200) tunnelpath = models.CharField(max_length=200)
tunnelname = models.CharField(max_length=200) tunnelname = models.CharField(max_length=200)
@ -231,4 +230,3 @@ class TunnelFile(models.Model):
class Meta: class Meta:
ordering = ('tunnelpath',) ordering = ('tunnelpath',)

103
core/views_caves.py
View File

@ -3,24 +3,52 @@
from __future__ import (absolute_import, division, from __future__ import (absolute_import, division,
print_function, unicode_literals) print_function, unicode_literals)
from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation import sys
from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm import os
import troggle.core.models as models import string
import troggle.settings as settings import subprocess
from troggle.helper import login_required_if_public import re
from django.forms.models import modelformset_factory
from django import forms
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.conf import settings
import re, urllib.parse
from django.shortcuts import get_object_or_404, render
import settings import settings
import urllib.parse
from PIL import Image, ImageDraw, ImageFont from PIL import Image, ImageDraw, ImageFont
import string, os, sys, subprocess from django import forms
from django.conf import settings
from django.forms.models import modelformset_factory
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
import troggle.settings as settings
import troggle.core.models as models
from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation
from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
from troggle.helper import login_required_if_public
class MapLocations(object):
p = [
("laser.0_7", "BNase", "Reference", "Bräuning Nase laser point"),
("226-96", "BZkn", "Reference", "Bräuning Zinken trig point"),
("vd1","VD1","Reference", "VD1 survey point"),
("laser.kt114_96","HSK","Reference", "Hinterer Schwarzmooskogel trig point"),
("2000","Nipple","Reference", "Nipple (Weiße Warze)"),
("3000","VSK","Reference", "Vorderer Schwarzmooskogel summit"),
("oldtopcamp", "OTC", "Reference", "Old Top Camp"),
("laser.0", "LSR0", "Reference", "Laser Point 0"),
("laser.0_1", "LSR1", "Reference", "Laser Point 0/1"),
("laser.0_3", "LSR3", "Reference", "Laser Point 0/3"),
("laser.0_5", "LSR5", "Reference", "Laser Point 0/5"),
("225-96", "BAlm", "Reference", "Bräuning Alm trig point")
]
def points(self):
for ent in Entrance.objects.all():
if ent.best_station():
areaName = ent.caveandentrance_set.all()[0].cave.getArea().short_name
self.p.append((ent.best_station(), "%s-%s" % (areaName, str(ent)[5:]), ent.needs_surface_work(), str(ent)))
return self.p
def __str__(self):
return "{} map locations".format(len(self.p))
def getCave(cave_id): def getCave(cave_id):
"""Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm.""" """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm."""
@ -58,8 +86,6 @@ def millenialcaves(request):
#RW messing around area #RW messing around area
return HttpResponse("Test text", content_type="text/plain") return HttpResponse("Test text", content_type="text/plain")
def cave3d(request, cave_id=''): def cave3d(request, cave_id=''):
cave = getCave(cave_id) cave = getCave(cave_id)
survexfilename = settings.SURVEX_DATA + cave.survex_file survexfilename = settings.SURVEX_DATA + cave.survex_file
@ -217,8 +243,6 @@ def qm(request,cave_id,qm_id,year,grade=None):
url += r'&grade=' + grade url += r'&grade=' + grade
return HttpResponseRedirect(url) return HttpResponseRedirect(url)
def ent(request, cave_id, ent_letter): def ent(request, cave_id, ent_letter):
cave = Cave.objects.filter(kataster_number = cave_id)[0] cave = Cave.objects.filter(kataster_number = cave_id)[0]
cave_and_ent = CaveAndEntrance.objects.filter(cave = cave).filter(entrance_letter = ent_letter)[0] cave_and_ent = CaveAndEntrance.objects.filter(cave = cave).filter(entrance_letter = ent_letter)[0]
@ -362,7 +386,6 @@ areacolours = {
'7' : '#808080' '7' : '#808080'
} }
for FONT in [ for FONT in [
"/usr/share/fonts/truetype/freefont/FreeSans.ttf", "/usr/share/fonts/truetype/freefont/FreeSans.ttf",
"/usr/X11R6/lib/X11/fonts/truetype/arial.ttf", "/usr/X11R6/lib/X11/fonts/truetype/arial.ttf",
@ -460,26 +483,10 @@ def prospecting_image(request, name):
draw.line([10+m100, TEXTSIZE * 3, 10+m100, TEXTSIZE*2], fill='#000000', width=LINEWIDTH) draw.line([10+m100, TEXTSIZE * 3, 10+m100, TEXTSIZE*2], fill='#000000', width=LINEWIDTH)
label = "100m" label = "100m"
draw.text([10 + (m100 - draw.textsize(label)[0]) / 2, TEXTSIZE/2], label, fill='#000000') draw.text([10 + (m100 - draw.textsize(label)[0]) / 2, TEXTSIZE/2], label, fill='#000000')
plot("laser.0_7", "BNase", "Reference", "Bräuning Nase laser point", name, draw, img)
plot("226-96", "BZkn", "Reference", "Bräuning Zinken trig point", name, draw, img) for p in MapLocations.points():
plot("vd1","VD1","Reference", "VD1 survey point", name, draw, img) surveypoint, number, point_type, label = p
plot("laser.kt114_96","HSK","Reference", "Hinterer Schwarzmooskogel trig point", name, draw, img) plot(surveypoint, number, point_type, label, name, draw, img)
plot("2000","Nipple","Reference", "Nipple (Weiße Warze)", name, draw, img)
plot("3000","VSK","Reference", "Vorderer Schwarzmooskogel summit", name, draw, img)
plot("topcamp", "TC", "Reference", "Top Camp", name, draw, img)
plot("laser.0", "LSR0", "Reference", "Laser Point 0", name, draw, img)
plot("laser.0_1", "LSR1", "Reference", "Laser Point 0/1", name, draw, img)
plot("laser.0_3", "LSR3", "Reference", "Laser Point 0/3", name, draw, img)
plot("laser.0_5", "LSR5", "Reference", "Laser Point 0/5", name, draw, img)
plot("225-96", "BAlm", "Reference", "Bräuning Alm trig point", name, draw, img)
for entrance in Entrance.objects.all():
station = entrance.best_station()
if station:
#try:
areaName = entrance.caveandentrance_set.all()[0].cave.getArea().short_name
plot(station, "%s-%s" % (areaName, str(entrance)[5:]), entrance.needs_surface_work(), str(entrance), name, draw, img)
#except:
# pass
for (N, E, D, num) in [(35975.37, 83018.21, 100,"177"), # Calculated from bearings for (N, E, D, num) in [(35975.37, 83018.21, 100,"177"), # Calculated from bearings
(35350.00, 81630.00, 50, "71"), # From Auer map (35350.00, 81630.00, 50, "71"), # From Auer map
@ -504,19 +511,3 @@ def prospecting_image(request, name):
del draw del draw
img.save(response, "PNG") img.save(response, "PNG")
return response return response
# Not used.
# All imported using parsers.survex.LoadPos() now
# STATIONS = {}
# poslineregex = re.compile("^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
# def LoadPos():
# call([settings.CAVERN, "--output=%s/all.3d" % settings.SURVEX_DATA, "%s/all.svx" % settings.SURVEX_DATA])
# call([settings.THREEDTOPOS, '%sall.3d' % settings.SURVEX_DATA], cwd = settings.SURVEX_DATA)
# posfile = open("%sall.pos" % settings.SURVEX_DATA)
# posfile.readline()#Drop header
# for line in posfile.readlines():
# r = poslineregex.match(line)
# if r:
# x, y, z, name = r.groups()
# STATIONS[name] = (x, y, z)

15
core/views_survex.py
View File

@ -23,13 +23,13 @@ survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau
*export [connecting stations] *export [connecting stations]
*title "area title" *title "area title"
*date 2999.99.99 *date 2099.99.99
*team Insts [Caver] *team Insts [Caver]
*team Insts [Caver] *team Insts [Caver]
*team Notes [Caver] *team Notes [Caver]
*instrument [set number] *instrument [set number]
;ref.: 2009#NN *ref: 2099#NN
*calibrate tape +0.0 ; +ve if tape was too short, -ve if too long *calibrate tape +0.0 ; +ve if tape was too short, -ve if too long
@ -41,7 +41,6 @@ survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau
*end [surveyname]""" *end [surveyname]"""
def ReplaceTabs(stext): def ReplaceTabs(stext):
res = [ ] res = [ ]
nsl = 0 nsl = 0
@ -254,7 +253,6 @@ def identifycavedircontents(gcavedir):
return subdirs, subsvx return subdirs, subsvx
# direct local non-database browsing through the svx file repositories # direct local non-database browsing through the svx file repositories
# perhaps should use the database and have a reload button for it # perhaps should use the database and have a reload button for it
def survexcaveslist(request): def survexcaveslist(request):
@ -307,10 +305,6 @@ def survexcaveslist(request):
return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves }) return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves })
# parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people # parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people
# doesn't use recursion. just writes it twice # doesn't use recursion. just writes it twice
def survexcavesingle(request, survex_cave): def survexcavesingle(request, survex_cave):
@ -319,8 +313,3 @@ def survexcavesingle(request, survex_cave):
if breload: if breload:
parsers.survex.ReloadSurvexCave(survex_cave) parsers.survex.ReloadSurvexCave(survex_cave)
return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave }) return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })

47
parsers/survex.py
View File

@ -1,18 +1,18 @@
import troggle.settings as settings import sys
import troggle.core.models as models import os
import troggle.settings as settings import re
import time
from datetime import datetime, timedelta
from subprocess import call, Popen, PIPE from subprocess import call, Popen, PIPE
from troggle.parsers.people import GetPersonExpeditionNameLookup
from django.utils.timezone import get_current_timezone from django.utils.timezone import get_current_timezone
from django.utils.timezone import make_aware from django.utils.timezone import make_aware
import re import troggle.settings as settings
import os import troggle.core.models as models
import time from troggle.parsers.people import GetPersonExpeditionNameLookup
from datetime import datetime, timedelta from troggle.core.views_caves import MapLocations
import sys
"""A 'survex block' is a *begin...*end set of cave data. """A 'survex block' is a *begin...*end set of cave data.
A 'survexscansfolder' is what we today call a "survey scans folder" or a "wallet". A 'survexscansfolder' is what we today call a "survey scans folder" or a "wallet".
@ -227,7 +227,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
# print(insp+'QM notes %s' % qm_notes) # print(insp+'QM notes %s' % qm_notes)
# If the QM isn't resolved (has a resolving station) then load it # If the QM isn't resolved (has a resolving station) then load it
if not qm_resolve_section or qm_resolve_section != '-' or qm_resolve_section is not 'None': if not qm_resolve_section or qm_resolve_section != '-' or qm_resolve_section != 'None':
from_section = models.SurvexBlock.objects.filter(name=qm_from_section) from_section = models.SurvexBlock.objects.filter(name=qm_from_section)
# If we can find a section (survex note chunck, named) # If we can find a section (survex note chunck, named)
if len(from_section) > 0: if len(from_section) > 0:
@ -474,7 +474,6 @@ def LoadAllSurvexBlocks():
poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$") poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
def LoadPos(): def LoadPos():
"""Run cavern to produce a complete .3d file, then run 3dtopos to produce a table of """Run cavern to produce a complete .3d file, then run 3dtopos to produce a table of
all survey point positions. Then lookup each position by name to see if we have it in the database all survey point positions. Then lookup each position by name to see if we have it in the database
@ -486,6 +485,8 @@ def LoadPos():
topdata = settings.SURVEX_DATA + settings.SURVEX_TOPNAME topdata = settings.SURVEX_DATA + settings.SURVEX_TOPNAME
print((' - Generating a list of Pos from %s.svx and then loading...' % (topdata))) print((' - Generating a list of Pos from %s.svx and then loading...' % (topdata)))
# TO DO - remove the cache file apparatus. Not needed. Only laser points and entrances loaded now.
# Be careful with the cache file. # Be careful with the cache file.
# If LoadPos has been run before, # If LoadPos has been run before,
# but without cave import being run before, # but without cave import being run before,
@ -532,27 +533,34 @@ def LoadPos():
# cavern defaults to using same cwd as supplied input file # cavern defaults to using same cwd as supplied input file
call([settings.CAVERN, "--output=%s.3d" % (topdata), "%s.svx" % (topdata)]) call([settings.CAVERN, "--output=%s.3d" % (topdata), "%s.svx" % (topdata)])
call([settings.THREEDTOPOS, '%s.3d' % (topdata)], cwd = settings.SURVEX_DATA) call([settings.THREEDTOPOS, '%s.3d' % (topdata)], cwd = settings.SURVEX_DATA)
print(" - This next bit takes a while. Matching ~32,000 survey positions. Be patient...") #print(" - This next bit takes a while. Matching ~32,000 survey positions. Be patient...")
mappoints = {}
for pt in MapLocations().points():
svxid, number, point_type, label = pt
mappoints[svxid]=True
posfile = open("%s.pos" % (topdata)) posfile = open("%s.pos" % (topdata))
posfile.readline() #Drop header posfile.readline() #Drop header
for line in posfile.readlines(): for line in posfile.readlines():
r = poslineregex.match(line) r = poslineregex.match(line)
if r: if r:
x, y, z, name = r.groups() # easting, northing, altitude x, y, z, id = r.groups() # easting, northing, altitude, survexstation
if name in notfoundbefore: if id in notfoundbefore:
skip[name] = 1 skip[id] = 1
else: else:
for sid in mappoints:
if id.endswith(sid):
try: try:
ss = models.SurvexStation.objects.lookup(name) ss = models.SurvexStation.objects.lookup(id)
ss.x = float(x) ss.x = float(x)
ss.y = float(y) ss.y = float(y)
ss.z = float(z) ss.z = float(z)
ss.save() ss.save()
found += 1 found += 1
except: except:
notfoundnow.append(name) notfoundnow.append(id)
print(" - %s stations not found in lookup of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip))) print(" - %s failed lookups of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip)))
if found > 10: # i.e. a previous cave import has been done if found > 10: # i.e. a previous cave import has been done
try: try:
@ -566,4 +574,3 @@ def LoadPos():
except: except:
print(" FAILURE WRITE opening cache file %s" % (cachefile)) print(" FAILURE WRITE opening cache file %s" % (cachefile))
raise raise