expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-25 08:41:51 +00:00
Code Issues 19 Projects 3 Activity

import fixes & statistics table

Browse Source
This commit is contained in:
Philip Sargent 2020-06-12 18:10:07 +01:00
parent b9a223c049
commit 269b8840ad
8 changed files with 182 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/views.py
View File

@ -5,4 +5,4 @@ from . import views_caves
from . import views_survex
from . import views_logbooks
from . import views_other
from . import views_statistics

2
core/views_logbooks.py
View File

@ -220,7 +220,7 @@ def pathsreport(request):
def experimental(request):
blockroots = models.SurvexBlock.objects.filter(name="root")
blockroots = models.SurvexBlock.objects.filter(name="rootblock")
if len(blockroots)>1:
print(" ! more than one root survexblock {}".format(len(blockroots)))
for sbr in blockroots:

23
core/views_other.py
View File

@ -20,29 +20,6 @@ print("** importing troggle/core/views_other.py")
def showrequest(request):
return HttpResponse(request.GET)
def stats(request):
statsDict={}
statsDict['expoCount'] = int(Expedition.objects.count())
statsDict['caveCount'] = int(Cave.objects.count())
statsDict['personCount'] = int(Person.objects.count())
statsDict['logbookEntryCount'] = int(LogbookEntry.objects.count())
legsbyexpo = [ ]
for expedition in Expedition.objects.all():
survexblocks = expedition.survexblock_set.all()
survexlegs = [ ]
survexleglength = 0.0
for survexblock in survexblocks:
survexlegs.extend(survexblock.survexleg_set.all())
survexleglength += survexblock.totalleglength
legsbyexpo.append((expedition, {"nsurvexlegs":len(survexlegs), "survexleglength":survexleglength/1000}))
legsbyexpo.reverse()
survexlegs = SurvexLeg.objects.all()
totalsurvexlength = sum([survexleg.tape for survexleg in survexlegs])
renderDict = {**statsDict, **{ "nsurvexlegs":len(survexlegs), "totalsurvexlength":totalsurvexlength/1000, "legsbyexpo":legsbyexpo }} # new syntax
return render(request,'statistics.html', renderDict)
def frontpage(request):
if request.user.is_authenticated():
return render(request,'tasks.html')

112
core/views_statistics.py Normal file
View File

@ -0,0 +1,112 @@
import datetime
import os.path
import re
import django.db.models
from django.db.models import Min, Max
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render, render_to_response
from django.template import Context, loader
from django.template.defaultfilters import slugify
from django.utils import timezone
from django.views.generic.list import ListView
from troggle.core.models import Expedition, Person, PersonExpedition
from troggle.core.models_caves import Cave, LogbookEntry
from troggle.core.models_survex import SurvexLeg, SurvexBlock
import troggle.settings as settings
from settings import *
def pathsreport(request):
pathsdict={
"ADMIN_MEDIA_PREFIX" : ADMIN_MEDIA_PREFIX,
"ADMIN_MEDIA_PREFIX" : ADMIN_MEDIA_PREFIX,
"CAVEDESCRIPTIONSX" : CAVEDESCRIPTIONS,
"DIR_ROOT" : DIR_ROOT,
"ENTRANCEDESCRIPTIONS" : ENTRANCEDESCRIPTIONS,
"EXPOUSER_EMAIL" : EXPOUSER_EMAIL,
"EXPOUSERPASS" :"<redacted>",
"EXPOUSER" : EXPOUSER,
"EXPOWEB" : EXPOWEB,
"EXPOWEB_URL" : EXPOWEB_URL,
"FILES" : FILES,
"JSLIB_URL" : JSLIB_URL,
"LOGFILE" : LOGFILE,
"LOGIN_REDIRECT_URL" : LOGIN_REDIRECT_URL,
"MEDIA_ADMIN_DIR" : MEDIA_ADMIN_DIR,
"MEDIA_ROOT" : MEDIA_ROOT,
"MEDIA_URL" : MEDIA_URL,
#"PHOTOS_ROOT" : PHOTOS_ROOT,
"PHOTOS_URL" : PHOTOS_URL,
"PYTHON_PATH" : PYTHON_PATH,
"REPOS_ROOT_PATH" : REPOS_ROOT_PATH,
"ROOT_URLCONF" : ROOT_URLCONF,
"STATIC_ROOT" : STATIC_ROOT,
"STATIC_URL" : STATIC_URL,
"SURVEX_DATA" : SURVEX_DATA,
"SURVEY_SCANS" : SURVEY_SCANS,
"SURVEYS" : SURVEYS,
"SURVEYS_URL" : SURVEYS_URL,
"SVX_URL" : SVX_URL,
"TEMPLATE_DIRS" : TEMPLATE_DIRS,
"THREEDCACHEDIR" : THREEDCACHEDIR,
"TINY_MCE_MEDIA_ROOT" : TINY_MCE_MEDIA_ROOT,
"TINY_MCE_MEDIA_URL" : TINY_MCE_MEDIA_URL,
"TUNNEL_DATA" : TUNNEL_DATA,
"URL_ROOT" : URL_ROOT
}
ncodes = len(pathsdict)
bycodeslist = sorted(pathsdict.items())
bypathslist = sorted(iter(pathsdict.items()), key=lambda x: x[1])
return render(request, 'pathsreport.html', {
"pathsdict":pathsdict,
"bycodeslist":bycodeslist,
"bypathslist":bypathslist,
"ncodes":ncodes})
def stats(request):
statsDict={}
statsDict['expoCount'] = "{:,}".format(Expedition.objects.count())
statsDict['caveCount'] = "{:,}".format(Cave.objects.count())
statsDict['personCount'] = "{:,}".format(Person.objects.count())
statsDict['logbookEntryCount'] = "{:,}".format(LogbookEntry.objects.count())
blockroots = SurvexBlock.objects.filter(name="rootblock")
if len(blockroots)>1:
print(" ! more than one root survexblock {}".format(len(blockroots)))
for sbr in blockroots:
print("{} {} {} {}".format(sbr.id, sbr.name, sbr.text, sbr.date))
sbr = blockroots[0]
totalsurvexlength = sbr.totalleglength
try:
nimportlegs = int(sbr.text)
except:
print("{} {} {} {}".format(sbr.id, sbr.name, sbr.text, sbr.date))
nimportlegs = -1
legsbyexpo = [ ]
addupsurvexlength = 0
for expedition in Expedition.objects.all():
survexblocks = expedition.survexblock_set.all()
legsyear=0
survexleglength = 0.0
for survexblock in survexblocks:
survexleglength += survexblock.totalleglength
try:
legsyear += int(survexblock.text)
except:
pass
addupsurvexlength += survexleglength
legsbyexpo.append((expedition, {"nsurvexlegs": "{:,}".format(legsyear),
"survexleglength":"{:,.0f}".format(survexleglength)}))
legsbyexpo.reverse()
survexlegs = SurvexLeg.objects.all()
renderDict = {**statsDict, **{ "nsurvexlegs": "{:,}".format(nimportlegs), "totalsurvexlength":totalsurvexlength/1000, "addupsurvexlength":addupsurvexlength/1000, "legsbyexpo":legsbyexpo }} # new syntax
return render(request,'statistics.html', renderDict)

53
databaseReset.py
View File

@ -18,7 +18,6 @@ print(" 1 settings on loading databaseReset.py")
from troggle.core.models_caves import Cave, Entrance
import troggle.parsers.caves
#import troggle.settings
import troggle.flatpages.models
import troggle.logbooksdump
import troggle.parsers.people
@ -54,6 +53,8 @@ def reinit_db():
in memory (django python models, not the database), so there is already a full load
of stuff known. Deleting the db file does not clear memory.
"""
print("Reinitialising db ",end="")
print(django.db.connections.databases['default']['NAME'])
currentdbname = settings.DATABASES['default']['NAME']
if currentdbname == ':memory:':
# closing connections should wipe the in-memory database
@ -62,11 +63,15 @@ def reinit_db():
print(" ! Closing another connection to db...")
conn.close()
elif django.db.connections.databases['default']['ENGINE'] == 'django.db.backends.sqlite3':
if os.path.isfile(currentdbname):
try:
print(" - deleting " + currentdbname)
os.remove(currentdbname)
except OSError:
print(" ! OSError on removing: " + currentdbname + " (Is the file open in another app?\n")
raise
else:
print(" - No database file found: " + currentdbname + " ..continuing, will create it.\n")
else:
cursor = django.db.connection.cursor()
cursor.execute("DROP DATABASE %s" % currentdbname)
@ -75,14 +80,13 @@ def reinit_db():
cursor.execute("USE %s" % currentdbname)
#Sync user - needed after reload
print(" - Migrating: " + settings.DATABASES['default']['NAME'])
print(django.db.connections.databases['default']['NAME'])
print(" - Migrating: " + django.db.connections.databases['default']['NAME'])
management.call_command('migrate', interactive=False)
print(" - done migration on: " + settings.DATABASES['default']['NAME'])
print("users in db already: ",len(User.objects.all()))
try:
print(" - Setting up admin user on: " + settings.DATABASES['default']['NAME'])
print(django.db.connections.databases['default']['NAME'])
print(" - Setting up admin user on: " + django.db.connections.databases['default']['NAME'])
print(" - user: {} ({:.5}...) <{}> ".format(expouser, expouserpass, expouseremail))
user = User.objects.create_user(expouser, expouseremail, expouserpass)
user.is_staff = True
@ -93,6 +97,8 @@ def reinit_db():
print(django.db.connections.databases['default']['NAME'])
print(" ! You probably have not got a clean db when you thought you had.\n")
print(" ! Also you are probably NOT running an in-memory db now.\n")
print("users in db: ",len(User.objects.all()))
print("tables in db: ",len(connection.introspection.table_names()))
memdumpsql(fn='integrityfail.sql')
django.db.connections.databases['default']['NAME'] = ':memory:'
#raise
@ -123,6 +129,9 @@ def restore_dbsettings():
def set_in_memory_dbsettings():
django.db.close_old_connections() # needed if MySQL running?
for conn in django.db.connections.all():
print(" ! Closing another connection to db in set_in_memory_dbsettings")
conn.close()
settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3',
'AUTOCOMMIT': True,
'ATOMIC_REQUESTS': False,
@ -251,7 +260,6 @@ class JobQueue():
json.dump(self.results, f)
return True
def runqonce(self):
"""Run all the jobs in the queue provided - once
"""
@ -309,9 +317,7 @@ class JobQueue():
self.loadprofiles()
store_dbsettings()
print("-- start ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
print(django.db.connections.databases['default']['NAME'])
print("-- start ", django.db.connections.databases['default']['ENGINE'], django.db.connections.databases['default']['NAME'])
if dbname ==":memory:":
# just run, and save the sql file
@ -426,7 +432,7 @@ def usage():
QMs - read in the QM csv files (older caves only)
scans - the survey scans in all the wallets (must run before survex)
survex - read in the survex files - all the survex blocks but not the x/y/z positions
survexpos - just the x/y/z Pos out of the survex files (not needed)
survexpos - just the x/y/z Pos out of the survex files (not needed) -- Never used.
tunnel - read in the Tunnel files - which scans the survey scans too
@ -459,15 +465,7 @@ if __name__ == "__main__":
else:
runlabel=None
store_dbsettings()
set_in_memory_dbsettings()
print(" - django.setup - next")
try:
django.setup()
except:
print(" ! COMPLICATED FAILURE. Does not occur with a valid 'troggle.sqlite' database in place.")
raise
print(" - django.setup - done")
jq = JobQueue(runlabel)
@ -493,9 +491,9 @@ if __name__ == "__main__":
jq.enq("scans",import_surveyscans)
jq.enq("logbooks",import_logbooks)
jq.enq("QMs",import_QMs)
jq.enq("survexblks",import_survexblks)
jq.enq("survexpos",import_survexpos)
jq.enq("tunnel",import_tunnelfiles)
jq.enq("survexblks",import_survexblks)
#jq.enq("survexpos",import_survexpos)
elif "scans" in sys.argv:
jq.enq("scans",import_surveyscans)
elif "survex" in sys.argv:
@ -524,7 +522,18 @@ if __name__ == "__main__":
print("%s not recognised as a command." % sys.argv[1])
exit()
#jq.run_now_django_tests(1)
store_dbsettings()
#set_in_memory_dbsettings()
print(" - django.setup - next")
try:
django.setup()
except:
print(" ! Cyclic reference failure. Can occur when the initial db is empty. Fixed now (in UploadFileForm) but easy to reintroduce..")
raise
print(" - django.setup - done")
#set_in_memory_dbsettings() # seems to be ignored. Appears to be set but in reality.
#jq.run_now_django_tests(1) # actually does set db to :memory: - but invisibly !
jq.run()
jq.showprofile()

23
parsers/caves.py
View File

@ -30,6 +30,7 @@ def readentrance(filename):
with open(os.path.join(settings.ENTRANCEDESCRIPTIONS, filename)) as f:
contents = f.read()
context = "in file %s" % filename
#print("Reading file ENTRANCE {} / {}".format(settings.ENTRANCEDESCRIPTIONS, filename))
entrancecontentslist = getXML(contents, "entrance", maxItems = 1, context = context)
if len(entrancecontentslist) == 1:
entrancecontents = entrancecontentslist[0]
@ -83,10 +84,26 @@ def readentrance(filename):
cached_primary_slug = slugs[0])
primary = True
for slug in slugs:
#print slug, filename
#print("entrance slug:{} filename:{}".format(slug, filename))
try:
cs = models_caves.EntranceSlug.objects.update_or_create(entrance = e,
slug = slug,
primary = primary)
except:
# need to cope with duplicates
print(" ! FAILED to get only one ENTRANCE when updating using: "+filename)
kents = models_caves.EntranceSlug.objects.all().filter(entrance = e,
slug = slug,
primary = primary)
for k in kents:
message = " ! - DUPLICATE in db. entrance:"+ str(k.entrance) + ", slug:" + str(k.slug())
models.DataIssue.objects.create(parser='caves', message=message)
print(message)
for k in kaves:
if k.slug() != None:
print(" ! - OVERWRITING this one: slug:"+ str(k.slug()))
k.notes = "DUPLICATE entrance found on import. Please fix\n" + k.notes
c = k
primary = False
def readcave(filename):
@ -94,7 +111,7 @@ def readcave(filename):
with open(os.path.join(settings.CAVEDESCRIPTIONS, filename)) as f:
contents = f.read()
context = " in file %s" % filename
#print "Reading file %s" % filename
#print("Reading file CAVE {}".format(filename))
cavecontentslist = getXML(contents, "cave", maxItems = 1, context = context)
#print cavecontentslist
if len(cavecontentslist) == 1:
@ -145,7 +162,7 @@ def readcave(filename):
filename = filename)
except:
# need to cope with duplicates
print(" ! FAILED to get only one cave when updating using: "+filename)
print(" ! FAILED to get only one CAVE when updating using: "+filename)
kaves = models_caves.Cave.objects.all().filter(kataster_number=kataster_number[0])
for k in kaves:
message = " ! - DUPLICATES in db. kataster:"+ str(k.kataster_number) + ", slug:" + str(k.slug())

5
parsers/survex.py
View File

@ -116,6 +116,7 @@ def LoadSurvexEquate(survexblock, sline):
def LoadSurvexLinePassage(survexblock, stardata, sline, comment):
# do not import *data passage.. data which is LRUD not tape/compass/clino
pass
stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
@ -371,7 +372,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
#survexblock.text = "".join(textlines)
# .text not used, using it for number of legs per block
legsinblock = survexlegsnumber - previousnlegs
print("LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber))
print(insp+"LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber))
survexblock.text = str(legsinblock)
survexblock.save()
# print(insp+' - End found: ')
@ -483,7 +484,7 @@ def LoadAllSurvexBlocks():
#Load all
# this is the first so id=1
survexblockroot = models.SurvexBlock(name="root", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0)
survexblockroot = models_survex.SurvexBlock(name="rootblock", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0)
survexblockroot.save()
fin = survexfile.OpenFile()
textlines = [ ]

9
templates/statistics.html
View File

@ -10,18 +10,19 @@
<p>{{ expoCount }} expeditions: {{ personCount }} people have contributed {{ caveCount }} caves and {{ logbookEntryCount }} logbook entries.
<p>Number of survey legs: {{nsurvexlegs}}<br />
Total length: {{totalsurvexlength|stringformat:".3f"}} m on importing survex files.<br />
Total length: {{addupsurvexlength|stringformat:".3f"}} m adding up all the years below.</p>
Total length: {{totalsurvexlength|stringformat:".1f"}} km added-up on importing survex files.<br />
Total length: {{addupsurvexlength|stringformat:".1f"}} km adding up all the years below.</p>
<p>These are raw tape lengths which include pitches, splays and surface-surveys.
<table>
<tr><th>Year</th><th>Surveys</th><th>Survey Legs</th><th>Total length<br>(km)</th></tr>
<tr><th>Year</th><th>Survex<br>Surveys</th><th>Survey Legs</th><th>Total length<br>(m)</th></tr>
{% for legs in legsbyexpo %}
<tr>
<td>{{legs.0.year}}</td>
<td style="text-align:right">{{legs.0.survexblock_set.all|length}}</td>
<td style="text-align:right">{{legs.1.nsurvexlegs|rjust:"10"}}</td>
<td style="text-align:right">{{legs.1.survexleglength|stringformat:".3f"}}</td>
<td style="text-align:right">{{legs.1.survexleglength}}</td>
</tr>
{% endfor %}
</table>