xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

tidying and prep for python3

Browse Source
This commit is contained in:
Philip Sargent 2020-05-24 13:35:47 +01:00
parent 49d5857b36
commit b69bdcd126
4 changed files with 75 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.txt
View File

@ -44,12 +44,12 @@ pip install pygraphviz # fails to install
pip install pyparsing pydot # installs fine
django extension graph_models # https://django-extensions.readthedocs.io/en/latest/graph_models.html
Or use a python3 virtual environment:
Or use a python3 virtual environment: (python3.5 not later)
$ cd troggle
$ cd ..
$ python3 -m venv pyth3d2
$ python3.5 -m venv pyth35d2
(creates folder with virtual env)
cd pyth3d2
cd pyth35d2
bin/activate
(now install everything - not working yet..)
$ pip install -r requirements.txt

92
databaseReset.py
View File

@ -1,20 +1,26 @@
from __future__ import (absolute_import, division,
print_function)
import os
import time
import timeit
import json
import settings
os.environ['PYTHONPATH'] = settings.PYTHON_PATH
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
from django.core import management
from django.db import connection, close_old_connections
from django.contrib.auth.models import User
from django.http import HttpResponse
from django.core.urlresolvers import reverse
from troggle.core.models import Cave, Entrance
import troggle.settings
import troggle.flatpages.models
import json
import troggle.logbooksdump
# NOTE databaseRest.py is *imported* by views_other.py as it is used in the control panel
# NOTE databaseReset.py is *imported* by views_other.py as it is used in the control panel
# presented there.
expouser=settings.EXPOUSER
@ -60,52 +66,52 @@ def dirsredirect():
f.save()
def import_caves():
import parsers.caves
import troggle.parsers.caves
print("Importing Caves")
parsers.caves.readcaves()
troggle.parsers.caves.readcaves()
def import_people():
import parsers.people
import troggle.parsers.people
print("Importing People (folk.csv)")
parsers.people.LoadPersonsExpos()
troggle.parsers.people.LoadPersonsExpos()
def import_logbooks():
import parsers.logbooks
import troggle.parsers.logbooks
print("Importing Logbooks")
parsers.logbooks.LoadLogbooks()
troggle.parsers.logbooks.LoadLogbooks()
def import_QMs():
print("Importing QMs (old caves)")
import parsers.QMs
import troggle.parsers.QMs
# import process itself runs on qm.csv in only 3 old caves, not the modern ones!
def import_survexblks():
import parsers.survex
import troggle.parsers.survex
print("Importing Survex Blocks")
parsers.survex.LoadAllSurvexBlocks()
troggle.parsers.survex.LoadAllSurvexBlocks()
def import_survexpos():
import parsers.survex
import troggle.parsers.survex
print("Importing Survex x/y/z Positions")
parsers.survex.LoadPos()
troggle.parsers.survex.LoadPos()
def import_surveyimgs():
"""This appears to store data in unused objects. The code is kept
for future re-working to manage progress against notes, plans and elevs.
"""
import parsers.surveys
print("Importing survey images")
parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
#import troggle.parsers.surveys
print("NOT Importing survey images")
#troggle.parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
def import_surveyscans():
import parsers.surveys
import troggle.parsers.surveys
print("Importing Survey Scans")
parsers.surveys.LoadListScans()
troggle.parsers.surveys.LoadListScans()
def import_tunnelfiles():
import parsers.surveys
import troggle.parsers.surveys
print("Importing Tunnel files")
parsers.surveys.LoadTunnelFiles()
troggle.parsers.surveys.LoadTunnelFiles()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# These functions moved to a different file - not used currently.
@ -152,7 +158,7 @@ class JobQueue():
for j in data:
self.results[j] = data[j]
except:
print "FAILURE parsing JSON file %s" % (self.tfile)
print("FAILURE parsing JSON file %s" % (self.tfile))
# Python bug: https://github.com/ShinNoNoir/twitterwebsearch/issues/12
f.close()
for j in self.results_order:
@ -176,7 +182,7 @@ class JobQueue():
"""Run all the jobs in the queue provided - once
"""
print "** Running job ", self.runlabel
print("** Running job ", self.runlabel)
jobstart = time.time()
self.results["date"].pop()
self.results["date"].append(jobstart)
@ -187,14 +193,14 @@ class JobQueue():
start = time.time()
i[1]() # looks ugly but invokes function passed in the second item in the tuple
duration = time.time()-start
print "\n*- Ended \"", i[0], "\" %.1f seconds" % duration
print("\n*- Ended \"", i[0], "\" %.1f seconds" % duration)
self.results[i[0]].pop() # the null item
self.results[i[0]].append(duration)
jobend = time.time()
jobduration = jobend-jobstart
print "** Ended job %s - %.1f seconds total." % (self.runlabel,jobduration)
print("** Ended job %s - %.1f seconds total." % (self.runlabel,jobduration))
return True
@ -221,7 +227,7 @@ class JobQueue():
else:
skipmem = True
print "-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE']
print("-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'])
#print "-- DATABASES.default", settings.DATABASES['default']
if dbname ==":memory:":
@ -251,8 +257,8 @@ class JobQueue():
'PORT': ''}
print "-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE']
print "-- DATABASES.default", settings.DATABASES['default']
print("-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'])
#print("-- DATABASES.default", settings.DATABASES['default'])
# but because the user may be expecting to add this to a db with lots of tables already there,
# the jobqueue may not start from scratch so we need to initialise the db properly first
@ -282,7 +288,7 @@ class JobQueue():
settings.DATABASES['default'] = dbdefault
settings.DATABASES['default']['ENGINE'] = dbengine
settings.DATABASES['default']['NAME'] = dbname
print "-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE']
print("-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'])
django.db.close_old_connections() # maybe not needed here
for j in self.results_order:
@ -308,9 +314,9 @@ class JobQueue():
elif k =="test":
break
elif k =="date":
print " days ago ",
print(" days ago ", end=' ')
else:
print '%10s (s)' % k,
print('%10s (s)' % k, end=' ')
percen=0
r = self.results[k]
@ -320,26 +326,30 @@ class JobQueue():
rp = r[i]
else:
rp = " - "
print '%8s' % rp,
print('%8s' % rp, end=' ')
elif k =="date":
# Calculate dates as days before present
if r[i]:
if i == len(r)-1:
print " this",
print(" this", end=' ')
else:
# prints one place to the left of where you expect
days = (r[i]-r[len(r)-1])/(24*60*60)
print '%8.2f' % days,
if r[len(r)-1]:
s = r[i]-r[len(r)-1]
else:
s = 0
days = (s)/(24*60*60)
print('%8.2f' % days, end=' ')
elif r[i]:
print '%8.1f' % r[i],
print('%8.1f' % r[i], end=' ')
if i == len(r)-1 and r[i-1]:
percen = 100* (r[i] - r[i-1])/r[i-1]
if abs(percen) >0.1:
print '%8.1f%%' % percen,
print('%8.1f%%' % percen, end=' ')
else:
print " - ",
print ""
print "\n"
print(" - ", end=' ')
print("")
print("\n")
return True
@ -438,13 +448,15 @@ if __name__ == "__main__":
elif "dumplogbooks" in sys.argv: # untested in 2020
dumplogbooks()
elif "profile" in sys.argv:
jq.loadprofiles()
jq.showprofile()
exit()
elif "help" in sys.argv:
usage()
exit()
else:
usage()
print("%s not recognised as a command." % sys.argv[1])
print(("%s not recognised as a command." % sys.argv[1]))
exit()
jq.run()

8
parsers/survex.py
View File

@ -27,6 +27,8 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
ssto = survexblock.MakeSurvexStation(ls[stardata["to"]])
survexleg = models.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto)
# this next fails for two surface survey svx files which use / for decimal point
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
if stardata["type"] == "normal":
try:
survexleg.tape = float(ls[stardata["tape"]])
@ -34,7 +36,7 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
print("! Tape misread in", survexblock.survexfile.path)
print(" Stardata:", stardata)
print(" Line:", ls)
message = ' ! Value Error: line %s in %s' % (ls, survexblock.survexfile.path)
message = ' ! Value Error: Tape misread in line %s in %s' % (ls, survexblock.survexfile.path)
models.DataIssue.objects.create(parser='survex', message=message)
survexleg.tape = 1000
try:
@ -43,7 +45,7 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
print("! Clino misread in", survexblock.survexfile.path)
print(" Stardata:", stardata)
print(" Line:", ls)
message = ' ! Value Error: line %s in %s' % (ls, survexblock.survexfile.path)
message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path)
models.DataIssue.objects.create(parser='survex', message=message)
lclino = error
try:
@ -52,7 +54,7 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
print("! Compass misread in", survexblock.survexfile.path)
print(" Stardata:", stardata)
print(" Line:", ls)
message = ' ! Value Error: line %s in %s' % (ls, survexblock.survexfile.path)
message = ' ! Value Error: Compass misread in line %s in %s' % (ls, survexblock.survexfile.path)
models.DataIssue.objects.create(parser='survex', message=message)
lcompass = error
if lclino == "up":

30
parsers/surveys.py
View File

@ -82,14 +82,14 @@ def get_or_create_placeholder(year):
# logging.info("added survey " + survey[header['Year']] + "#" + surveyobj.wallet_number + "\r")
# dead
def listdir(*directories):
try:
return os.listdir(os.path.join(settings.SURVEYS, *directories))
except:
import urllib.request, urllib.parse, urllib.error
url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories))
folders = urllib.request.urlopen(url.replace("#", "%23")).readlines()
return [folder.rstrip(r"/") for folder in folders]
# def listdir(*directories):
# try:
# return os.listdir(os.path.join(settings.SURVEYS, *directories))
# except:
# import urllib.request, urllib.parse, urllib.error
# url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories))
# folders = urllib.request.urlopen(url.replace("#", "%23")).readlines()
# return [folder.rstrip(r"/") for folder in folders]
# add survey scans
# def parseSurveyScans(expedition, logfile=None):
@ -171,13 +171,13 @@ def listdir(*directories):
# parseSurveyScans(expedition)
# dead
def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the thumbnail engine can't handle them (uses PIL)
file=Image.open(filePath)
print(filePath)
if 'interlace' in file.info:
return file.info['interlace']
else:
return False
# def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the thumbnail engine can't handle them (uses PIL)
# file=Image.open(filePath)
# print(filePath)
# if 'interlace' in file.info:
# return file.info['interlace']
# else:
# return False
# handles url or file, so we can refer to a set of scans on another server