xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Made a jobqueue to time how long importing takes

Browse Source
This commit is contained in:
Philip Sargent 2020-04-15 04:09:28 +01:00
parent 5e48687347
commit 9db7d8e589
1 changed files with 127 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
databaseReset.py
View File

@ -1,5 +1,6 @@
import os
import time
import timeit
import settings
os.environ['PYTHONPATH'] = settings.PYTHON_PATH
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
@ -79,7 +80,7 @@ def pageredirects():
f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
f.save()
def reset():
def reset(): # unused now that we have a jobqueue
""" Wipe the troggle database and import everything from legacy data
"""
reload_db()
@ -148,6 +149,93 @@ def dumplogbooks():
f.write(unicode(output).encode( "utf-8" ))
f.close()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
class JobQueue():
""" A list of import operations to run. Always reports times
in the same order """
#Constructor creates a list
def __init__(self,run):
self.runlabel = run
self.queue = [] # tuples of (jobname, jobfunction)
self.results = {}
self.results_order=[
"date","runlabel","reload", "caves", "people",
"logbooks", "surveyscans", "QMs", "survex"
"tunnel", "surveys", "test", "makedirs", "redirect" ]
for k in self.results_order:
self.results[k]=[]
#Adding elements to queue
def enq(self,label,func):
self.queue.append((label,func))
return True
#Removing the last element from the queue
def deq(self):
if len(self.queue)>0:
return self.queue.pop()
return ("Queue Empty!")
def size(self):
return len(self.queue)
def run(self):
import json
tfile = "import_times.json"
if os.path.isfile(tfile):
try:
f = open(tfile, "r")
data = json.load(f)
for j in data:
self.results[j] = data[j]
except:
print "FAILURE parsing JSON file %s" % (tfile)
# Python bug: https://github.com/ShinNoNoir/twitterwebsearch/issues/12
f.close()
print "** Running job ", self.runlabel
for i in self.queue:
#print "*- Running \"", i[0], "\""
start = time.time()
i[1]()
duration = time.time()-start
#print "\n*- Ended \"", i[0], "\""
self.results[i[0]].append(duration)
self.results["date"].append(start)
self.results["runlabel"].append(self.runlabel)
print "** Ended all jobs."
#print self.results
with open(tfile, 'w') as f:
json.dump(self.results, f)
for i in self.results_order:
percen=0
if i == "runlabel":
pass
if i =="date":
# Calculate dates as days before present to one decimal place
pass
elif len(self.results[i])>0:
lst = self.results[i]
e = len(lst)-1
percen = 100* (lst[e] - lst[e-1])/lst[e-1]
if abs(percen) >0.1:
print '%15s %8.1f%%' % (i, percen)
else:
print '%15s ' % (i)
return True
def importtest():
from random import randrange
k = 0
for i in range(5+randrange(15)):
for j in range(i):
k += i
#print k,
return True
def usage():
print("""Usage is 'python databaseReset.py <command>'
@ -160,14 +248,16 @@ def usage():
reload_db - clear database i.e. delete everything
scans - NOT the scanned surveynotes ?!
survex - read in the survex files - all the survex blocks
survexpos - just the Pos out of the survex files
surveys - read in the scanned surveynotes
tunnel - read in the Tunnel files - which scans the surveyscans too
survexpos - just the Pos out of the survex files (not part of reset)
resetend - (archaic?)
writecaves - *disabled* (archaic?)
autologbooks - read in autologbooks (what are these?)
dumplogbooks - write out autologbooks (not working?)
test - testing...
""")
if __name__ == "__main__":
@ -176,38 +266,56 @@ if __name__ == "__main__":
import django
django.setup()
if "caves" in sys.argv:
import_caves()
runlabel = sys.argv[len(sys.argv)-1]
jq = JobQueue(runlabel)
if "test" in sys.argv:
jq.enq("test",importtest)
jq.enq("caves",importtest)
jq.enq("people",importtest)
elif "caves" in sys.argv:
jq.enq("caves",import_caves)
elif "logbooks" in sys.argv:
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
import_logbooks()
jq.enq("logbooks",import_logbooks)
elif "people" in sys.argv:
import_people()
jq.enq("logbooks",import_people)
elif "QMs" in sys.argv:
import_QMs()
jq.enq("QMs",import_QMs)
elif "reload_db" in sys.argv:
reload_db()
jq.enq("reload_db",reload_db)
elif "reset" in sys.argv:
reset()
jq.enq("reload",reload_db)
jq.enq("makedirs",make_dirs)
jq.enq("redirect",pageredirects)
jq.enq("caves",import_caves)
jq.enq("logbooks",import_people)
jq.enq("scans",import_surveyscans)
jq.enq("logbooks",import_logbooks)
jq.enq("QMs",import_QMs)
jq.enq("survex",import_survex)
jq.enq("tunnel",import_tunnelfiles)
jq.enq("surveys",import_surveys)
elif "scans" in sys.argv:
import_surveyscans()
jq.enq("scans",import_surveyscans)
elif "survex" in sys.argv:
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
import_survex()
jq.enq("survex",import_survex)
elif "survexpos" in sys.argv:
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
import parsers.survex
parsers.survex.LoadPos()
jq.enq("survexpos",parsers.survex.LoadPos)
elif "surveys" in sys.argv:
import_surveys()
jq.enq("surveys",import_surveys)
elif "tunnel" in sys.argv:
import_tunnelfiles()
jq.enq("tunnel",import_tunnelfiles)
elif "help" in sys.argv:
usage()
elif "resetend" in sys.argv:
import_QMs()
import_tunnelfiles()
import_surveys()
jq.enq("QMs",import_QMs)
jq.enq("tunnel",import_tunnelfiles)
jq.enq("surveys",import_surveys)
#import_descriptions() # no longer present
#parse_descriptions() # no longer present
# elif "writeCaves" in sys.argv:
@ -219,3 +327,5 @@ if __name__ == "__main__":
else:
print("%s not recognised" % sys.argv)
usage()
jq.run()