xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Thorough spring clean and profiling

Browse Source
This commit is contained in:
Philip Sargent
2020-04-27 23:51:41 +01:00
parent 2b39dec560
commit a8460065a4
4 changed files with 129 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
databaseReset.py
View File

@@ -18,7 +18,10 @@ expouser=settings.EXPOUSER
expouserpass=settings.EXPOUSERPASS
expouseremail=settings.EXPOUSER_EMAIL
def reload_db():
def reinit_db():
"""Rebuild database from scratch. Deletes the file first if sqlite is used,
otherwise it drops the database and creates it.
"""
if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
try:
os.remove(databasename)
@@ -30,26 +33,27 @@ def reload_db():
cursor.execute("CREATE DATABASE %s" % databasename)
cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % databasename)
cursor.execute("USE %s" % databasename)
management.call_command('syncdb', interactive=False)
user = User.objects.create_user(expouser, expouseremail, expouserpass)
user.is_staff = True
user.is_superuser = True
user.save()
syncuser()
def syncuser():
"""Sync user - needed after reload"""
"""Sync user - needed after reload
"""
print("Synchronizing user")
management.call_command('syncdb', interactive=False)
user = User.objects.create_user(expouser, expouseremail, expouserpass)
user.is_staff = True
user.is_superuser = True
user.save()
def make_dirs():
"""Make directories that troggle requires"""
def dirsredirect():
"""Make directories that troggle requires and sets up page redirects
"""
#should also deal with permissions here.
if not os.path.isdir(settings.PHOTOS_ROOT):
os.mkdir(settings.PHOTOS_ROOT)
for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
f.save()
def import_caves():
import parsers.caves
@@ -58,38 +62,49 @@ def import_caves():
def import_people():
import parsers.people
print("Importing People (folk.csv)")
parsers.people.LoadPersonsExpos()
def import_logbooks():
import parsers.logbooks
print("Importing Logbooks")
parsers.logbooks.LoadLogbooks()
def import_survex():
import parsers.survex
parsers.survex.LoadAllSurvexBlocks()
parsers.survex.LoadPos()
def import_QMs():
print("Importing QMs (old caves)")
import parsers.QMs
# import process itself runs on qm.csv in only 3 caves, not 264!
def import_survex():
import parsers.survex
print("Importing Survex Blocks")
parsers.survex.LoadAllSurvexBlocks()
print("Importing Survex Positions")
parsers.survex.LoadPos()
def import_survexpos():
import parsers.survex
print("Importing Survex Positions")
parsers.survex.LoadPos()
def import_surveys():
"""This appears to store data in unused objects. The code is kept
for future re-working to manage progress against notes, plans and elevs.
"""
import parsers.surveys
print("Importing surveys")
parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
def import_surveyscans():
import parsers.surveys
print("Importing Survey Scans")
parsers.surveys.LoadListScans()
def import_tunnelfiles():
import parsers.surveys
print("Importing Tunnel files")
parsers.surveys.LoadTunnelFiles()
def pageredirects():
for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
f.save()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def import_auto_logbooks():
import parsers.logbooks
@@ -154,9 +169,9 @@ class JobQueue():
self.queue = [] # tuples of (jobname, jobfunction)
self.results = {}
self.results_order=[
"date","runlabel","reload", "caves", "people",
"date","runlabel","reinit", "caves", "people",
"logbooks", "scans", "QMs", "survex",
"tunnel", "surveys", "test", "makedirs", "redirect", "syncuser" ]
"tunnel", "surveys", "test", "dirsredirect", "syncuser", "survexpos" ]
for k in self.results_order:
self.results[k]=[]
self.tfile = "import_profile.json"
@@ -191,12 +206,12 @@ class JobQueue():
self.results["runlabel"].append(self.runlabel)
for i in self.queue:
start = time.time()
i[1]() # looks ugly but invokes function passed in the second item in the tuple
duration = time.time()-start
print "\n*- Ended \"", i[0], "\" %.1f seconds" % duration
self.results[i[0]].append(duration)
start = time.time()
i[1]() # looks ugly but invokes function passed in the second item in the tuple
duration = time.time()-start
print "\n*- Ended \"", i[0], "\" %.1f seconds" % duration
self.results[i[0]].append(duration)
with open(self.tfile, 'w') as f:
json.dump(self.results, f)
@@ -207,7 +222,7 @@ class JobQueue():
# currently uses django db whatever it was. CHANGE this to explicitly use
# a new sqlite3 db and then import the sql dump of that into the troggle db
# instead of loading directly into the troggle sqlite db.
# in-menmor ":memory:" sqlite is ~ 7x faster and all of troggle can be
# in-memory ":memory:" sqlite is ~ 7x faster and all of troggle can be
# loaded in 6 minutes that way
djconn = django.db.connection
from dump import _iterdump
@@ -221,52 +236,76 @@ class JobQueue():
def showprofile(self):
"""Prints out the time it took to run the jobqueue"""
for k in self.results_order:
percen=0
lst = self.results[k]
if k == "runlabel":
r = lst[len(lst)-1]
print '%15s %s' % (k,r)
if k =="dirsredirect":
break
elif k =="syncuser":
break
elif k =="test":
break
elif k =="date":
# Calculate dates as days before present to one decimal place
r = lst[len(lst)-1]
if len(lst)>2:
days = (lst[len(lst)-2]-r)/(24*60*60)
print '%15s %8.1f days ago' % (k,days)
elif len(lst)>2:
e = len(lst)-1
percen = 100* (lst[e] - lst[e-1])/lst[e-1]
if abs(percen) >0.1:
print '%15s %8.1f%%' % (k, percen)
else:
print '%15s ' % (k)
print " days ago ",
else:
print '%9s (s)' % k,
percen=0
r = self.results[k]
#print "min=",min
for i in range(len(r)):
if k == "runlabel":
if r[i]:
rp = r[i]
else:
rp = " - "
print '%8s' % rp,
elif k =="date":
# Calculate dates as days before present
if r[i]:
if i == len(r)-1:
print " this",
else:
# prints one place to the left of where you expect
days = (r[i]-r[len(r)-1])/(24*60*60)
print '%8.2f' % days,
elif r[i]:
print '%8.1f' % r[i],
if i == len(r)-1 and r[i-1]:
percen = 100* (r[i] - r[i-1])/r[i-1]
if abs(percen) >0.1:
print '%8.1f%%' % percen,
else:
print " - ",
print ""
return True
def usage():
print("""Usage is 'python databaseReset.py <command> [runlabel]'
where command is:
reset - this is normal usage, clear database and reread everything from files - time-consuming
reset - normal usage: clear database and reread everything from files - time-consuming
caves - read in the caves
logbooks - read in just the logbooks
logbooks - read in the logbooks
people - read in the people from folk.csv
QMs - read in the QM csv files
reload_db - clear database (delete everything) and make empty tables
scans - NOT the scanned surveynotes ?!
QMs - read in the QM csv files (older caves only)
reinit - clear database (delete everything) and make empty tables. Import nothing.
scans - the survey scans in all the wallets
survex - read in the survex files - all the survex blocks
surveys - read in the scanned surveynotes
tunnel - read in the Tunnel files - which scans the surveyscans too
survexpos - just the Pos out of the survex files (not part of reset)
tunnel - read in the Tunnel files - which scans the survey scans too
resetend - (archaic?)
writecaves - *disabled* (archaic?)
autologbooks - read in autologbooks (what are these?)
dumplogbooks - write out autologbooks (not working?)
syncuser - needed after reloading database rom SQL backup
syncuser - needed after reloading database from SQL backup
surveys - read in scans by expo, must run after "people". Not used.
test - testing...
and [runlabel] is an optional string identifying this run of the script
in the stored profiling data 'import-profile.json'
caves and logbooks must be run on an empty db before the others as they
set up db tables used by the others.
""")
if __name__ == "__main__":
@@ -275,31 +314,35 @@ if __name__ == "__main__":
import django
django.setup()
runlabel = sys.argv[len(sys.argv)-1]
if len(sys.argv)>2:
runlabel = sys.argv[len(sys.argv)-1]
else:
runlabel=None
jq = JobQueue(runlabel)
if "test" in sys.argv:
jq.enq("reload",reload_db)
jq.enq("makedirs",make_dirs)
if len(sys.argv)==1:
usage()
exit()
elif "test" in sys.argv:
jq.enq("reinit",reinit_db)
jq.enq("dirsredirect",dirsredirect)
jq.enq("caves",import_caves)
jq.enq("survex",import_survex)
jq.enq("surveys",import_surveys)
#jq.enq("people",import_people)
#jq.enq("logbooks",import_logbooks)
elif "caves" in sys.argv:
jq.enq("caves",import_caves)
elif "logbooks" in sys.argv:
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
jq.enq("logbooks",import_logbooks)
elif "people" in sys.argv:
jq.enq("logbooks",import_people)
jq.enq("people",import_people)
elif "QMs" in sys.argv:
jq.enq("QMs",import_QMs)
elif "reload_db" in sys.argv:
jq.enq("reload",reload_db)
elif "reset" in sys.argv:
jq.enq("reload",reload_db)
jq.enq("makedirs",make_dirs)
jq.enq("redirect",pageredirects)
jq.enq("reinit",reinit_db)
jq.enq("dirsredirect",dirsredirect)
jq.enq("caves",import_caves)
jq.enq("people",import_people)
jq.enq("scans",import_surveyscans)
@@ -307,16 +350,12 @@ if __name__ == "__main__":
jq.enq("QMs",import_QMs)
jq.enq("survex",import_survex)
jq.enq("tunnel",import_tunnelfiles)
jq.enq("surveys",import_surveys)
elif "scans" in sys.argv:
jq.enq("scans",import_surveyscans)
elif "survex" in sys.argv:
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
jq.enq("survex",import_survex)
elif "survexpos" in sys.argv:
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
import parsers.survex
jq.enq("survexpos",parsers.survex.LoadPos)
jq.enq("survexpos",import_survexpos)
elif "surveys" in sys.argv:
jq.enq("surveys",import_surveys)
elif "tunnel" in sys.argv:
@@ -336,8 +375,9 @@ if __name__ == "__main__":
elif "dumplogbooks" in sys.argv:
dumplogbooks()
else:
print("%s not recognised" % sys.argv)
usage()
print("%s not recognised as a command." % sys.argv[1])
exit()
jq.run()
jq.showprofile()