xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

dbReset now loads into memory first (fast err checking), then into db

Browse Source
This commit is contained in:
Philip Sargent
2020-04-30 23:15:57 +01:00
parent 76a6b501f3
commit 39c622d5bf
4 changed files with 123 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
databaseReset.py
View File

@@ -5,7 +5,7 @@ import settings
os.environ['PYTHONPATH'] = settings.PYTHON_PATH
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
from django.core import management
from django.db import connection
from django.db import connection, close_old_connections
from django.contrib.auth.models import User
from django.http import HttpResponse
from django.core.urlresolvers import reverse
@@ -13,6 +13,9 @@ from troggle.core.models import Cave, Entrance
import troggle.flatpages.models
import json
# NOTE databaseRest.py is *imported* by views_other.py as it is used in the control panel
# presented there.
databasename=settings.DATABASES['default']['NAME']
expouser=settings.EXPOUSER
expouserpass=settings.EXPOUSERPASS
@@ -22,17 +25,18 @@ def reinit_db():
"""Rebuild database from scratch. Deletes the file first if sqlite is used,
otherwise it drops the database and creates it.
"""
currentdbname = settings.DATABASES['default']['NAME']
if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
try:
os.remove(databasename)
os.remove(currentdbname)
except OSError:
pass
else:
cursor = connection.cursor()
cursor.execute("DROP DATABASE %s" % databasename)
cursor.execute("CREATE DATABASE %s" % databasename)
cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % databasename)
cursor.execute("USE %s" % databasename)
cursor.execute("DROP DATABASE %s" % currentdbname)
cursor.execute("CREATE DATABASE %s" % currentdbname)
cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % currentdbname)
cursor.execute("USE %s" % currentdbname)
syncuser()
def syncuser():
@@ -73,7 +77,7 @@ def import_logbooks():
def import_QMs():
print("Importing QMs (old caves)")
import parsers.QMs
# import process itself runs on qm.csv in only 3 caves, not 264!
# import process itself runs on qm.csv in only 3 old caves, not the modern ones!
def import_survexblks():
import parsers.survex
@@ -159,7 +163,7 @@ def dumplogbooks():
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
class JobQueue():
"""A list of import operations to run. Always reports times
"""A list of import operations to run. Always reports profile times
in the same order.
"""
def __init__(self,run):
@@ -173,7 +177,7 @@ class JobQueue():
for k in self.results_order:
self.results[k]=[]
self.tfile = "import_profile.json"
self.htmlfile = "profile.html"
self.htmlfile = "profile.html" # for HTML results table. Not yet done.
#Adding elements to queue - enqueue
def enq(self,label,func):
@@ -186,7 +190,9 @@ class JobQueue():
# return self.queue.pop()
# return ("Queue Empty!")
def run(self):
def loadprofiles(self):
"""Load timings for previous runs from file
"""
if os.path.isfile(self.tfile):
try:
f = open(self.tfile, "r")
@@ -197,9 +203,26 @@ class JobQueue():
print "FAILURE parsing JSON file %s" % (self.tfile)
# Python bug: https://github.com/ShinNoNoir/twitterwebsearch/issues/12
f.close()
for j in self.results_order:
self.results[j].append(None) # append a placeholder
return True
def saveprofiles(self):
with open(self.tfile, 'w') as f:
json.dump(self.results, f)
return True
def memdumpsql(self):
djconn = django.db.connection
from dump import _iterdump
with open('memdump.sql', 'w') as f:
for line in _iterdump(djconn):
f.write('%s\n' % line.encode("utf8"))
return True
def runqonce(self):
"""Run all the jobs in the queue provided once
"""
print "** Running job ", self.runlabel
jobstart = time.time()
@@ -216,26 +239,68 @@ class JobQueue():
self.results[i[0]].pop() # the null item
self.results[i[0]].append(duration)
with open(self.tfile, 'w') as f:
json.dump(self.results, f)
jobend = time.time()
jobduration = jobend-jobstart
print "** Ended all jobs. %.1f seconds" % jobduration
print "** Ended job %s - %.1f seconds total." % (self.runlabel,jobduration)
return True
def run(self):
self.loadprofiles()
# currently uses django db whatever it was. CHANGE this to explicitly use
# a new sqlite3 db and then import the sql dump of that into the troggle db
# instead of loading directly into the troggle sqlite db.
# in-memory ":memory:" sqlite is ~ 7x faster and all of troggle can be
# loaded in 6 minutes that way
djconn = django.db.connection
from dump import _iterdump
with open('memdump.sql', 'w') as f:
for line in _iterdump(djconn):
f.write('%s\n' % line.encode("utf8"))
dbengine = settings.DATABASES['default']['ENGINE']
dbname = settings.DATABASES['default']['NAME']
# now import the memory image sql into
####(to do)
if dbname ==":memory:":
# just run, and save the sql file
print "-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE']
self.runqonce()
self.memdumpsql()
self.saveprofiles()
else:
# run all the imports through :memory: first
settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
settings.DATABASES['default']['NAME'] = ":memory:"
print "-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE']
# but because the user may be expecting to add this to a db with lots of tables already there,
# the jobque may not start from scratch so we need to initialise the db properly first.
# But initiating twice crashes, so be sure to do it once only.
if ("reinit",reinit_db) not in self.queue:
reinit_db()
if ("dirsredirect",dirsredirect) not in self.queue:
dirsredirect()
if ("caves",import_caves) not in self.queue:
import_caves()
if ("people",import_people) not in self.queue:
import_people()
django.db.close_old_connections() # maybe not needed here
self.runqonce()
self.memdumpsql()
self.showprofile()
# restore the original db and import again
# if we wanted to, we could re-import the SQL generated in the first pass to be
# blazing fast. But for the present just re-import the lot.
settings.DATABASES['default']['ENGINE'] = dbengine
settings.DATABASES['default']['NAME'] = dbname
print "-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE']
for j in self.results_order:
self.results[j].pop() # throw away results from :memory: run
self.results[j].append(None) # append a placeholder
django.db.close_old_connections() # magic rune. works. found by looking in django.db__init__.py
#django.setup() # should this be needed?
self.runqonce() # crashes because it thinks it has no migrations to apply, when it does.
self.saveprofiles()
return True
def showprofile(self):
@@ -277,9 +342,10 @@ class JobQueue():
percen = 100* (r[i] - r[i-1])/r[i-1]
if abs(percen) >0.1:
print '%8.1f%%' % percen,
else:
print " - ",
else:
print " - ",
print ""
print "\n"
return True
@@ -333,8 +399,8 @@ if __name__ == "__main__":
jq.enq("reinit",reinit_db)
jq.enq("dirsredirect",dirsredirect)
jq.enq("caves",import_caves)
jq.enq("survexblks",import_survexblks)
jq.enq("survexpos",import_survexpos)
jq.enq("people",import_people)
jq.enq("scans",import_surveyscans)
elif "caves" in sys.argv:
jq.enq("caves",import_caves)
elif "logbooks" in sys.argv: