forked from expo/troggle
fixed circular ref on setup & in-memory db
This commit is contained in:
Philip Sargent
parent
9237a6262e
commit
538a3b6ca8
@ -151,31 +151,36 @@ def get_name(pe):
|
||||
return pe.person.first_name
|
||||
|
||||
class UploadFileForm(forms.Form):
|
||||
title = forms.CharField(max_length=50)
|
||||
file = forms.FileField()
|
||||
html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30}))
|
||||
lon_utm = forms.FloatField(required=False)
|
||||
lat_utm = forms.FloatField(required=False)
|
||||
slug = forms.CharField(max_length=50)
|
||||
date = forms.DateField(required=False)
|
||||
# Because this has EXECUTABLE statements in its signature (the fields) they get
|
||||
# executed when this module is LOADED. Which barfs horribly.
|
||||
# so all replaced by an __init__ method instead.
|
||||
def __init__(self):
|
||||
title = forms.CharField(max_length=50)
|
||||
file = forms.FileField()
|
||||
html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30}))
|
||||
lon_utm = forms.FloatField(required=False)
|
||||
lat_utm = forms.FloatField(required=False)
|
||||
slug = forms.CharField(max_length=50)
|
||||
date = forms.DateField(required=False)
|
||||
|
||||
caves = [cave.slug for cave in Cave.objects.all()]
|
||||
#caves.sort()
|
||||
caves = ["-----"] + caves
|
||||
cave = forms.ChoiceField([(c, c) for c in caves], required=False)
|
||||
#This next line is the one that causes django.setup() to BARF LOUDLY
|
||||
caves = [cave.slug for cave in Cave.objects.all()]
|
||||
#caves.sort() # sort needs rewriting for python3
|
||||
caves = ["-----"] + caves
|
||||
cave = forms.ChoiceField([(c, c) for c in caves], required=False)
|
||||
|
||||
entrance = forms.ChoiceField([("-----", "Please select a cave"), ], required=False)
|
||||
qm = forms.ChoiceField([("-----", "Please select a cave"), ], required=False)
|
||||
entrance = forms.ChoiceField([("-----", "Please select a cave"), ], required=False)
|
||||
qm = forms.ChoiceField([("-----", "Please select a cave"), ], required=False)
|
||||
|
||||
expeditions = [e.year for e in Expedition.objects.all()]
|
||||
expeditions.sort()
|
||||
expeditions = ["-----"] + expeditions
|
||||
expedition = forms.ChoiceField([(e, e) for e in expeditions], required=False)
|
||||
expeditions = [e.year for e in Expedition.objects.all()]
|
||||
expeditions.sort()
|
||||
expeditions = ["-----"] + expeditions
|
||||
expedition = forms.ChoiceField([(e, e) for e in expeditions], required=False)
|
||||
|
||||
logbookentry = forms.ChoiceField([("-----", "Please select an expedition"), ], required=False)
|
||||
logbookentry = forms.ChoiceField([("-----", "Please select an expedition"), ], required=False)
|
||||
|
||||
person = forms.ChoiceField([("-----", "Please select an expedition"), ], required=False)
|
||||
person = forms.ChoiceField([("-----", "Please select an expedition"), ], required=False)
|
||||
|
||||
survey_point = forms.CharField()
|
||||
survey_point = forms.CharField()
|
||||
|
||||
|
||||
|
||||
222
databaseReset.py
222
databaseReset.py
@ -9,11 +9,13 @@ os.environ['PYTHONPATH'] = settings.PYTHON_PATH
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
|
||||
|
||||
from django.core import management
|
||||
from django.db import connection, close_old_connections
|
||||
from django.db import connection, close_old_connections, connections
|
||||
from django.contrib.auth.models import User
|
||||
from django.http import HttpResponse
|
||||
from django.core.urlresolvers import reverse
|
||||
|
||||
print(" 1 settings on loading databaseReset.py")
|
||||
|
||||
from troggle.core.models_caves import Cave, Entrance
|
||||
import troggle.parsers.caves
|
||||
#import troggle.settings
|
||||
@ -36,67 +38,117 @@ if os.geteuid() == 0:
|
||||
print("This script should be run as expo not root - quitting")
|
||||
exit()
|
||||
|
||||
dbengine = ""
|
||||
dbname = ""
|
||||
dbdefault =""
|
||||
|
||||
expouser=settings.EXPOUSER
|
||||
expouserpass=settings.EXPOUSERPASS
|
||||
expouseremail=settings.EXPOUSER_EMAIL
|
||||
print(" - settings: {} ({:.5}...) <{}> on module loading".format(expouser, expouserpass, expouseremail))
|
||||
|
||||
|
||||
|
||||
def reinit_db():
|
||||
"""Rebuild database from scratch. Deletes the file first if sqlite is used,
|
||||
otherwise it drops the database and creates it.
|
||||
Note - initial loading of troggle.sqlite will already have populated the models
|
||||
in memory (django python models, not the database), so there is already a full load
|
||||
of stuff known. Deleting the db file does not clear memory.
|
||||
"""
|
||||
django.db.close_old_connections() # wipes an in-memory sqlite db
|
||||
currentdbname = settings.DATABASES['default']['NAME']
|
||||
if settings.DATABASES['default']['NAME'] == ':memory:':
|
||||
pass
|
||||
elif settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
|
||||
if currentdbname == ':memory:':
|
||||
# closing connections should wipe the in-memory database
|
||||
django.db.close_old_connections()
|
||||
for conn in django.db.connections.all():
|
||||
print(" ! Closing another connection to db...")
|
||||
conn.close()
|
||||
elif django.db.connections.databases['default']['ENGINE'] == 'django.db.backends.sqlite3':
|
||||
try:
|
||||
os.remove(currentdbname)
|
||||
except OSError:
|
||||
print(" ! OSError on removing: " + currentdbname + " (Is the file open in another app?\n")
|
||||
raise
|
||||
else:
|
||||
cursor = connection.cursor()
|
||||
cursor = django.db.connection.cursor()
|
||||
cursor.execute("DROP DATABASE %s" % currentdbname)
|
||||
cursor.execute("CREATE DATABASE %s" % currentdbname)
|
||||
cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % currentdbname)
|
||||
cursor.execute("USE %s" % currentdbname)
|
||||
print(" - SETTINGS: {} ({:.5}...) <{}> before calling syncuser()".format(expouser, expouserpass, expouseremail))
|
||||
syncuser()
|
||||
|
||||
def syncuser():
|
||||
"""Sync user - needed after reload
|
||||
"""
|
||||
print(" - Synchronizing user on: " + settings.DATABASES['default']['NAME'])
|
||||
#Sync user - needed after reload
|
||||
print(" - Migrating: " + settings.DATABASES['default']['NAME'])
|
||||
print(django.db.connections.databases['default']['NAME'])
|
||||
|
||||
management.call_command('migrate', interactive=False)
|
||||
print(" - done migration on: " + settings.DATABASES['default']['NAME'])
|
||||
try:
|
||||
print(" - Setting up admin user on: " + settings.DATABASES['default']['NAME'])
|
||||
print(django.db.connections.databases['default']['NAME'])
|
||||
print(" - user: {} ({:.5}...) <{}> ".format(expouser, expouserpass, expouseremail))
|
||||
user = User.objects.create_user(expouser, expouseremail, expouserpass)
|
||||
user.is_staff = True
|
||||
user.is_superuser = True
|
||||
user.save()
|
||||
except:
|
||||
print(" ! INTEGRITY ERROR user on: " + settings.DATABASES['default']['NAME'])
|
||||
print(django.db.connections.databases['default']['NAME'])
|
||||
print(" ! You probably have not got a clean db when you thought you had.\n")
|
||||
raise
|
||||
print(" ! Also you are probably NOT running an in-memory db now.\n")
|
||||
memdumpsql(fn='integrityfail.sql')
|
||||
django.db.connections.databases['default']['NAME'] = ':memory:'
|
||||
#raise
|
||||
|
||||
def memdumpsql(fn):
|
||||
djconn = django.db.connection
|
||||
from dump import _iterdump
|
||||
with open(fn, 'w') as f:
|
||||
for line in _iterdump(djconn):
|
||||
f.write('%s\n' % line.encode("utf8"))
|
||||
return True
|
||||
|
||||
def store_dbsettings():
|
||||
global dbengine
|
||||
global dbname
|
||||
global dbdefault
|
||||
dbengine = settings.DATABASES['default']['ENGINE']
|
||||
dbname = settings.DATABASES['default']['NAME']
|
||||
dbdefault = settings.DATABASES['default']
|
||||
|
||||
def restore_dbsettings():
|
||||
settings.DATABASES['default'] = dbdefault
|
||||
settings.DATABASES['default']['ENGINE'] = dbengine
|
||||
settings.DATABASES['default']['NAME'] = dbname
|
||||
django.db.connections.databases['default'] = dbdefault
|
||||
django.db.connections.databases['default']['ENGINE'] = dbengine
|
||||
django.db.connections.databases['default']['NAME'] = dbname
|
||||
|
||||
def set_in_memory_dbsettings():
|
||||
django.db.close_old_connections() # needed if MySQL running?
|
||||
settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3',
|
||||
'AUTOCOMMIT': True,
|
||||
'ATOMIC_REQUESTS': False,
|
||||
'NAME': ':memory:',
|
||||
'CONN_MAX_AGE': 0,
|
||||
'TIME_ZONE': 'UTC',
|
||||
'OPTIONS': {},
|
||||
'HOST': '',
|
||||
'USER': '',
|
||||
'TEST': {'COLLATION': None, 'CHARSET': None, 'NAME': None, 'MIRROR': None},
|
||||
'PASSWORD': '',
|
||||
'PORT': ''}
|
||||
settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
|
||||
settings.DATABASES['default']['NAME'] = ':memory:'
|
||||
django.db.connections.databases['default']['ENGINE'] = 'django.db.backends.sqlite3'
|
||||
django.db.connections.databases['default']['NAME'] = ':memory:'
|
||||
|
||||
def dirsredirect():
|
||||
"""Make directories that troggle requires and sets up page redirects
|
||||
"""
|
||||
#should also deal with permissions here.
|
||||
#if not os.path.isdir(settings.PHOTOS_ROOT):
|
||||
#os.mkdir(settings.PHOTOS_ROOT)
|
||||
for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
|
||||
f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
|
||||
f.save()
|
||||
|
||||
def import_caves():
|
||||
print("Importing Caves")
|
||||
print("Importing Caves to ",end="")
|
||||
print(django.db.connections.databases['default']['NAME'])
|
||||
troggle.parsers.caves.readcaves()
|
||||
|
||||
def import_people():
|
||||
print("Importing People (folk.csv)")
|
||||
print("Importing People (folk.csv) to ",end="")
|
||||
print(django.db.connections.databases['default']['NAME'])
|
||||
troggle.parsers.people.LoadPersonsExpos()
|
||||
|
||||
def import_surveyscans():
|
||||
@ -152,9 +204,6 @@ class JobQueue():
|
||||
"""A list of import operations to run. Always reports profile times
|
||||
in the same order.
|
||||
"""
|
||||
dbengine = ""
|
||||
dbname = ""
|
||||
dbdefault =""
|
||||
|
||||
def __init__(self,run):
|
||||
self.runlabel = run
|
||||
@ -163,7 +212,7 @@ class JobQueue():
|
||||
self.results_order=[
|
||||
"date","runlabel","reinit", "caves", "people",
|
||||
"logbooks", "QMs", "scans", "survexblks", "survexpos",
|
||||
"tunnel", "surveyimgs", "test", "dirsredirect", "syncuser" ]
|
||||
"tunnel", "surveyimgs", "test" ]
|
||||
for k in self.results_order:
|
||||
self.results[k]=[]
|
||||
self.tfile = "import_profile.json"
|
||||
@ -202,19 +251,13 @@ class JobQueue():
|
||||
json.dump(self.results, f)
|
||||
return True
|
||||
|
||||
def memdumpsql(self):
|
||||
djconn = django.db.connection
|
||||
from dump import _iterdump
|
||||
with open('memdump.sql', 'w') as f:
|
||||
for line in _iterdump(djconn):
|
||||
f.write('%s\n' % line.encode("utf8"))
|
||||
return True
|
||||
|
||||
def runqonce(self):
|
||||
"""Run all the jobs in the queue provided - once
|
||||
"""
|
||||
|
||||
print("** Running job ", self.runlabel)
|
||||
print("** Running job ", self.runlabel,end=" to ")
|
||||
print(django.db.connections.databases['default']['NAME'])
|
||||
jobstart = time.time()
|
||||
self.results["date"].pop()
|
||||
self.results["date"].append(jobstart)
|
||||
@ -235,43 +278,18 @@ class JobQueue():
|
||||
print("** Ended job %s - %.1f seconds total." % (self.runlabel,jobduration))
|
||||
return True
|
||||
|
||||
def store_dbsettings(self):
|
||||
self.dbengine = settings.DATABASES['default']['ENGINE']
|
||||
self.dbname = settings.DATABASES['default']['NAME']
|
||||
self.dbdefault = settings.DATABASES['default']
|
||||
|
||||
def restore_dbsettings(self):
|
||||
settings.DATABASES['default'] = self.dbdefault
|
||||
settings.DATABASES['default']['ENGINE'] = self.dbengine
|
||||
settings.DATABASES['default']['NAME'] = self.dbname
|
||||
|
||||
def set_in_memory_dbsettings(self):
|
||||
django.db.close_old_connections() # needed if MySQL running?
|
||||
settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3',
|
||||
'AUTOCOMMIT': True,
|
||||
'ATOMIC_REQUESTS': False,
|
||||
'NAME': ':memory:',
|
||||
'CONN_MAX_AGE': 0,
|
||||
'TIME_ZONE': 'UTC',
|
||||
'OPTIONS': {},
|
||||
'HOST': '',
|
||||
'USER': '',
|
||||
'TEST': {'COLLATION': None, 'CHARSET': None, 'NAME': None, 'MIRROR': None},
|
||||
'PASSWORD': '',
|
||||
'PORT': ''}
|
||||
settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
|
||||
settings.DATABASES['default']['NAME'] = ":memory:"
|
||||
|
||||
def append_placeholders(self):
|
||||
for j in self.results_order:
|
||||
self.results[j].append(None) # append a placeholder
|
||||
|
||||
def run_now_django_tests(self,n):
|
||||
self.store_dbsettings()
|
||||
store_dbsettings()
|
||||
# this leaves the db set to :memory: whatever it was initially
|
||||
management.call_command('test', verbosity=n)
|
||||
django.db.close_old_connections()
|
||||
self.restore_dbsettings()
|
||||
restore_dbsettings()
|
||||
# and whatever I do, it stays that way !
|
||||
|
||||
def skip_memory_phase(self):
|
||||
if not self.runlabel:
|
||||
@ -289,36 +307,33 @@ class JobQueue():
|
||||
relinquish some kind of db connection (not fixed yet)
|
||||
"""
|
||||
self.loadprofiles()
|
||||
self.store_dbsettings()
|
||||
store_dbsettings()
|
||||
|
||||
print("-- start ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
|
||||
print(django.db.connections.databases['default']['NAME'])
|
||||
|
||||
print("-- phase 0 ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
|
||||
#print "-- DATABASES.default", settings.DATABASES['default']
|
||||
|
||||
if self.dbname ==":memory:":
|
||||
if dbname ==":memory:":
|
||||
# just run, and save the sql file
|
||||
self.runqonce()
|
||||
self.memdumpsql() # saved contents of scratch db, could be imported later..
|
||||
memdumpsql('memdump.sql') # saved contents of scratch db, could be imported later..
|
||||
self.saveprofiles()
|
||||
elif self.skip_memory_phase():
|
||||
self.runqonce()
|
||||
self.saveprofiles()
|
||||
else:
|
||||
# run all the imports through :memory: first
|
||||
self.set_in_memory_dbsettings()
|
||||
set_in_memory_dbsettings()
|
||||
|
||||
print("-- phase 1 ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
|
||||
#print("-- DATABASES.default", settings.DATABASES['default'])
|
||||
|
||||
# but because the user may be expecting to add this to a db with lots of tables already there,
|
||||
# the jobqueue may not start from scratch so we need to initialise the db properly first
|
||||
# because we are using an empty :memory: database
|
||||
# But initiating twice crashes it; so be sure to do it once only.
|
||||
|
||||
# Damn. syncdb() is still calling MySQL somehow **conn_params not sqlite3. So crashes on expo server.
|
||||
# Damn. migrate() is still calling MySQL somehow **conn_params not sqlite3. So crashes on expo server.
|
||||
if ("reinit",reinit_db) not in self.queue:
|
||||
reinit_db()
|
||||
if ("dirsredirect",dirsredirect) not in self.queue:
|
||||
dirsredirect()
|
||||
if ("caves",import_caves) not in self.queue:
|
||||
import_caves() # sometime extract the initialising code from this and put in reinit...
|
||||
if ("people",import_people) not in self.queue:
|
||||
@ -327,21 +342,22 @@ class JobQueue():
|
||||
django.db.close_old_connections() # maybe not needed here
|
||||
|
||||
self.runqonce()
|
||||
self.memdumpsql()
|
||||
memdumpsql('memdump2.sql')
|
||||
self.showprofile()
|
||||
|
||||
# restore the original db and import again
|
||||
# if we wanted to, we could re-import the SQL generated in the first pass to be
|
||||
# blazing fast. But for the present just re-import the lot.
|
||||
self.restore_dbsettings()
|
||||
restore_dbsettings()
|
||||
print("-- phase 2 ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
|
||||
|
||||
print(django.db.connections.databases['default']['NAME'])
|
||||
|
||||
django.db.close_old_connections() # maybe not needed here
|
||||
for j in self.results_order:
|
||||
self.results[j].pop() # throw away results from :memory: run
|
||||
self.append_placeholders()
|
||||
|
||||
django.db.close_old_connections() # magic rune. works. found by looking in django.db__init__.py
|
||||
django.db.close_old_connections()
|
||||
#django.setup() # should this be needed?
|
||||
|
||||
self.runqonce()
|
||||
@ -353,13 +369,7 @@ class JobQueue():
|
||||
"""Prints out the time it took to run the jobqueue
|
||||
"""
|
||||
for k in self.results_order:
|
||||
if k =="dirsredirect":
|
||||
break
|
||||
if k =="surveyimgs":
|
||||
break
|
||||
elif k =="syncuser":
|
||||
break
|
||||
elif k =="test":
|
||||
if k =="test":
|
||||
break
|
||||
elif k =="date":
|
||||
print(" days ago ", end=' ')
|
||||
@ -384,6 +394,8 @@ class JobQueue():
|
||||
# prints one place to the left of where you expect
|
||||
if r[len(r)-1]:
|
||||
s = r[i]-r[len(r)-1]
|
||||
elif r[len(r)-2]:
|
||||
s = r[i]-r[len(r)-2]
|
||||
else:
|
||||
s = 0
|
||||
days = (s)/(24*60*60)
|
||||
@ -408,34 +420,35 @@ def usage():
|
||||
profile - print the profile from previous runs. Import nothing.
|
||||
|
||||
reset - normal usage: clear database and reread everything from files - time-consuming
|
||||
caves - read in the caves (must run first after reset)
|
||||
people - read in the people from folk.csv (must run before logbooks)
|
||||
caves - read in the caves (must run first after initialisation)
|
||||
people - read in the people from folk.csv (must run after 'caves')
|
||||
logbooks - read in the logbooks
|
||||
QMs - read in the QM csv files (older caves only)
|
||||
scans - the survey scans in all the wallets (must run before survex)
|
||||
survex - read in the survex files - all the survex blocks but not the x/y/z positions
|
||||
survexpos - just the x/y/z Pos out of the survex files
|
||||
survexpos - just the x/y/z Pos out of the survex files (not needed)
|
||||
|
||||
tunnel - read in the Tunnel files - which scans the survey scans too
|
||||
|
||||
reinit - clear database (delete everything) and make empty tables. Import nothing.
|
||||
syncuser - needed after reloading database from SQL backup
|
||||
autologbooks - Not used. read in autologbooks (what are these?)
|
||||
dumplogbooks - Not used. write out autologbooks (not working?)
|
||||
surveyimgs - Not used. read in scans by-expo, must run after "people".
|
||||
|
||||
and [runlabel] is an optional string identifying this run of the script
|
||||
in the stored profiling data 'import-profile.json'
|
||||
|
||||
if [runlabel] is absent or begins with "F-" then it will skip the :memory: pass
|
||||
|
||||
caves and logbooks must be run on an empty db before the others as they
|
||||
set up db tables used by the others.
|
||||
|
||||
the in-memory phase is on an empty db, so always runs reinit, caves & people for this phase
|
||||
the commands are first run on an in-memory empty database before being run on
|
||||
the actual persistent database. This is very fast and checks for import errors.
|
||||
|
||||
the initial in-memory phase is on an empty db, so always runs caves & people for this phase
|
||||
""")
|
||||
|
||||
if __name__ == "__main__":
|
||||
django.setup()
|
||||
|
||||
if os.geteuid() == 0:
|
||||
print("Do not run as root or using sudo - file permissions for cache files and logs will break")
|
||||
@ -446,8 +459,17 @@ if __name__ == "__main__":
|
||||
else:
|
||||
runlabel=None
|
||||
|
||||
store_dbsettings()
|
||||
set_in_memory_dbsettings()
|
||||
print(" - django.setup - next")
|
||||
try:
|
||||
django.setup()
|
||||
except:
|
||||
print(" ! COMPLICATED FAILURE. Does not occur with a valid 'troggle.sqlite' database in place.")
|
||||
raise
|
||||
print(" - django.setup - done")
|
||||
|
||||
jq = JobQueue(runlabel)
|
||||
#jq.run_now_django_tests(1)
|
||||
|
||||
if len(sys.argv)==1:
|
||||
usage()
|
||||
@ -455,7 +477,7 @@ if __name__ == "__main__":
|
||||
elif "test" in sys.argv:
|
||||
jq.enq("caves",import_caves)
|
||||
jq.enq("people",import_people)
|
||||
jq.run_now_django_tests(2)
|
||||
#jq.run_now_django_tests(2)
|
||||
elif "caves" in sys.argv:
|
||||
jq.enq("caves",import_caves)
|
||||
elif "logbooks" in sys.argv:
|
||||
@ -466,7 +488,6 @@ if __name__ == "__main__":
|
||||
jq.enq("QMs",import_QMs)
|
||||
elif "reset" in sys.argv:
|
||||
jq.enq("reinit",reinit_db)
|
||||
jq.enq("dirsredirect",dirsredirect)
|
||||
jq.enq("caves",import_caves)
|
||||
jq.enq("people",import_people)
|
||||
jq.enq("scans",import_surveyscans)
|
||||
@ -493,9 +514,6 @@ if __name__ == "__main__":
|
||||
# writeCaves()
|
||||
elif "profile" in sys.argv:
|
||||
jq.loadprofiles()
|
||||
# need to increment everything runq does
|
||||
print("!! - days before appears as 0.00 - to be fixed")
|
||||
jq.append_placeholders()
|
||||
jq.showprofile()
|
||||
exit()
|
||||
elif "help" in sys.argv:
|
||||
@ -506,5 +524,7 @@ if __name__ == "__main__":
|
||||
print("%s not recognised as a command." % sys.argv[1])
|
||||
exit()
|
||||
|
||||
#jq.run_now_django_tests(1)
|
||||
|
||||
jq.run()
|
||||
jq.showprofile()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user