expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2025-12-15 00:27:20 +00:00
Code Issues 18 Projects 3 Activity

attempt to simplify wnt horribly wrong

Browse Source
This commit is contained in:
Philip Sargent
2020-06-06 22:51:55 +01:00
parent f8a3c8f5bc
commit fda50ed570
9 changed files with 135 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
databaseReset.py
View File

@@ -1,5 +1,4 @@
from __future__ import (absolute_import, division,
print_function)
import sys
import os
import time
import timeit
@@ -16,9 +15,18 @@ from django.http import HttpResponse
from django.core.urlresolvers import reverse
from troggle.core.models_caves import Cave, Entrance
import troggle.settings
import troggle.parsers.caves
#import troggle.settings
import troggle.flatpages.models
import troggle.logbooksdump
import troggle.parsers.people
import troggle.parsers.surveys
import troggle.parsers.logbooks
import troggle.parsers.QMs
import troggle.core.models
import troggle.core.models_survex
import django
# NOTE databaseReset.py is *imported* by views_other.py as it is used in the control panel
# presented there.
@@ -31,9 +39,9 @@ if os.geteuid() == 0:
expouser=settings.EXPOUSER
expouserpass=settings.EXPOUSERPASS
expouseremail=settings.EXPOUSER_EMAIL
print(" - SETTINGS: {} ({:.5}...) <{}> on module loading".format(expouser, expouserpass, expouseremail))
def call_django_tests(n):
management.call_command('test', verbosity=n)
def reinit_db():
"""Rebuild database from scratch. Deletes the file first if sqlite is used,
@@ -51,6 +59,7 @@ def reinit_db():
cursor.execute("CREATE DATABASE %s" % currentdbname)
cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % currentdbname)
cursor.execute("USE %s" % currentdbname)
print(" - SETTINGS: {} ({:.5}...) <{}> before calling syncuser()".format(expouser, expouserpass, expouseremail))
syncuser()
def syncuser():
@@ -74,37 +83,32 @@ def dirsredirect():
f.save()
def import_caves():
import troggle.parsers.caves
print("Importing Caves")
troggle.parsers.caves.readcaves()
def import_people():
import troggle.parsers.people
print("Importing People (folk.csv)")
troggle.parsers.people.LoadPersonsExpos()
def import_surveyscans():
print("Importing Survey Scans")
troggle.parsers.surveys.LoadListScans()
def import_logbooks():
import troggle.parsers.logbooks
print("Importing Logbooks")
troggle.parsers.logbooks.LoadLogbooks()
def import_QMs():
print("Importing QMs (old caves)")
import troggle.parsers.QMs
# import process itself runs on qm.csv in only 3 old caves, not the modern ones!
troggle.parsers.QMs.Load_QMs()
def import_surveyscans():
import troggle.parsers.surveys
print("Importing Survey Scans")
troggle.parsers.surveys.LoadListScans()
def import_survexblks():
import troggle.parsers.survex
# when this import is moved to the top with the rest it all crashes horribly
import troggle.parsers.survex
print("Importing Survex Blocks")
troggle.parsers.survex.LoadAllSurvexBlocks()
def import_survexpos():
import troggle.parsers.survex
print("Importing Survex x/y/z Positions")
troggle.parsers.survex.LoadPos()
@@ -117,7 +121,6 @@ def import_surveyimgs():
#troggle.parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
def import_tunnelfiles():
import troggle.parsers.surveys
print("Importing Tunnel files")
troggle.parsers.surveys.LoadTunnelFiles()
@@ -140,6 +143,10 @@ class JobQueue():
"""A list of import operations to run. Always reports profile times
in the same order.
"""
dbengine = ""
dbname = ""
dbdefault =""
def __init__(self,run):
self.runlabel = run
self.queue = [] # tuples of (jobname, jobfunction)
@@ -217,10 +224,54 @@ class JobQueue():
jobend = time.time()
jobduration = jobend-jobstart
print("** Ended job %s - %.1f seconds total." % (self.runlabel,jobduration))
return True
def store_dbsettings(self):
self.dbengine = settings.DATABASES['default']['ENGINE']
self.dbname = settings.DATABASES['default']['NAME']
self.dbdefault = settings.DATABASES['default']
def restore_dbsettings(self):
settings.DATABASES['default'] = self.dbdefault
settings.DATABASES['default']['ENGINE'] = self.dbengine
settings.DATABASES['default']['NAME'] = self.dbname
def set_in_memory_dbsettings(self):
django.db.close_old_connections() # needed if MySQL running?
settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3',
'AUTOCOMMIT': True,
'ATOMIC_REQUESTS': False,
'NAME': ':memory:',
'CONN_MAX_AGE': 0,
'TIME_ZONE': 'UTC',
'OPTIONS': {},
'HOST': '',
'USER': '',
'TEST': {'COLLATION': None, 'CHARSET': None, 'NAME': None, 'MIRROR': None},
'PASSWORD': '',
'PORT': ''}
settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
settings.DATABASES['default']['NAME'] = ":memory:"
def append_placeholders(self):
for j in self.results_order:
self.results[j].append(None) # append a placeholder
def run_now_django_tests(self,n):
self.store_dbsettings()
# this leaves the db set to :memory: whatever it was initially
management.call_command('test', verbosity=n)
django.db.close_old_connections()
self.restore_dbsettings()
def skip_memory_phase(self):
if not self.runlabel:
return True
else:
if self.runlabel == "" or self.runlabel[0:2] == "F-":
return True
return False
def run(self):
"""First runs all the jobs in the queue against a scratch in-memory db
then re-runs the import against the db specified in settings.py
@@ -229,51 +280,24 @@ class JobQueue():
relinquish some kind of db connection (not fixed yet)
"""
self.loadprofiles()
# save db settings for later
dbengine = settings.DATABASES['default']['ENGINE']
dbname = settings.DATABASES['default']['NAME']
dbdefault = settings.DATABASES['default']
skipmem = False
if self.runlabel:
if self.runlabel == "":
skipmem = True
elif self.runlabel[0:2] == "F-":
skipmem = True
else:
skipmem = True
print("-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'])
self.store_dbsettings()
print("-- phase 0 ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
#print "-- DATABASES.default", settings.DATABASES['default']
if dbname ==":memory:":
if self.dbname ==":memory:":
# just run, and save the sql file
self.runqonce()
self.memdumpsql() # saved contents of scratch db, could be imported later..
self.saveprofiles()
elif skipmem:
elif self.skip_memory_phase():
self.runqonce()
self.saveprofiles()
else:
django.db.close_old_connections() # needed if MySQL running?
# run all the imports through :memory: first
settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
settings.DATABASES['default']['NAME'] = ":memory:"
settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3',
'AUTOCOMMIT': True,
'ATOMIC_REQUESTS': False,
'NAME': ':memory:',
'CONN_MAX_AGE': 0,
'TIME_ZONE': 'UTC',
'OPTIONS': {},
'HOST': '',
'USER': '',
'TEST': {'COLLATION': None, 'CHARSET': None, 'NAME': None, 'MIRROR': None},
'PASSWORD': '',
'PORT': ''}
self.set_in_memory_dbsettings()
print("-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'])
print("-- phase 1 ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
#print("-- DATABASES.default", settings.DATABASES['default'])
# but because the user may be expecting to add this to a db with lots of tables already there,
@@ -281,7 +305,6 @@ class JobQueue():
# because we are using an empty :memory: database
# But initiating twice crashes it; so be sure to do it once only.
# Damn. syncdb() is still calling MySQL somehow **conn_params not sqlite3. So crashes on expo server.
if ("reinit",reinit_db) not in self.queue:
reinit_db()
@@ -301,20 +324,18 @@ class JobQueue():
# restore the original db and import again
# if we wanted to, we could re-import the SQL generated in the first pass to be
# blazing fast. But for the present just re-import the lot.
settings.DATABASES['default'] = dbdefault
settings.DATABASES['default']['ENGINE'] = dbengine
settings.DATABASES['default']['NAME'] = dbname
print("-- ", settings.DATABASES['default']['NAME'], settings.DATABASES['default']['ENGINE'])
self.restore_dbsettings()
print("-- phase 2 ", settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
django.db.close_old_connections() # maybe not needed here
for j in self.results_order:
self.results[j].pop() # throw away results from :memory: run
self.results[j].append(None) # append a placeholder
self.append_placeholders()
django.db.close_old_connections() # magic rune. works. found by looking in django.db__init__.py
#django.setup() # should this be needed?
self.runqonce() # crashes because it thinks it has no migrations to apply, when it does.
self.runqonce()
self.saveprofiles()
return True
@@ -405,9 +426,6 @@ def usage():
""")
if __name__ == "__main__":
import troggle.core.models
import sys
import django
django.setup()
if os.geteuid() == 0:
@@ -419,16 +437,16 @@ if __name__ == "__main__":
else:
runlabel=None
call_django_tests(1)
jq = JobQueue(runlabel)
jq.run_now_django_tests(1)
if len(sys.argv)==1:
usage()
exit()
elif "test" in sys.argv:
call_django_tests(2)
jq.enq("caves",import_caves)
jq.enq("people",import_people)
#jq.run_now_django_tests(2)
elif "caves" in sys.argv:
jq.enq("caves",import_caves)
elif "logbooks" in sys.argv:
@@ -466,6 +484,9 @@ if __name__ == "__main__":
# writeCaves()
elif "profile" in sys.argv:
jq.loadprofiles()
# need to increment everything runq does
print("!! - days before appears as 0.00 - to be fixed")
jq.append_placeholders()
jq.showprofile()
exit()
elif "help" in sys.argv: