forked from expo/troggle
467 lines
19 KiB
Python
467 lines
19 KiB
Python
import json
|
|
import locale
|
|
import os
|
|
import resource
|
|
import sys
|
|
import time
|
|
import timeit
|
|
|
|
import settings
|
|
|
|
""" Command-line utility for loading cave data files into troggle's database.
|
|
|
|
The command line options select which combination of classes of data will be imported,
|
|
e.g. cave data, logbook data, cavers (people) data. The set of imports requested are put
|
|
into a job queue in a valid order, toegether with any necessary initialisation if it is
|
|
a complete reset, and the queue is then executed.
|
|
|
|
In future all these functions may be moved to a control panel webpage running within the
|
|
troggle application.
|
|
"""
|
|
print(" - settings on loading databaseReset.py", flush=True)
|
|
|
|
os.environ['PYTHONPATH'] = str(settings.PYTHON_PATH)
|
|
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
|
|
|
|
print(" - settings on loading databaseReset.py")
|
|
|
|
import django
|
|
|
|
print(f" - Memory footprint before loading Django: {resource.getrusage(resource.RUSAGE_SELF)[2] / 1024.0:.3f} MB")
|
|
try:
|
|
django.setup()
|
|
except:
|
|
print(" ! Cyclic reference failure. Can occur when the initial db is empty. Fixed now (in UploadFileForm) but easy to reintroduce..")
|
|
raise
|
|
print(f" - Memory footprint after loading Django: {resource.getrusage(resource.RUSAGE_SELF)[2] / 1024.0:.3f} MB")
|
|
|
|
from django.contrib.auth.models import User
|
|
from django.core import management
|
|
from django.db import (close_old_connections, connection, connections,
|
|
transaction)
|
|
from django.http import HttpResponse
|
|
from django.urls import reverse
|
|
|
|
import troggle.core.models.survex
|
|
from troggle.core.models.caves import Cave, Entrance
|
|
from troggle.core.models.troggle import DataIssue
|
|
from troggle.core.utils import get_process_memory
|
|
from troggle.parsers.imports import (import_caves, import_drawingsfiles,
|
|
import_ents, import_loadpos,
|
|
import_logbook, import_logbooks,
|
|
import_people, import_QMs, import_survex,
|
|
import_surveyscans)
|
|
|
|
if os.geteuid() == 0:
|
|
# This protects the server from having the wrong file permissions written on logs and caches
|
|
print("This script should be run as expo not root - quitting")
|
|
exit()
|
|
|
|
expouser=settings.EXPOUSER
|
|
expouserpass=settings.EXPOUSERPASS
|
|
expouseremail=settings.EXPOUSER_EMAIL
|
|
|
|
expoadminuser=settings.EXPOADMINUSER
|
|
expoadminuserpass=settings.EXPOADMINUSERPASS
|
|
expoadminuseremail=settings.EXPOADMINUSER_EMAIL
|
|
|
|
def reinit_db():
|
|
"""Rebuild database from scratch. Deletes the file first if sqlite is used,
|
|
otherwise it drops the database and creates it.
|
|
Note - initial loading of troggle.sqlite will already have populated the models
|
|
in memory (django python models, not the database), so there is already a full load
|
|
of stuff known. Deleting the db file does not clear memory.
|
|
"""
|
|
print("Reinitialising db ",end="")
|
|
print(django.db.connections.databases['default']['NAME'])
|
|
currentdbname = settings.DATABASES['default']['NAME']
|
|
if currentdbname == ':memory:':
|
|
# closing connections should wipe the in-memory database
|
|
django.db.close_old_connections()
|
|
for conn in django.db.connections.all():
|
|
print(" ! Closing another connection to db...")
|
|
conn.close()
|
|
elif django.db.connections.databases['default']['ENGINE'] == 'django.db.backends.sqlite3':
|
|
if os.path.isfile(currentdbname):
|
|
try:
|
|
print(" - deleting " + currentdbname)
|
|
os.remove(currentdbname)
|
|
except OSError:
|
|
print(" ! OSError on removing: " + currentdbname + "\n ! Is the file open in another app? Is the server running?\n")
|
|
raise
|
|
else:
|
|
print(" - No database file found: " + currentdbname + " ..continuing, will create it.\n")
|
|
else:
|
|
print(f" - Attempting to nuke : {currentdbname}\n")
|
|
# this is now completely failing to nuke MariaDB adequately, and it crashes when creating Area objects with a no null parent message
|
|
# when null parents are explciitly allowed in the model.
|
|
cursor = django.db.connection.cursor()
|
|
cursor.execute(f"DROP DATABASE {currentdbname}")
|
|
cursor.execute(f"CREATE DATABASE {currentdbname}")
|
|
cursor.execute(f"ALTER DATABASE {currentdbname} CHARACTER SET=utf8")
|
|
cursor.execute(f"USE {currentdbname}")
|
|
print(f" - Nuked : {currentdbname}\n")
|
|
|
|
print(" - Migrating: " + django.db.connections.databases['default']['NAME'])
|
|
|
|
if django.db.connections.databases['default']['ENGINE'] == 'django.db.backends.sqlite3':
|
|
#with transaction.atomic():
|
|
management.call_command('makemigrations','core', interactive=False)
|
|
management.call_command('migrate', interactive=False)
|
|
management.call_command('migrate','core', interactive=False)
|
|
else:
|
|
management.call_command('makemigrations','core', interactive=False)
|
|
management.call_command('migrate', interactive=False)
|
|
management.call_command('migrate','core', interactive=False)
|
|
|
|
|
|
print(" - done migration on: " + settings.DATABASES['default']['NAME'])
|
|
print("users in db already: ",len(User.objects.all()))
|
|
with transaction.atomic():
|
|
try:
|
|
print(" - Setting up expo user on: " + django.db.connections.databases['default']['NAME'])
|
|
print(f" - user: {expouser} ({expouserpass:.5}...) <{expouseremail}> ")
|
|
user = User.objects.create_user(expouser, expouseremail, expouserpass)
|
|
user.is_staff = False
|
|
user.is_superuser = False
|
|
user.save()
|
|
except:
|
|
print(" ! INTEGRITY ERROR user on: " + settings.DATABASES['default']['NAME'])
|
|
print(django.db.connections.databases['default']['NAME'])
|
|
print(" ! You probably have not got a clean db when you thought you had.\n")
|
|
print(" ! Also you are probably NOT running an in-memory db now.\n")
|
|
print("users in db: ",len(User.objects.all()))
|
|
print("tables in db: ",len(connection.introspection.table_names()))
|
|
memdumpsql(fn='integrityfail.sql')
|
|
django.db.connections.databases['default']['NAME'] = ':memory:'
|
|
#raise
|
|
|
|
with transaction.atomic():
|
|
try:
|
|
print(" - Setting up expoadmin user on: " + django.db.connections.databases['default']['NAME'])
|
|
print(f" - user: {expoadminuser} ({expoadminuserpass:.5}...) <{expoadminuseremail}> ")
|
|
user = User.objects.create_user(expoadminuser, expoadminuseremail, expoadminuserpass)
|
|
user.is_staff = True
|
|
user.is_superuser = True
|
|
user.save()
|
|
except:
|
|
print(" ! INTEGRITY ERROR user on: " + settings.DATABASES['default']['NAME'])
|
|
print(django.db.connections.databases['default']['NAME'])
|
|
print(" ! You probably have not got a clean db when you thought you had.\n")
|
|
print(" ! Also you are probably NOT running an in-memory db now.\n")
|
|
print("users in db: ",len(User.objects.all()))
|
|
print("tables in db: ",len(connection.introspection.table_names()))
|
|
memdumpsql(fn='integrityfail.sql')
|
|
django.db.connections.databases['default']['NAME'] = ':memory:'
|
|
#raise
|
|
|
|
def memdumpsql(fn):
|
|
'''Unused option to dump SQL. Aborted attempt to create a cache for loading data
|
|
'''
|
|
djconn = django.db.connection
|
|
from dump import _iterdump
|
|
with open(fn, 'w') as f:
|
|
for line in _iterdump(djconn):
|
|
f.write(f"{line.encode('utf8')}\n")
|
|
return True
|
|
|
|
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
|
|
|
class JobQueue():
|
|
"""A list of import operations to run. Always reports profile times
|
|
of the import operations in the same order.
|
|
"""
|
|
|
|
def __init__(self,run):
|
|
'''Initialises the job queue object with a fixed order for reporting
|
|
options during a run. Imports the timings from previous runs.
|
|
'''
|
|
self.runlabel = run
|
|
self.queue = [] # tuples of (jobname, jobfunction)
|
|
self.results = {}
|
|
self.results_order=[
|
|
"date","runlabel","reinit", "caves", "people",
|
|
"logbooks", "QMs", "scans", "survex",
|
|
"drawings", "test" ]
|
|
for k in self.results_order:
|
|
self.results[k]=[]
|
|
self.tfile = "import_profile.json"
|
|
self.htmlfile = "profile.html" # for HTML results table. Not yet done.
|
|
|
|
|
|
def enq(self,label,func):
|
|
'''Enqueue: Adding elements to queue
|
|
'''
|
|
self.queue.append((label,func))
|
|
return True
|
|
|
|
def loadprofiles(self):
|
|
"""Load timings for previous imports for each data import type
|
|
"""
|
|
if os.path.isfile(self.tfile):
|
|
try:
|
|
f = open(self.tfile, "r")
|
|
data = json.load(f)
|
|
for j in data:
|
|
self.results[j] = data[j]
|
|
except:
|
|
print(f"FAILURE parsing JSON file {self.tfile}")
|
|
# Python bug: https://github.com/ShinNoNoir/twitterwebsearch/issues/12
|
|
f.close()
|
|
for j in self.results_order:
|
|
self.results[j].append(None) # append a placeholder
|
|
return True
|
|
|
|
def dellastprofile(self):
|
|
"""trim one set of data from the results
|
|
"""
|
|
for j in self.results_order:
|
|
self.results[j].pop() # delete last item
|
|
return True
|
|
|
|
def delfirstprofile(self):
|
|
"""trim one set of data from the results
|
|
"""
|
|
for j in self.results_order:
|
|
self.results[j].pop(0) # delete zeroth item
|
|
return True
|
|
|
|
def saveprofiles(self):
|
|
"""Save timings for the set of imports just completed
|
|
"""
|
|
with open(self.tfile, 'w') as f:
|
|
json.dump(self.results, f)
|
|
return True
|
|
|
|
def runqonce(self):
|
|
"""Run all the jobs in the queue provided - once
|
|
"""
|
|
print("** Running job ", self.runlabel,end=" to ")
|
|
print(django.db.connections.databases['default']['NAME'])
|
|
jobstart = time.time()
|
|
print(f"-- Initial memory in use {get_process_memory():.3f} MB")
|
|
self.results["date"].pop()
|
|
self.results["date"].append(jobstart)
|
|
self.results["runlabel"].pop()
|
|
self.results["runlabel"].append(self.runlabel)
|
|
|
|
for runfunction in self.queue:
|
|
start = time.time()
|
|
memstart = get_process_memory()
|
|
jobname, jobparser = runfunction
|
|
#--------------------
|
|
jobparser() # invokes function passed in the second item in the tuple
|
|
#--------------------
|
|
memend = get_process_memory()
|
|
duration = time.time()-start
|
|
#print(" - MEMORY start:{:.3f} MB end:{:.3f} MB change={:.3f} MB".format(memstart,memend, ))
|
|
print("\n*- Ended \"", jobname, f"\" {duration:.1f} seconds + {memend - memstart:.3f} MB ({memend:.3f} MB)")
|
|
self.results[jobname].pop() # the null item
|
|
self.results[jobname].append(duration)
|
|
|
|
|
|
jobend = time.time()
|
|
jobduration = jobend-jobstart
|
|
print(f"** Ended job {self.runlabel} - {jobduration:.1f} seconds total.")
|
|
return True
|
|
|
|
|
|
def append_placeholders(self):
|
|
'''Ads a dummy timing for each option, to fix off by one error
|
|
'''
|
|
for j in self.results_order:
|
|
self.results[j].append(None) # append a placeholder
|
|
|
|
def run_now_django_tests(self,n):
|
|
"""Runs the standard django test harness system which is in troggle/core/TESTS/tests.py
|
|
"""
|
|
management.call_command('test', verbosity=n)
|
|
django.db.close_old_connections()
|
|
|
|
def run(self):
|
|
"""Initialises profile timings record, initiates relational database, runs the job queue saving the imported data as an SQL image and saves the timing profile data.
|
|
"""
|
|
self.loadprofiles()
|
|
print("-- start ", django.db.connections.databases['default']['ENGINE'], django.db.connections.databases['default']['NAME'])
|
|
self.runqonce()
|
|
if settings.DATABASES['default']['NAME'] ==":memory:":
|
|
memdumpsql('memdump.sql') # saved contents of in-memory db, could be imported later..
|
|
self.saveprofiles()
|
|
return True
|
|
|
|
def showprofile(self):
|
|
"""Prints out the time it took to run the jobqueue
|
|
"""
|
|
for k in self.results_order:
|
|
if k =="test":
|
|
break
|
|
elif k =="date":
|
|
print(" days ago ", end=' ')
|
|
else:
|
|
print('%10s (s)' % k, end=' ')
|
|
percen=0
|
|
r = self.results[k]
|
|
|
|
for i in range(len(r)):
|
|
if k == "runlabel":
|
|
if r[i]:
|
|
rp = r[i]
|
|
else:
|
|
rp = " - "
|
|
print('%8s' % rp, end=' ')
|
|
elif k =="date":
|
|
# Calculate dates as days before present
|
|
if r[i]:
|
|
if i == len(r)-1:
|
|
print(" this", end=' ')
|
|
else:
|
|
# prints one place to the left of where you expect
|
|
if r[len(r)-1]:
|
|
s = r[i]-r[len(r)-1]
|
|
elif r[len(r)-2]:
|
|
s = r[i]-r[len(r)-2]
|
|
else:
|
|
s = 0
|
|
days = (s)/(24*60*60)
|
|
print(f'{days:8.2f}', end=' ')
|
|
elif r[i]:
|
|
print(f'{r[i]:8.1f}', end=' ')
|
|
if i == len(r)-1 and r[i-1]:
|
|
percen = 100* (r[i] - r[i-1])/r[i-1]
|
|
if abs(percen) >0.1:
|
|
print(f'{percen:8.1f}%', end=' ')
|
|
else:
|
|
print(" - ", end=' ')
|
|
print("")
|
|
print("\n")
|
|
return True
|
|
|
|
|
|
def usage():
|
|
'''Prints command line options, can print history of previous runs with timings
|
|
'''
|
|
print("""Usage is 'python databaseReset.py <command> [runlabel]'
|
|
where command is:
|
|
test - testing... imports people and prints profile. Deletes nothing.
|
|
profile - print the profile from previous runs. Import nothing.
|
|
- del - deletes last entry
|
|
- delfirst - deletes first entry
|
|
|
|
reset - normal usage: clear database and reread everything from files
|
|
|
|
init - initialisation. Automatic if you run reset.
|
|
caves - read in the caves (must run first after initialisation)
|
|
people - read in the people from folk.csv (must run after 'caves')
|
|
logbooks - read in the logbooks
|
|
QMs - read in the QM csv files (older caves only)
|
|
scans - the survey scans in all the wallets (must run before survex)
|
|
drawings - read in the Tunnel & Therion files - which scans the survey scans too
|
|
survex - read in the survex files - all the survex blocks and entrances x/y/z
|
|
ents - read just the entrances x/y/z (must run after survex)
|
|
|
|
dumplogbooks - Not used. write out autologbooks (not working? use http://localhost:8000/controlpanel )
|
|
logbook - read a single logbook. Defautl set in python code
|
|
|
|
and [runlabel] is an optional string identifying this run of the script
|
|
in the stored profiling data 'import-profile.json'
|
|
|
|
caves and logbooks must be run on an empty db before the others as they
|
|
set up db tables used by the others.
|
|
|
|
Note that running the subfunctions will not produce a consistent website
|
|
- only the full 'reset' does that.
|
|
""")
|
|
|
|
if __name__ == "__main__":
|
|
|
|
if os.geteuid() == 0:
|
|
print("Do not run as root or using sudo - file permissions for cache files and logs will break")
|
|
print("Aborting run.")
|
|
exit()
|
|
|
|
if sys.getfilesystemencoding() != "utf-8":
|
|
print("UTF-8 is NOT the default file encoding. You must fix this.")
|
|
print(f'- {sys.getdefaultencoding()=}')
|
|
print(f'- {sys.getfilesystemencoding()=}')
|
|
print(f'- {locale.getdefaultlocale()=}')
|
|
print(f'- {locale.getpreferredencoding()=}')
|
|
print("Aborting run.")
|
|
exit()
|
|
|
|
if len(sys.argv)>2:
|
|
runlabel = sys.argv[len(sys.argv)-1]
|
|
else:
|
|
runlabel=None
|
|
|
|
jq = JobQueue(runlabel)
|
|
|
|
if len(sys.argv)==1:
|
|
usage()
|
|
exit()
|
|
elif "init" in sys.argv:
|
|
jq.enq("reinit",reinit_db)
|
|
elif "ents" in sys.argv:
|
|
jq.enq("survex",import_ents)
|
|
elif "test2" in sys.argv:
|
|
jq.enq("QMs",import_QMs)
|
|
jq.enq("drawings",import_drawingsfiles)
|
|
jq.enq("survex",import_survex)
|
|
elif "caves" in sys.argv:
|
|
jq.enq("caves",import_caves)
|
|
elif "logbooks" in sys.argv:
|
|
jq.enq("logbooks",import_logbooks)
|
|
elif "logbook" in sys.argv:
|
|
jq.enq("logbooks",import_logbook) # default year set in imports.py
|
|
elif "people" in sys.argv:
|
|
jq.enq("people",import_people)
|
|
elif "QMs" in sys.argv:
|
|
jq.enq("QMs",import_QMs)
|
|
elif "reset" in sys.argv:
|
|
jq.enq("reinit",reinit_db)
|
|
jq.enq("caves",import_caves)
|
|
jq.enq("people",import_people)
|
|
jq.enq("scans",import_surveyscans)
|
|
jq.enq("logbooks",import_logbooks)
|
|
jq.enq("QMs",import_QMs)
|
|
jq.enq("drawings",import_drawingsfiles)
|
|
jq.enq("survex",import_survex)
|
|
elif "scans" in sys.argv:
|
|
jq.enq("scans",import_surveyscans)
|
|
elif "survex" in sys.argv:
|
|
jq.enq("survex",import_survex)
|
|
elif "loadpos" in sys.argv:
|
|
jq.enq("survex",import_loadpos)
|
|
elif "drawings" in sys.argv:
|
|
jq.enq("drawings",import_drawingsfiles)
|
|
elif "dumplogbooks" in sys.argv: # untested in 2020
|
|
dumplogbooks()
|
|
# elif "writecaves" in sys.argv: # untested in 2020 - will overwrite input files!!
|
|
# writeCaves()
|
|
elif "profile" in sys.argv:
|
|
if runlabel == 'del' :
|
|
jq.loadprofiles()
|
|
jq.dellastprofile()
|
|
jq.dellastprofile() # twice because loadprofiles adds a dummy
|
|
jq.showprofile()
|
|
jq.saveprofiles()
|
|
if runlabel == 'delfirst' :
|
|
jq.loadprofiles()
|
|
jq.dellastprofile() # remove the dummy
|
|
jq.delfirstprofile()
|
|
jq.showprofile()
|
|
jq.saveprofiles()
|
|
else:
|
|
jq.loadprofiles()
|
|
jq.showprofile()
|
|
exit()
|
|
elif "help" in sys.argv:
|
|
usage()
|
|
exit()
|
|
else:
|
|
usage()
|
|
print(f"{sys.argv[1]} not recognised as a command.")
|
|
exit()
|
|
|
|
jq.run()
|
|
jq.showprofile()
|