xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Thorough spring clean and profiling

Browse Source
This commit is contained in:
Philip Sargent 2020-04-27 23:51:41 +01:00
parent 2b39dec560
commit a8460065a4
4 changed files with 129 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
databaseReset.py
View File

@ -18,7 +18,10 @@ expouser=settings.EXPOUSER
expouserpass=settings.EXPOUSERPASS expouserpass=settings.EXPOUSERPASS
expouseremail=settings.EXPOUSER_EMAIL expouseremail=settings.EXPOUSER_EMAIL
def reload_db(): def reinit_db():
"""Rebuild database from scratch. Deletes the file first if sqlite is used,
otherwise it drops the database and creates it.
"""
if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
try: try:
os.remove(databasename) os.remove(databasename)
@ -30,26 +33,27 @@ def reload_db():
cursor.execute("CREATE DATABASE %s" % databasename) cursor.execute("CREATE DATABASE %s" % databasename)
cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % databasename) cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % databasename)
cursor.execute("USE %s" % databasename) cursor.execute("USE %s" % databasename)
management.call_command('syncdb', interactive=False) syncuser()
user = User.objects.create_user(expouser, expouseremail, expouserpass)
user.is_staff = True
user.is_superuser = True
user.save()
def syncuser(): def syncuser():
"""Sync user - needed after reload""" """Sync user - needed after reload
"""
print("Synchronizing user")
management.call_command('syncdb', interactive=False) management.call_command('syncdb', interactive=False)
user = User.objects.create_user(expouser, expouseremail, expouserpass) user = User.objects.create_user(expouser, expouseremail, expouserpass)
user.is_staff = True user.is_staff = True
user.is_superuser = True user.is_superuser = True
user.save() user.save()
def dirsredirect():
def make_dirs(): """Make directories that troggle requires and sets up page redirects
"""Make directories that troggle requires""" """
#should also deal with permissions here. #should also deal with permissions here.
if not os.path.isdir(settings.PHOTOS_ROOT): if not os.path.isdir(settings.PHOTOS_ROOT):
os.mkdir(settings.PHOTOS_ROOT) os.mkdir(settings.PHOTOS_ROOT)
for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
f.save()
def import_caves(): def import_caves():
import parsers.caves import parsers.caves
@ -58,38 +62,49 @@ def import_caves():
def import_people(): def import_people():
import parsers.people import parsers.people
print("Importing People (folk.csv)")
parsers.people.LoadPersonsExpos() parsers.people.LoadPersonsExpos()
def import_logbooks(): def import_logbooks():
import parsers.logbooks import parsers.logbooks
print("Importing Logbooks")
parsers.logbooks.LoadLogbooks() parsers.logbooks.LoadLogbooks()
def import_survex():
import parsers.survex
parsers.survex.LoadAllSurvexBlocks()
parsers.survex.LoadPos()
def import_QMs(): def import_QMs():
print("Importing QMs (old caves)")
import parsers.QMs import parsers.QMs
# import process itself runs on qm.csv in only 3 caves, not 264! # import process itself runs on qm.csv in only 3 caves, not 264!
def import_survex():
import parsers.survex
print("Importing Survex Blocks")
parsers.survex.LoadAllSurvexBlocks()
print("Importing Survex Positions")
parsers.survex.LoadPos()
def import_survexpos():
import parsers.survex
print("Importing Survex Positions")
parsers.survex.LoadPos()
def import_surveys(): def import_surveys():
"""This appears to store data in unused objects. The code is kept
for future re-working to manage progress against notes, plans and elevs.
"""
import parsers.surveys import parsers.surveys
print("Importing surveys")
parsers.surveys.parseSurveys(logfile=settings.LOGFILE) parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
def import_surveyscans(): def import_surveyscans():
import parsers.surveys import parsers.surveys
print("Importing Survey Scans")
parsers.surveys.LoadListScans() parsers.surveys.LoadListScans()
def import_tunnelfiles(): def import_tunnelfiles():
import parsers.surveys import parsers.surveys
print("Importing Tunnel files")
parsers.surveys.LoadTunnelFiles() parsers.surveys.LoadTunnelFiles()
def pageredirects():
for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
f.save()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def import_auto_logbooks(): def import_auto_logbooks():
import parsers.logbooks import parsers.logbooks
@ -154,9 +169,9 @@ class JobQueue():
self.queue = [] # tuples of (jobname, jobfunction) self.queue = [] # tuples of (jobname, jobfunction)
self.results = {} self.results = {}
self.results_order=[ self.results_order=[
"date","runlabel","reload", "caves", "people", "date","runlabel","reinit", "caves", "people",
"logbooks", "scans", "QMs", "survex", "logbooks", "scans", "QMs", "survex",
"tunnel", "surveys", "test", "makedirs", "redirect", "syncuser" ] "tunnel", "surveys", "test", "dirsredirect", "syncuser", "survexpos" ]
for k in self.results_order: for k in self.results_order:
self.results[k]=[] self.results[k]=[]
self.tfile = "import_profile.json" self.tfile = "import_profile.json"
@ -207,7 +222,7 @@ class JobQueue():
# currently uses django db whatever it was. CHANGE this to explicitly use # currently uses django db whatever it was. CHANGE this to explicitly use
# a new sqlite3 db and then import the sql dump of that into the troggle db # a new sqlite3 db and then import the sql dump of that into the troggle db
# instead of loading directly into the troggle sqlite db. # instead of loading directly into the troggle sqlite db.
# in-menmor ":memory:" sqlite is ~ 7x faster and all of troggle can be # in-memory ":memory:" sqlite is ~ 7x faster and all of troggle can be
# loaded in 6 minutes that way # loaded in 6 minutes that way
djconn = django.db.connection djconn = django.db.connection
from dump import _iterdump from dump import _iterdump
@ -221,52 +236,76 @@ class JobQueue():
def showprofile(self): def showprofile(self):
"""Prints out the time it took to run the jobqueue""" """Prints out the time it took to run the jobqueue"""
for k in self.results_order: for k in self.results_order:
percen=0 if k =="dirsredirect":
lst = self.results[k] break
if k == "runlabel": elif k =="syncuser":
r = lst[len(lst)-1] break
print '%15s %s' % (k,r) elif k =="test":
break
elif k =="date": elif k =="date":
# Calculate dates as days before present to one decimal place print " days ago ",
r = lst[len(lst)-1]
if len(lst)>2:
days = (lst[len(lst)-2]-r)/(24*60*60)
print '%15s %8.1f days ago' % (k,days)
elif len(lst)>2:
e = len(lst)-1
percen = 100* (lst[e] - lst[e-1])/lst[e-1]
if abs(percen) >0.1:
print '%15s %8.1f%%' % (k, percen)
else: else:
print '%15s ' % (k) print '%9s (s)' % k,
percen=0
r = self.results[k]
#print "min=",min
for i in range(len(r)):
if k == "runlabel":
if r[i]:
rp = r[i]
else:
rp = " - "
print '%8s' % rp,
elif k =="date":
# Calculate dates as days before present
if r[i]:
if i == len(r)-1:
print " this",
else:
# prints one place to the left of where you expect
days = (r[i]-r[len(r)-1])/(24*60*60)
print '%8.2f' % days,
elif r[i]:
print '%8.1f' % r[i],
if i == len(r)-1 and r[i-1]:
percen = 100* (r[i] - r[i-1])/r[i-1]
if abs(percen) >0.1:
print '%8.1f%%' % percen,
else:
print " - ",
print ""
return True return True
def usage(): def usage():
print("""Usage is 'python databaseReset.py <command> [runlabel]' print("""Usage is 'python databaseReset.py <command> [runlabel]'
where command is: where command is:
reset - this is normal usage, clear database and reread everything from files - time-consuming reset - normal usage: clear database and reread everything from files - time-consuming
caves - read in the caves caves - read in the caves
logbooks - read in just the logbooks logbooks - read in the logbooks
people - read in the people from folk.csv people - read in the people from folk.csv
QMs - read in the QM csv files QMs - read in the QM csv files (older caves only)
reload_db - clear database (delete everything) and make empty tables reinit - clear database (delete everything) and make empty tables. Import nothing.
scans - NOT the scanned surveynotes ?! scans - the survey scans in all the wallets
survex - read in the survex files - all the survex blocks survex - read in the survex files - all the survex blocks
surveys - read in the scanned surveynotes
tunnel - read in the Tunnel files - which scans the surveyscans too
survexpos - just the Pos out of the survex files (not part of reset) survexpos - just the Pos out of the survex files (not part of reset)
tunnel - read in the Tunnel files - which scans the survey scans too
resetend - (archaic?) resetend - (archaic?)
writecaves - *disabled* (archaic?) writecaves - *disabled* (archaic?)
autologbooks - read in autologbooks (what are these?) autologbooks - read in autologbooks (what are these?)
dumplogbooks - write out autologbooks (not working?) dumplogbooks - write out autologbooks (not working?)
syncuser - needed after reloading database rom SQL backup syncuser - needed after reloading database from SQL backup
surveys - read in scans by expo, must run after "people". Not used.
test - testing... test - testing...
and [runlabel] is an optional string identifying this run of the script and [runlabel] is an optional string identifying this run of the script
in the stored profiling data 'import-profile.json' in the stored profiling data 'import-profile.json'
caves and logbooks must be run on an empty db before the others as they
set up db tables used by the others.
""") """)
if __name__ == "__main__": if __name__ == "__main__":
@ -275,31 +314,35 @@ if __name__ == "__main__":
import django import django
django.setup() django.setup()
if len(sys.argv)>2:
runlabel = sys.argv[len(sys.argv)-1] runlabel = sys.argv[len(sys.argv)-1]
else:
runlabel=None
jq = JobQueue(runlabel) jq = JobQueue(runlabel)
if "test" in sys.argv: if len(sys.argv)==1:
jq.enq("reload",reload_db) usage()
jq.enq("makedirs",make_dirs) exit()
elif "test" in sys.argv:
jq.enq("reinit",reinit_db)
jq.enq("dirsredirect",dirsredirect)
jq.enq("caves",import_caves) jq.enq("caves",import_caves)
jq.enq("survex",import_survex) #jq.enq("people",import_people)
jq.enq("surveys",import_surveys) #jq.enq("logbooks",import_logbooks)
elif "caves" in sys.argv: elif "caves" in sys.argv:
jq.enq("caves",import_caves) jq.enq("caves",import_caves)
elif "logbooks" in sys.argv: elif "logbooks" in sys.argv:
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
jq.enq("logbooks",import_logbooks) jq.enq("logbooks",import_logbooks)
elif "people" in sys.argv: elif "people" in sys.argv:
jq.enq("logbooks",import_people) jq.enq("people",import_people)
elif "QMs" in sys.argv: elif "QMs" in sys.argv:
jq.enq("QMs",import_QMs) jq.enq("QMs",import_QMs)
elif "reload_db" in sys.argv: elif "reload_db" in sys.argv:
jq.enq("reload",reload_db) jq.enq("reload",reload_db)
elif "reset" in sys.argv: elif "reset" in sys.argv:
jq.enq("reload",reload_db) jq.enq("reinit",reinit_db)
jq.enq("makedirs",make_dirs) jq.enq("dirsredirect",dirsredirect)
jq.enq("redirect",pageredirects)
jq.enq("caves",import_caves) jq.enq("caves",import_caves)
jq.enq("people",import_people) jq.enq("people",import_people)
jq.enq("scans",import_surveyscans) jq.enq("scans",import_surveyscans)
@ -307,16 +350,12 @@ if __name__ == "__main__":
jq.enq("QMs",import_QMs) jq.enq("QMs",import_QMs)
jq.enq("survex",import_survex) jq.enq("survex",import_survex)
jq.enq("tunnel",import_tunnelfiles) jq.enq("tunnel",import_tunnelfiles)
jq.enq("surveys",import_surveys)
elif "scans" in sys.argv: elif "scans" in sys.argv:
jq.enq("scans",import_surveyscans) jq.enq("scans",import_surveyscans)
elif "survex" in sys.argv: elif "survex" in sys.argv:
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
jq.enq("survex",import_survex) jq.enq("survex",import_survex)
elif "survexpos" in sys.argv: elif "survexpos" in sys.argv:
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex jq.enq("survexpos",import_survexpos)
import parsers.survex
jq.enq("survexpos",parsers.survex.LoadPos)
elif "surveys" in sys.argv: elif "surveys" in sys.argv:
jq.enq("surveys",import_surveys) jq.enq("surveys",import_surveys)
elif "tunnel" in sys.argv: elif "tunnel" in sys.argv:
@ -336,8 +375,9 @@ if __name__ == "__main__":
elif "dumplogbooks" in sys.argv: elif "dumplogbooks" in sys.argv:
dumplogbooks() dumplogbooks()
else: else:
print("%s not recognised" % sys.argv)
usage() usage()
print("%s not recognised as a command." % sys.argv[1])
exit()
jq.run() jq.run()
jq.showprofile() jq.showprofile()

8
parsers/caves.py
View File

@ -152,7 +152,7 @@ def readcave(filename):
slug = slug, slug = slug,
primary = primary) primary = primary)
except: except:
message = "Can't find text (slug): %s, skipping %s" % (slug, context) message = " ! Can't find text (slug): %s, skipping %s" % (slug, context)
models.DataIssue.objects.create(parser='caves', message=message) models.DataIssue.objects.create(parser='caves', message=message)
print(message) print(message)
@ -164,7 +164,7 @@ def readcave(filename):
entrance = models.Entrance.objects.get(entranceslug__slug = slug) entrance = models.Entrance.objects.get(entranceslug__slug = slug)
ce = models.CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance) ce = models.CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance)
except: except:
message = "Entrance text (slug) %s missing %s" % (slug, context) message = " ! Entrance text (slug) %s missing %s" % (slug, context)
models.DataIssue.objects.create(parser='caves', message=message) models.DataIssue.objects.create(parser='caves', message=message)
print(message) print(message)
@ -172,14 +172,14 @@ def readcave(filename):
def getXML(text, itemname, minItems = 1, maxItems = None, printwarnings = True, context = ""): def getXML(text, itemname, minItems = 1, maxItems = None, printwarnings = True, context = ""):
items = re.findall("<%(itemname)s>(.*?)</%(itemname)s>" % {"itemname": itemname}, text, re.S) items = re.findall("<%(itemname)s>(.*?)</%(itemname)s>" % {"itemname": itemname}, text, re.S)
if len(items) < minItems and printwarnings: if len(items) < minItems and printwarnings:
message = "%(count)i %(itemname)s found, at least %(min)i expected" % {"count": len(items), message = " ! %(count)i %(itemname)s found, at least %(min)i expected" % {"count": len(items),
"itemname": itemname, "itemname": itemname,
"min": minItems} + context "min": minItems} + context
models.DataIssue.objects.create(parser='caves', message=message) models.DataIssue.objects.create(parser='caves', message=message)
print(message) print(message)
if maxItems is not None and len(items) > maxItems and printwarnings: if maxItems is not None and len(items) > maxItems and printwarnings:
message = "%(count)i %(itemname)s found, no more than %(max)i expected" % {"count": len(items), message = " ! %(count)i %(itemname)s found, no more than %(max)i expected" % {"count": len(items),
"itemname": itemname, "itemname": itemname,
"max": maxItems} + context "max": maxItems} + context
models.DataIssue.objects.create(parser='caves', message=message) models.DataIssue.objects.create(parser='caves', message=message)

4
parsers/people.py
View File

@ -58,7 +58,7 @@ def LoadPersonsExpos():
header = dict(zip(headers, range(len(headers)))) header = dict(zip(headers, range(len(headers))))
# make expeditions # make expeditions
print("Loading expeditions") print(" - Loading expeditions")
years = headers[5:] years = headers[5:]
for year in years: for year in years:
@ -68,7 +68,7 @@ def LoadPersonsExpos():
save_carefully(models.Expedition, lookupAttribs, nonLookupAttribs) save_carefully(models.Expedition, lookupAttribs, nonLookupAttribs)
# make persons # make persons
print("Loading personexpeditions") print(" - Loading personexpeditions")
for personline in personreader: for personline in personreader:
name = personline[header["Name"]] name = personline[header["Name"]]

14
parsers/surveys.py
View File

@ -150,17 +150,19 @@ def parseSurveyScans(expedition, logfile=None):
scanObj.save() scanObj.save()
except (IOError, OSError): except (IOError, OSError):
yearPath=os.path.join(settings.SURVEY_SCANS, "surveyscans", expedition.year) yearPath=os.path.join(settings.SURVEY_SCANS, "surveyscans", expedition.year)
print("No folder found for " + expedition.year + " at:- " + yearPath) print(" ! No folder found for " + expedition.year + " at:- " + yearPath)
# dead # dead
def parseSurveys(logfile=None): def parseSurveys(logfile=None):
try: try:
readSurveysFromCSV() readSurveysFromCSV()
except (IOError, OSError): except (IOError, OSError):
print("Survey CSV not found..") print(" ! Survey CSV not found..")
pass pass
print " - Loading scans by expedition year"
for expedition in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then for expedition in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then
print "%s" % expedition,
parseSurveyScans(expedition) parseSurveyScans(expedition)
# dead # dead
@ -190,7 +192,7 @@ def GetListDir(sdir):
def LoadListScansFile(survexscansfolder): def LoadListScansFile(survexscansfolder):
gld = [ ] gld = [ ]
# flatten out any directories in these book files # flatten out any directories in these wallet folders - should not be any
for (fyf, ffyf, fisdiryf) in GetListDir(survexscansfolder.fpath): for (fyf, ffyf, fisdiryf) in GetListDir(survexscansfolder.fpath):
if fisdiryf: if fisdiryf:
gld.extend(GetListDir(ffyf)) gld.extend(GetListDir(ffyf))
@ -199,7 +201,7 @@ def LoadListScansFile(survexscansfolder):
for (fyf, ffyf, fisdiryf) in gld: for (fyf, ffyf, fisdiryf) in gld:
#assert not fisdiryf, ffyf #assert not fisdiryf, ffyf
if re.search(r"\.(?:png|jpg|jpeg)(?i)$", fyf): if re.search(r"\.(?:png|jpg|jpeg|pdf|jpeg|svg)(?i)$", fyf):
survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder) survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)
survexscansingle.save() survexscansingle.save()
@ -208,7 +210,7 @@ def LoadListScansFile(survexscansfolder):
# and builds up the models we can access later # and builds up the models we can access later
def LoadListScans(): def LoadListScans():
print('Loading Survey Scans...') print(' - Loading Survey Scans... (deleting all objects first)')
SurvexScanSingle.objects.all().delete() SurvexScanSingle.objects.all().delete()
SurvexScansFolder.objects.all().delete() SurvexScansFolder.objects.all().delete()
@ -221,12 +223,14 @@ def LoadListScans():
# iterate into the surveyscans directory # iterate into the surveyscans directory
print ' - ',
for f, ff, fisdir in GetListDir(os.path.join(settings.SURVEY_SCANS, "surveyscans")): for f, ff, fisdir in GetListDir(os.path.join(settings.SURVEY_SCANS, "surveyscans")):
if not fisdir: if not fisdir:
continue continue
# do the year folders # do the year folders
if re.match(r"\d\d\d\d$", f): if re.match(r"\d\d\d\d$", f):
print "%s" % f,
for fy, ffy, fisdiry in GetListDir(ff): for fy, ffy, fisdiry in GetListDir(ff):
if fisdiry: if fisdiry:
assert fisdiry, ffy assert fisdiry, ffy