forked from expo/troggle
768ec83037
Big overhaul of people processing, fullname added to the model lastname is now names -1 unless you only have one (yes you Wookey) this allows for Jon Arne Toft and Wookey to live it the same DB names can now have html chars in them, this should be real unicode but that can only happen when we go to Python 3!
246 lines
8.1 KiB
Python
246 lines
8.1 KiB
Python
import os
|
|
import time
|
|
import settings
|
|
os.environ['PYTHONPATH'] = settings.PYTHON_PATH
|
|
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
|
|
from django.core import management
|
|
from django.db import connection
|
|
from django.contrib.auth.models import User
|
|
from django.http import HttpResponse
|
|
from django.core.urlresolvers import reverse
|
|
from troggle.core.models import Cave, Entrance
|
|
import troggle.flatpages.models
|
|
|
|
databasename=settings.DATABASES['default']['NAME']
|
|
expouser=settings.EXPOUSER
|
|
expouserpass=settings.EXPOUSERPASS
|
|
expouseremail=settings.EXPOUSER_EMAIL
|
|
|
|
def reload_db():
|
|
if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
|
|
try:
|
|
os.remove(databasename)
|
|
except OSError:
|
|
pass
|
|
else:
|
|
cursor = connection.cursor()
|
|
cursor.execute("DROP DATABASE %s" % databasename)
|
|
cursor.execute("CREATE DATABASE %s" % databasename)
|
|
cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % databasename)
|
|
cursor.execute("USE %s" % databasename)
|
|
management.call_command('syncdb', interactive=False)
|
|
user = User.objects.create_user(expouser, expouseremail, expouserpass)
|
|
user.is_staff = True
|
|
user.is_superuser = True
|
|
user.save()
|
|
|
|
def make_dirs():
|
|
"""Make directories that troggle requires"""
|
|
#should also deal with permissions here.
|
|
if not os.path.isdir(settings.PHOTOS_ROOT):
|
|
os.mkdir(settings.PHOTOS_ROOT)
|
|
|
|
def import_caves():
|
|
import parsers.caves
|
|
print("Importing Caves")
|
|
parsers.caves.readcaves()
|
|
|
|
def import_people():
|
|
import parsers.people
|
|
parsers.people.LoadPersonsExpos()
|
|
|
|
def import_logbooks():
|
|
# The below line was causing errors I didn't understand (it said LOGFILE was a string), and I couldn't be bothered to figure
|
|
# what was going on so I just catch the error with a try. - AC 21 May
|
|
try:
|
|
settings.LOGFILE.write('\nBegun importing logbooks at ' + time.asctime() +'\n'+'-'*60)
|
|
except:
|
|
pass
|
|
|
|
import parsers.logbooks
|
|
parsers.logbooks.LoadLogbooks()
|
|
|
|
def import_survex():
|
|
import parsers.survex
|
|
parsers.survex.LoadAllSurvexBlocks()
|
|
parsers.survex.LoadPos()
|
|
|
|
def import_QMs():
|
|
import parsers.QMs
|
|
|
|
def import_surveys():
|
|
import parsers.surveys
|
|
parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
|
|
|
|
def import_surveyscans():
|
|
import parsers.surveys
|
|
parsers.surveys.LoadListScans()
|
|
|
|
def import_tunnelfiles():
|
|
import parsers.surveys
|
|
parsers.surveys.LoadTunnelFiles()
|
|
|
|
|
|
def reset():
|
|
""" Wipe the troggle database and import everything from legacy data
|
|
"""
|
|
reload_db()
|
|
make_dirs()
|
|
pageredirects()
|
|
import_caves()
|
|
import_people()
|
|
import_surveyscans()
|
|
import_survex()
|
|
import_logbooks()
|
|
import_QMs()
|
|
try:
|
|
import_tunnelfiles()
|
|
except:
|
|
print("Tunnel files parser broken.")
|
|
|
|
import_surveys()
|
|
|
|
|
|
def import_auto_logbooks():
|
|
import parsers.logbooks
|
|
import os
|
|
for pt in troggle.core.models.PersonTrip.objects.all():
|
|
pt.delete()
|
|
for lbe in troggle.core.models.LogbookEntry.objects.all():
|
|
lbe.delete()
|
|
for expedition in troggle.core.models.Expedition.objects.all():
|
|
directory = os.path.join(settings.EXPOWEB,
|
|
"years",
|
|
expedition.year,
|
|
"autologbook")
|
|
for root, dirs, filenames in os.walk(directory):
|
|
for filename in filenames:
|
|
print(os.path.join(root, filename))
|
|
parsers.logbooks.parseAutoLogBookEntry(os.path.join(root, filename))
|
|
|
|
#Temporary function until definative source of data transfered.
|
|
from django.template.defaultfilters import slugify
|
|
from django.template import Context, loader
|
|
def dumplogbooks():
|
|
def get_name(pe):
|
|
if pe.nickname:
|
|
return pe.nickname
|
|
else:
|
|
return pe.person.first_name
|
|
for lbe in troggle.core.models.LogbookEntry.objects.all():
|
|
dateStr = lbe.date.strftime("%Y-%m-%d")
|
|
directory = os.path.join(settings.EXPOWEB,
|
|
"years",
|
|
lbe.expedition.year,
|
|
"autologbook")
|
|
if not os.path.isdir(directory):
|
|
os.mkdir(directory)
|
|
filename = os.path.join(directory,
|
|
dateStr + "." + slugify(lbe.title)[:50] + ".html")
|
|
if lbe.cave:
|
|
print(lbe.cave.reference())
|
|
trip = {"title": lbe.title, "html":lbe.text, "cave": lbe.cave.reference(), "caveOrLocation": "cave"}
|
|
else:
|
|
trip = {"title": lbe.title, "html":lbe.text, "location":lbe.place, "caveOrLocation": "location"}
|
|
pts = [pt for pt in lbe.persontrip_set.all() if pt.personexpedition]
|
|
persons = [{"name": get_name(pt.personexpedition), "TU": pt.time_underground, "author": pt.is_logbook_entry_author} for pt in pts]
|
|
f = open(filename, "wb")
|
|
template = loader.get_template('dataformat/logbookentry.html')
|
|
context = Context({'trip': trip,
|
|
'persons': persons,
|
|
'date': dateStr,
|
|
'expeditionyear': lbe.expedition.year})
|
|
output = template.render(context)
|
|
f.write(unicode(output).encode( "utf-8" ))
|
|
f.close()
|
|
|
|
def pageredirects():
|
|
for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
|
|
f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
|
|
f.save()
|
|
|
|
def writeCaves():
|
|
for cave in Cave.objects.all():
|
|
cave.writeDataFile()
|
|
for entrance in Entrance.objects.all():
|
|
entrance.writeDataFile()
|
|
|
|
def usage():
|
|
print("""Usage is 'python databaseReset.py <command>'
|
|
where command is:
|
|
reset - this is normal usage, clear database and reread everything
|
|
desc
|
|
caves - read in the caves
|
|
logbooks - read in the logbooks
|
|
autologbooks
|
|
dumplogbooks
|
|
people
|
|
QMs - read in the QM files
|
|
resetend
|
|
scans - read in the scanned surveynotes
|
|
survex - read in the survex files
|
|
survexpos
|
|
surveys
|
|
tunnel - read in the Tunnel files
|
|
writeCaves
|
|
""")
|
|
|
|
if __name__ == "__main__":
|
|
import troggle.core.models
|
|
import sys
|
|
import django
|
|
django.setup()
|
|
if "desc" in sys.argv:
|
|
resetdesc()
|
|
elif "scans" in sys.argv:
|
|
import_surveyscans()
|
|
elif "caves" in sys.argv:
|
|
import_caves()
|
|
elif "people" in sys.argv:
|
|
import_people()
|
|
elif "QMs" in sys.argv:
|
|
import_QMs()
|
|
elif "tunnel" in sys.argv:
|
|
import_tunnelfiles()
|
|
elif "reset" in sys.argv:
|
|
reset()
|
|
elif "resetend" in sys.argv:
|
|
#import_logbooks()
|
|
import_QMs()
|
|
try:
|
|
import_tunnelfiles()
|
|
except:
|
|
print("Tunnel files parser broken.")
|
|
import_surveys()
|
|
import_descriptions()
|
|
parse_descriptions()
|
|
elif "survex" in sys.argv:
|
|
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
|
|
import_survex()
|
|
elif "survexpos" in sys.argv:
|
|
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
|
|
import parsers.survex
|
|
parsers.survex.LoadPos()
|
|
elif "logbooks" in sys.argv:
|
|
# management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
|
|
import_logbooks()
|
|
elif "autologbooks" in sys.argv:
|
|
import_auto_logbooks()
|
|
elif "dumplogbooks" in sys.argv:
|
|
dumplogbooks()
|
|
elif "writeCaves" in sys.argv:
|
|
writeCaves()
|
|
elif "surveys" in sys.argv:
|
|
import_surveys()
|
|
elif "help" in sys.argv:
|
|
usage()
|
|
elif "reload_db" in sys.argv:
|
|
reload_db()
|
|
else:
|
|
print("%s not recognised" % sys.argv)
|
|
usage()
|
|
|
|
|
|
|
|
|