troggle/databaseReset.py

import os
import time
import settings
os.environ['PYTHONPATH'] = settings.PYTHON_PATH
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
from django.core import management
from django.db import connection
from django.contrib.auth.models import User
from django.http import HttpResponse
from  django.core.urlresolvers import reverse
from core.models import Cave, Entrance
import flatpages.models

databasename=settings.DATABASES['default']['NAME']

def reload_db():
    if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
        try:
            os.remove(databasename)
        except OSError:
            pass
    else: 
        cursor = connection.cursor()
        cursor.execute("DROP DATABASE %s" % databasename)
        cursor.execute("CREATE DATABASE %s" % databasename)
        cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % databasename)
        cursor.execute("USE %s" % databasename)
    management.call_command('syncdb', interactive=False)
    user = User.objects.create_user('expo', 'wookey@wookware.org', 'putrealpasswordinhere')
    user.is_staff = True
    user.is_superuser = True
    user.save()

def make_dirs():
    """Make directories that troggle requires"""
    #should also deal with permissions here.
    if not os.path.isdir(settings.PHOTOS_ROOT):
        os.mkdir(settings.PHOTOS_ROOT)

def import_caves():
    import parsers.caves
    print "importing caves"
    parsers.caves.readcaves()

def import_people():
    import parsers.people
    parsers.people.LoadPersonsExpos()

def import_logbooks():
    # The below line was causing errors I didn't understand (it said LOGFILE was a string), and I couldn't be bothered to figure
    # what was going on so I just catch the error with a try. - AC 21 May
    try:
        settings.LOGFILE.write('\nBegun importing logbooks at ' + time.asctime() +'\n'+'-'*60)
    except:
        pass
    
    import parsers.logbooks
    parsers.logbooks.LoadLogbooks()

def import_survex():
    import parsers.survex
    parsers.survex.LoadAllSurvexBlocks()
    parsers.survex.LoadPos()

def import_QMs():
    import parsers.QMs

def import_surveys():
    import parsers.surveys
    parsers.surveys.parseSurveys(logfile=settings.LOGFILE)

def import_surveyscans():
    import parsers.surveys
    parsers.surveys.LoadListScans()

def import_tunnelfiles():
    import parsers.surveys
    parsers.surveys.LoadTunnelFiles()


def reset():
    """ Wipe the troggle database and import everything from legacy data
    """
    reload_db()
    make_dirs()
    pageredirects()
    import_caves()
    import_people()
    import_surveyscans()
    import_survex()
    import_logbooks()
    import_QMs()
    try:
        import_tunnelfiles()
    except:
        print "Tunnel files parser broken."
    
    import_surveys()


def import_auto_logbooks():
    import parsers.logbooks
    import os
    for pt in core.models.PersonTrip.objects.all():
        pt.delete()
    for lbe in core.models.LogbookEntry.objects.all():
        lbe.delete()
    for expedition in core.models.Expedition.objects.all():
        directory = os.path.join(settings.EXPOWEB, 
                                 "years", 
                                 expedition.year, 
                                 "autologbook")       
        for root, dirs, filenames in os.walk(directory):
            for filename in filenames:
                print os.path.join(root, filename)
                parsers.logbooks.parseAutoLogBookEntry(os.path.join(root, filename))

#Temporary function until definative source of data transfered.
from django.template.defaultfilters import slugify
from django.template import Context, loader
def dumplogbooks():
    def get_name(pe):
            if pe.nickname:
                return pe.nickname
            else:
                return pe.person.first_name
    for lbe in core.models.LogbookEntry.objects.all():
            dateStr = lbe.date.strftime("%Y-%m-%d")
            directory = os.path.join(settings.EXPOWEB, 
                                "years", 
                                lbe.expedition.year, 
                                "autologbook")
            if not os.path.isdir(directory):
                os.mkdir(directory)
            filename = os.path.join(directory, 
                                    dateStr + "." + slugify(lbe.title)[:50] + ".html")
            if lbe.cave:
                print lbe.cave.reference()
                trip = {"title": lbe.title, "html":lbe.text, "cave": lbe.cave.reference(), "caveOrLocation": "cave"}
            else:
                trip = {"title": lbe.title, "html":lbe.text, "location":lbe.place, "caveOrLocation": "location"}
            pts = [pt for pt in lbe.persontrip_set.all() if pt.personexpedition]
            persons = [{"name": get_name(pt.personexpedition), "TU": pt.time_underground, "author": pt.is_logbook_entry_author} for pt in pts]
            f = open(filename, "wb")
            template = loader.get_template('dataformat/logbookentry.html')
            context = Context({'trip': trip, 
                               'persons': persons,
                               'date': dateStr,
                               'expeditionyear': lbe.expedition.year})
            output = template.render(context)
            f.write(unicode(output).encode( "utf-8" ))
            f.close()

def pageredirects():
    for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
        f = flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
        f.save()

def writeCaves():
    for cave in Cave.objects.all():
        cave.writeDataFile()
    for entrance in Entrance.objects.all():
        entrance.writeDataFile()

def usage():
    print """Usage is 'python databaseReset.py <command>'
             where command is:
             reset - this is normal usage, clear database and reread everything
             desc
             caves - read in the caves
             logbooks - read in the logbooks
             autologbooks
             dumplogbooks
             people
             QMs      - read in the QM files
             resetend
             scans    - read in the scanned surveynotes
             survex   - read in the survex files
             survexpos
             tunnel   - read in the Tunnel files
             writeCaves
             """ 

if __name__ == "__main__":
    import core.models
    import sys
    if "desc" in sys.argv:
        resetdesc()
    elif "scans" in sys.argv:
        import_surveyscans()
    elif "caves" in sys.argv:
        reload_db()
        make_dirs()
        pageredirects()
        import_caves()
    elif "people" in sys.argv:
        import_people()
    elif "QMs" in sys.argv:
        import_QMs()
    elif "tunnel" in sys.argv:
        import_tunnelfiles()
    elif "reset" in sys.argv:
        reset()
    elif "resetend" in sys.argv:
        #import_logbooks()
        import_QMs()
        try:
            import_tunnelfiles()
        except:
            print "Tunnel files parser broken."
        import_surveys()
        import_descriptions()
        parse_descriptions()
    elif "survex" in sys.argv:
        management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
        import_survex()
    elif "survexpos" in sys.argv:
        management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
        import parsers.survex
        parsers.survex.LoadPos()    
    elif "logbooks" in sys.argv:
        management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
        import_logbooks()
    elif "autologbooks" in sys.argv:
        import_auto_logbooks()
    elif "dumplogbooks" in sys.argv:
        dumplogbooks()
    elif "writeCaves" in sys.argv:
        writeCaves()
    elif "help" in sys.argv:
        usage()
    else:
        print "%s not recognised" % sys.argv
        usage()