troggle/parsers/QMs.py

import csv
import os
import re
from datetime import *

from django.conf import settings

from troggle.core.models import DataIssue
from troggle.core.models_caves import QM, Cave, LogbookEntry
from utils import save_carefully


def deleteQMs():
    QM.objects.all().delete()

def parseCaveQMs(cave,inputFile):
    """Runs through the CSV file at inputFile (which is a relative path from expoweb) and saves each QM as a QM instance."""

    if cave=='204-steinBH':
        try:
            steinBr=Cave.objects.get(official_name="Steinbrückenhöhle")
        except Cave.DoesNotExist:
            print("Steinbruckenhoehle is not in the database. Please run parsers.")
            return
    elif cave=='234-Hauch':
        try:
            hauchHl=Cave.objects.get(official_name="Hauchhöhle")
        except Cave.DoesNotExist:
            print("Hauchhoehle is not in the database. Please run parsers.")
            return
    elif cave =='161-KH':
        try:
            kh=Cave.objects.get(official_name="Kaninchenhöhle")
        except Cave.DoesNotExist:
            print("KH is not in the database. Please run parsers.")
        parse_KH_QMs(kh, inputFile=inputFile) 
        return

    qmPath = settings.EXPOWEB+inputFile
    qmCSVContents = open(qmPath,'rU')
    dialect=csv.Sniffer().sniff(qmCSVContents.read())
    qmCSVContents.seek(0,0)
    qmReader = csv.reader(qmCSVContents,dialect=dialect)
    next(qmReader) # Skip header row
    for line in qmReader:
        try:
            year=int(line[0][1:5])
            #check if placeholder exists for given year, create it if not
            message = " ! - "+ str(year) + " logbook: placeholder entry for '" + cave + "' created. Should be re-attached to the actual trip."
            if cave=='204-steinBH':
                placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="204", title="placeholder for QMs in 204", text=message, defaults={"date": date(year, 1, 1),"cave_slug":str(steinBr)})
            elif cave=='234-Hauch':
                placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="234", title="placeholder for QMs in 234", text=message, defaults={"date": date(year, 1, 1),"cave_slug":str(hauchHl)})
            if hadToCreate:
                print(message)
                DataIssue.objects.create(parser='QMs', message=message)
            QMnum=re.match(r".*?-\d*?-X?(?P<numb>\d*)",line[0]).group("numb")
            newQM = QM()
            newQM.found_by=placeholder
            newQM.number=QMnum
            if line[1]=="Dig":
                newQM.grade="D"
            else:
                newQM.grade=line[1]
            newQM.area=line[2]
            newQM.location_description=line[3]
            
            newQM.completion_description=line[4]
            newQM.nearest_station_description=line[5]
            if newQM.completion_description:  # Troggle checks if QMs are completed by checking if they have a ticked_off_by trip. In the table, completion is indicated by the presence of a completion discription.
                newQM.ticked_off_by=placeholder

            newQM.comment=line[6]
            try:
                preexistingQM=QM.objects.get(number=QMnum, found_by__date__year=year)  #if we don't have this one in the DB, save it
                if preexistingQM.new_since_parsing==False:  #if the pre-existing QM has not been modified, overwrite it
                    preexistingQM.delete()
                    newQM.save()
                    #print((" - overwriting " + str(preexistingQM) +"\r"))
                else:  # otherwise, print that it was ignored
                    print((" - preserving " + str(preexistingQM) + ", which was edited in admin \r"))
                    
            except QM.DoesNotExist:         #if there is no pre-existing QM, save the new one
                newQM.save() 
                # print("QM "+str(newQM) + ' added to database\r')
                
        except KeyError: #check on this one
            continue
        except IndexError:
            print(("Index error in " + str(line)))
            continue

def parse_KH_QMs(kh, inputFile):
    """import QMs from the 1623-161 (Kaninchenh<6E>hle) html pages
    """
    khQMs=open(settings.EXPOWEB+inputFile,'r')
    khQMs=khQMs.readlines()
    for line in khQMs:
        res=re.search('name=\"[CB](?P<year>\d*)-(?P<cave>\d*)-(?P<number>\d*).*</a> (?P<grade>[ABDCV])<dd>(?P<description>.*)\[(?P<nearest_station>.*)\]',line)
        if res:
            res=res.groupdict()
            year=int(res['year'])
            #check if placeholder exists for given year, create it if not
            message = " ! - "+ str(year) + " logbook: placeholder entry for '161 KH' created. Should be re-attached to the actual trip."
            placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="161", title="placeholder for QMs in 161", text=message, defaults={"date": date((year), 1, 1),"cave_slug":str(kh)})
            if hadToCreate:
                print(message)
                DataIssue.objects.create(parser='QMs', message=message)
            lookupArgs={
                'found_by':placeholder,
                'number':res['number']
                }
            nonLookupArgs={
                'grade':res['grade'],
                'nearest_station_name':res['nearest_station'],
                'location_description':res['description']
                }
 
            save_carefully(QM,lookupArgs,nonLookupArgs)
        
def Load_QMs():
    parseCaveQMs(cave='204-steinBH',inputFile=r"1623/204/qm.csv")
    parseCaveQMs(cave='234-Hauch',inputFile=r"1623/234/qm.csv")
    parseCaveQMs(cave='161-KH', inputFile="1623/161/qmtodo.htm")
    #parseCaveQMs(cave='balkonhoehle',inputFile=r"1623/264/qm.csv")