troggle/parsers/survex.py

import troggle.settings as settings
import troggle.core.models as models
import troggle.settings as settings

from subprocess import call, Popen, PIPE

from troggle.parsers.people import GetPersonExpeditionNameLookup
import re
import os


def LoadSurvexLineLeg(survexblock, stardata, sline, comment):
    ls = sline.lower().split()
    ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]])
    ssto = survexblock.MakeSurvexStation(ls[stardata["to"]])
    
    survexleg = models.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto)
    if stardata["type"] == "normal":
        try:
            survexleg.tape = float(ls[stardata["tape"]])
        except ValueError: 
            print("Tape misread in", survexblock.survexfile.path)
            print("Stardata:", stardata)
            print("Line:", ls)
            survexleg.tape = 1000
        try:
            lclino = ls[stardata["clino"]]
        except:
            print("Clino misread in", survexblock.survexfile.path)
            print("Stardata:", stardata)
            print("Line:", ls)
            lclino = error
        try:
            lcompass = ls[stardata["compass"]]
        except:
            print("Compass misread in", survexblock.survexfile.path)
            print("Stardata:", stardata)
            print("Line:", ls)
            lcompass = error
        if lclino == "up":
            survexleg.compass = 0.0
            survexleg.clino = 90.0
        elif lclino == "down":
            survexleg.compass = 0.0
            survexleg.clino = -90.0
        elif lclino == "-" or lclino == "level":
            try:
                survexleg.compass = float(lcompass)
            except ValueError:
                print("Compass misread in", survexblock.survexfile.path)
                print("Stardata:", stardata)
                print("Line:", ls)
                survexleg.compass = 1000
            survexleg.clino = -90.0
        else:
            assert re.match(r"[\d\-+.]+$", lcompass), ls
            assert re.match(r"[\d\-+.]+$", lclino) and lclino != "-", ls
            survexleg.compass = float(lcompass)
            survexleg.clino = float(lclino)
        
        # only save proper legs
        survexleg.save()
    
    itape = stardata.get("tape")
    if itape:
        try:
            survexblock.totalleglength += float(ls[itape])
        except ValueError:
            print("Length not added")
        survexblock.save()


def LoadSurvexEquate(survexblock, sline):
    #print sline #
    stations = sline.split()
    assert len(stations) > 1
    for station in stations:
        survexblock.MakeSurvexStation(station)


def LoadSurvexLinePassage(survexblock, stardata, sline, comment):
    pass
    

stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"}

def RecursiveLoad(survexblock, survexfile, fin, textlines):
    iblankbegins = 0
    text = [ ]
    stardata = stardatadefault
    teammembers = [ ]
 
# uncomment to print out all files during parsing 
    print("Reading file: " + survexblock.survexfile.path)
    while True:
        svxline = fin.readline().decode("latin1")
        if not svxline:
            return
        textlines.append(svxline)

        # break the line at the comment
        sline, comment = re.match(r"([^;]*?)\s*(?:;\s*(.*))?\n?$", svxline.strip()).groups()
        
        # detect ref line pointing to the scans directory
        mref = comment and re.match(r'.*?ref.*?(\d+)\s*#\s*(\d+)', comment)
        if mref:
            refscan = "%s#%s" % (mref.group(1), mref.group(2))
            survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan)
            if survexscansfolders:
                survexblock.survexscansfolder = survexscansfolders[0]
                #survexblock.refscandir = "%s/%s%%23%s" % (mref.group(1), mref.group(1), mref.group(2))
                survexblock.save()   
            continue
        
        if not sline:
            continue
        
        # detect the star command
        mstar = re.match(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$', sline)
        if not mstar:
            if "from" in stardata:
                LoadSurvexLineLeg(survexblock, stardata, sline, comment)
            elif stardata["type"] == "passage":
                LoadSurvexLinePassage(survexblock, stardata, sline, comment)
            #Missing "station" in stardata.
            continue
        
        # detect the star command
        cmd, line = mstar.groups()
        cmd = cmd.lower()
        if re.match("include$(?i)", cmd):
            includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line))
            includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave)
            includesurvexfile.save()
            includesurvexfile.SetDirectory()
            if includesurvexfile.exists():
                fininclude = includesurvexfile.OpenFile()
                RecursiveLoad(survexblock, includesurvexfile, fininclude, textlines)
        
        elif re.match("begin$(?i)", cmd):
            if line: 
                name = line.lower()
                survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile, totalleglength=0.0)
                survexblockdown.save()
                textlinesdown = [ ]
                RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown)
            else:
                iblankbegins += 1
        
        elif re.match("end$(?i)", cmd):
            if iblankbegins:
                iblankbegins -= 1
            else:
                survexblock.text = "".join(textlines)
                survexblock.save()
                return
        
        elif re.match("date$(?i)", cmd):
            if len(line) == 10:
                survexblock.date = re.sub(r"\.", "-", line)
                expeditions = models.Expedition.objects.filter(year=line[:4])
                if expeditions:
                    assert len(expeditions) == 1
                    survexblock.expedition = expeditions[0]
                    survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date)
                    survexblock.save()
                    
        elif re.match("team$(?i)", cmd):
            mteammember = re.match(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line)
            if mteammember:
                for tm in re.split(r" and | / |, | & | \+ |^both$|^none$(?i)", mteammember.group(2)):
                    if tm:
                        personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
                        if (personexpedition, tm) not in teammembers:
                            teammembers.append((personexpedition, tm))
                            personrole = models.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
                            personrole.expeditionday = survexblock.expeditionday
                            if personexpedition:
                                personrole.person=personexpedition.person
                            personrole.save()
                            
        elif cmd == "title":
            survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave)
            survextitle.save()
            
        elif cmd == "require":
            # should we check survex version available for processing?
            pass

        elif cmd == "data":
            ls = line.lower().split()
            stardata = { "type":ls[0] }
            for i in range(0, len(ls)):
                stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1
            if ls[0] in ["normal", "cartesian", "nosurvey"]:
                assert (("from" in stardata and "to" in stardata) or "station" in stardata), line
            elif ls[0] == "default":
                stardata = stardatadefault
            else:
                assert ls[0] == "passage", line
        
        elif cmd == "equate":
            LoadSurvexEquate(survexblock, line)

        elif cmd == "fix":
            survexblock.MakeSurvexStation(line.split()[0])

        else:
            if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument",
                           "calibrate", "set", "infer", "alias", "ref", "cs", "declination", "case"]:
                print("Unrecognised command in line:", cmd, line, survexblock, survexblock.survexfile.path)


def ReloadSurvexCave(survex_cave, area):
    print(survex_cave, area)
    cave = models.Cave.objects.get(kataster_number=survex_cave, area__short_name=area)
    print(cave)
    #cave = models.Cave.objects.get(kataster_number=survex_cave)
    cave.survexblock_set.all().delete()
    cave.survexfile_set.all().delete()
    cave.survexdirectory_set.all().delete()
    
    survexfile = models.SurvexFile(path="caves-" + cave.kat_area() + "/" + survex_cave + "/" + survex_cave, cave=cave)
    survexfile.save()
    survexfile.SetDirectory()
    
    survexblockroot = models.SurvexBlock(name="root", survexpath="caves-" + cave.kat_area(), begin_char=0, cave=cave, survexfile=survexfile, totalleglength=0.0)
    survexblockroot.save()
    fin = survexfile.OpenFile()
    textlines = [ ]
    RecursiveLoad(survexblockroot, survexfile, fin, textlines)
    survexblockroot.text = "".join(textlines)
    survexblockroot.save()


def LoadAllSurvexBlocks():

    print('Loading All Survex Blocks...')

    models.SurvexBlock.objects.all().delete()
    models.SurvexFile.objects.all().delete()
    models.SurvexDirectory.objects.all().delete()
    models.SurvexEquate.objects.all().delete()
    models.SurvexLeg.objects.all().delete()
    models.SurvexTitle.objects.all().delete()
    models.SurvexPersonRole.objects.all().delete()
    models.SurvexStation.objects.all().delete()

    print(" - Data flushed")

    survexfile = models.SurvexFile(path="all", cave=None)
    survexfile.save()
    survexfile.SetDirectory()

    #Load all
    survexblockroot = models.SurvexBlock(name="root", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0)
    survexblockroot.save()
    fin = survexfile.OpenFile()
    textlines = [ ]
    RecursiveLoad(survexblockroot, survexfile, fin, textlines)
    survexblockroot.text = "".join(textlines)
    survexblockroot.save()


    #Load each cave, 
    #FIXME this should be dealt with load all above
    print(" - Reloading all caves")
    caves = models.Cave.objects.all()
    for cave in caves:
        if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves-" + cave.kat_area(), cave.kataster_number)):
            if cave.kataster_number not in ['40']:
                print("loading", cave, cave.kat_area())
                ReloadSurvexCave(cave.kataster_number, cave.kat_area())


poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")


def LoadPos():

    print('Loading Pos....')

    call([settings.CAVERN, "--output=%s/all.3d" % settings.SURVEX_DATA, "%s/all.svx" % settings.SURVEX_DATA])
    call([settings.THREEDTOPOS, '%sall.3d' % settings.SURVEX_DATA], cwd = settings.SURVEX_DATA)
    posfile = open("%sall.pos" % settings.SURVEX_DATA)
    posfile.readline() #Drop header
    for line in posfile.readlines():
        r = poslineregex.match(line)                
        if r:
            x, y, z, name = r.groups()
            try:
                ss = models.SurvexStation.objects.lookup(name)
                ss.x = float(x)
                ss.y = float(y)
                ss.z = float(z)
                ss.save()
            except:
                print("%s not parsed in survex" % name)