import troggle.settings as settings import troggle.core.models as models import troggle.settings as settings from subprocess import call, Popen, PIPE from troggle.parsers.people import GetPersonExpeditionNameLookup import re import os def LoadSurvexLineLeg(survexblock, stardata, sline, comment): ls = sline.lower().split() ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]]) ssto = survexblock.MakeSurvexStation(ls[stardata["to"]]) survexleg = models.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto) if stardata["type"] == "normal": try: survexleg.tape = float(ls[stardata["tape"]]) except ValueError: print("Tape misread in", survexblock.survexfile.path) print("Stardata:", stardata) print("Line:", ls) survexleg.tape = 1000 try: lclino = ls[stardata["clino"]] except: print("Clino misread in", survexblock.survexfile.path) print("Stardata:", stardata) print("Line:", ls) lclino = error try: lcompass = ls[stardata["compass"]] except: print("Compass misread in", survexblock.survexfile.path) print("Stardata:", stardata) print("Line:", ls) lcompass = error if lclino == "up": survexleg.compass = 0.0 survexleg.clino = 90.0 elif lclino == "down": survexleg.compass = 0.0 survexleg.clino = -90.0 elif lclino == "-" or lclino == "level": try: survexleg.compass = float(lcompass) except ValueError: print("Compass misread in", survexblock.survexfile.path) print("Stardata:", stardata) print("Line:", ls) survexleg.compass = 1000 survexleg.clino = -90.0 else: assert re.match(r"[\d\-+.]+$", lcompass), ls assert re.match(r"[\d\-+.]+$", lclino) and lclino != "-", ls survexleg.compass = float(lcompass) survexleg.clino = float(lclino) # only save proper legs survexleg.save() itape = stardata.get("tape") if itape: try: survexblock.totalleglength += float(ls[itape]) except ValueError: print("Length not added") survexblock.save() def LoadSurvexEquate(survexblock, sline): #print sline # stations = sline.split() assert len(stations) > 1 for station in stations: survexblock.MakeSurvexStation(station) def LoadSurvexLinePassage(survexblock, stardata, sline, comment): pass stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4} stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"} def RecursiveLoad(survexblock, survexfile, fin, textlines): iblankbegins = 0 text = [ ] stardata = stardatadefault teammembers = [ ] # uncomment to print out all files during parsing print("Reading file:", survexblock.survexfile.path) while True: svxline = fin.readline().decode("latin1") if not svxline: return textlines.append(svxline) # break the line at the comment sline, comment = re.match(r"([^;]*?)\s*(?:;\s*(.*))?\n?$", svxline.strip()).groups() # detect ref line pointing to the scans directory mref = comment and re.match(r'.*?ref.*?(\d+)\s*#\s*(\d+)', comment) if mref: refscan = "%s#%s" % (mref.group(1), mref.group(2)) survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan) if survexscansfolders: survexblock.survexscansfolder = survexscansfolders[0] #survexblock.refscandir = "%s/%s%%23%s" % (mref.group(1), mref.group(1), mref.group(2)) survexblock.save() continue if not sline: continue # detect the star command mstar = re.match(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$', sline) if not mstar: if "from" in stardata: LoadSurvexLineLeg(survexblock, stardata, sline, comment) elif stardata["type"] == "passage": LoadSurvexLinePassage(survexblock, stardata, sline, comment) #Missing "station" in stardata. continue # detect the star command cmd, line = mstar.groups() cmd = cmd.lower() if re.match("include$(?i)", cmd): includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line)) includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave) includesurvexfile.save() includesurvexfile.SetDirectory() if includesurvexfile.exists(): fininclude = includesurvexfile.OpenFile() RecursiveLoad(survexblock, includesurvexfile, fininclude, textlines) elif re.match("begin$(?i)", cmd): if line: name = line.lower() survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile, totalleglength=0.0) survexblockdown.save() textlinesdown = [ ] RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown) else: iblankbegins += 1 elif re.match("end$(?i)", cmd): if iblankbegins: iblankbegins -= 1 else: survexblock.text = "".join(textlines) survexblock.save() return elif re.match("date$(?i)", cmd): if len(line) == 10: survexblock.date = re.sub(r"\.", "-", line) expeditions = models.Expedition.objects.filter(year=line[:4]) if expeditions: assert len(expeditions) == 1 survexblock.expedition = expeditions[0] survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date) survexblock.save() elif re.match("team$(?i)", cmd): mteammember = re.match(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line) if mteammember: for tm in re.split(r" and | / |, | & | \+ |^both$|^none$(?i)", mteammember.group(2)): if tm: personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower()) if (personexpedition, tm) not in teammembers: teammembers.append((personexpedition, tm)) personrole = models.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm) personrole.expeditionday = survexblock.expeditionday if personexpedition: personrole.person=personexpedition.person personrole.save() elif cmd == "title": survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave) survextitle.save() elif cmd == "require": # should we check survex version available for processing? pass elif cmd == "data": ls = line.lower().split() stardata = { "type":ls[0] } for i in range(0, len(ls)): stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1 if ls[0] in ["normal", "cartesian", "nosurvey"]: assert (("from" in stardata and "to" in stardata) or "station" in stardata), line elif ls[0] == "default": stardata = stardatadefault else: assert ls[0] == "passage", line elif cmd == "equate": LoadSurvexEquate(survexblock, line) elif cmd == "fix": survexblock.MakeSurvexStation(line.split()[0]) else: if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument", "calibrate", "set", "infer", "alias", "ref", "cs", "declination", "case"]: print("Unrecognised command in line:", cmd, line, survexblock, survexblock.survexfile.path) def ReloadSurvexCave(survex_cave, area): print(survex_cave, area) cave = models.Cave.objects.get(kataster_number=survex_cave, area__short_name=area) print(cave) #cave = models.Cave.objects.get(kataster_number=survex_cave) cave.survexblock_set.all().delete() cave.survexfile_set.all().delete() cave.survexdirectory_set.all().delete() survexfile = models.SurvexFile(path="caves-" + cave.kat_area() + "/" + survex_cave + "/" + survex_cave, cave=cave) survexfile.save() survexfile.SetDirectory() survexblockroot = models.SurvexBlock(name="root", survexpath="caves-" + cave.kat_area(), begin_char=0, cave=cave, survexfile=survexfile, totalleglength=0.0) survexblockroot.save() fin = survexfile.OpenFile() textlines = [ ] RecursiveLoad(survexblockroot, survexfile, fin, textlines) survexblockroot.text = "".join(textlines) survexblockroot.save() def LoadAllSurvexBlocks(): print('Loading All Survex Blocks...') models.SurvexBlock.objects.all().delete() models.SurvexFile.objects.all().delete() models.SurvexDirectory.objects.all().delete() models.SurvexEquate.objects.all().delete() models.SurvexLeg.objects.all().delete() models.SurvexTitle.objects.all().delete() models.SurvexPersonRole.objects.all().delete() models.SurvexStation.objects.all().delete() print(" - Data flushed") survexfile = models.SurvexFile(path="all", cave=None) survexfile.save() survexfile.SetDirectory() #Load all survexblockroot = models.SurvexBlock(name="root", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0) survexblockroot.save() fin = survexfile.OpenFile() textlines = [ ] RecursiveLoad(survexblockroot, survexfile, fin, textlines) survexblockroot.text = "".join(textlines) survexblockroot.save() #Load each cave, #FIXME this should be dealt with load all above print(" - Reloading all caves") caves = models.Cave.objects.all() for cave in caves: if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves-" + cave.kat_area(), cave.kataster_number)): if cave.kataster_number not in ['40']: print("loading", cave, cave.kat_area()) ReloadSurvexCave(cave.kataster_number, cave.kat_area()) poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$") def LoadPos(): print('Loading Pos....') call([settings.CAVERN, "--output=%s/all.3d" % settings.SURVEX_DATA, "%s/all.svx" % settings.SURVEX_DATA]) call([settings.THREEDTOPOS, '%sall.3d' % settings.SURVEX_DATA], cwd = settings.SURVEX_DATA) posfile = open("%sall.pos" % settings.SURVEX_DATA) posfile.readline() #Drop header for line in posfile.readlines(): r = poslineregex.match(line) if r: x, y, z, name = r.groups() try: ss = models.SurvexStation.objects.lookup(name) ss.x = float(x) ss.y = float(y) ss.z = float(z) ss.save() except: print("%s not parsed in survex" % name)