Backed out changeset: 4552f42bdf54

parsers/survex.py

@@ -5,14 +5,9 @@ import troggle.settings as settings
 from subprocess import call, Popen, PIPE
 
 from troggle.parsers.people import GetPersonExpeditionNameLookup
-from django.utils.timezone import get_current_timezone
-from django.utils.timezone import make_aware
-
 import re
 import os
-from datetime import datetime
 
-line_leg_regex = re.compile(r"[\d\-+.]+$")
 
 def LoadSurvexLineLeg(survexblock, stardata, sline, comment):
     ls = sline.lower().split()
@@ -58,8 +53,8 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment):
                 survexleg.compass = 1000
             survexleg.clino = -90.0
         else:
-            assert line_leg_regex.match(lcompass), ls
-            assert line_leg_regex.match(lclino) and lclino != "-", ls
+            assert re.match(r"[\d\-+.]+$", lcompass), ls
+            assert re.match(r"[\d\-+.]+$", lclino) and lclino != "-", ls
             survexleg.compass = float(lcompass)
             survexleg.clino = float(lclino)
         
@@ -90,12 +85,6 @@ def LoadSurvexLinePassage(survexblock, stardata, sline, comment):
 stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
 stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"}
 
-regex_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$")
-regex_ref     = re.compile(r'.*?ref.*?(\d+)\s*#\s*(\d+)')
-regex_star    = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$')
-regex_team    = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)")
-regex_team_member        = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)")
-
 def RecursiveLoad(survexblock, survexfile, fin, textlines):
     iblankbegins = 0
     text = [ ]
@@ -103,25 +92,18 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
     teammembers = [ ]
  
 # uncomment to print out all files during parsing 
-    print("  - Reading file: " + survexblock.survexfile.path)
-    stamp = datetime.now()
-    lineno = 0
+    print("Reading file: " + survexblock.survexfile.path)
     while True:
         svxline = fin.readline().decode("latin1")
         if not svxline:
-            print('   - Not survex')
             return
         textlines.append(svxline)
 
-        lineno += 1
-
-        print('   - Line: %d' % lineno)
-
         # break the line at the comment
-        sline, comment = regex_comment.match(svxline.strip()).groups()
+        sline, comment = re.match(r"([^;]*?)\s*(?:;\s*(.*))?\n?$", svxline.strip()).groups()
         
         # detect ref line pointing to the scans directory
-        mref = comment and regex_ref.match(comment)
+        mref = comment and re.match(r'.*?ref.*?(\d+)\s*#\s*(\d+)', comment)
         if mref:
             refscan = "%s#%s" % (mref.group(1), mref.group(2))
             survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan)
@@ -135,15 +117,12 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
             continue
         
         # detect the star command
-        mstar = regex_star.match(sline)
+        mstar = re.match(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$', sline)
         if not mstar:
             if "from" in stardata:
                 LoadSurvexLineLeg(survexblock, stardata, sline, comment)
-                print('   - From: ')
-                #print(stardata)
             elif stardata["type"] == "passage":
                 LoadSurvexLinePassage(survexblock, stardata, sline, comment)
-                print('   - Pasage: ')
             #Missing "station" in stardata.
             continue
         
@@ -152,7 +131,6 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
         cmd = cmd.lower()
         if re.match("include$(?i)", cmd):
             includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line))
-            print('   - Include file found including - ' + includepath)
             includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave)
             includesurvexfile.save()
             includesurvexfile.SetDirectory()
@@ -163,7 +141,6 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
         elif re.match("begin$(?i)", cmd):
             if line: 
                 name = line.lower()
-                #print('   - Begin found for: ' + name)
                 survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile, totalleglength=0.0)
                 survexblockdown.save()
                 textlinesdown = [ ]
@@ -177,16 +154,11 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
             else:
                 survexblock.text = "".join(textlines)
                 survexblock.save()
-                print('   - End found: ')
-                endstamp = datetime.now()
-                timetaken = endstamp - stamp
-                print('   - Time to process: ' + str(timetaken))
                 return
         
         elif re.match("date$(?i)", cmd):
             if len(line) == 10:
-                #print('   - Date found: ' + line)
-                survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone())
+                survexblock.date = re.sub(r"\.", "-", line)
                 expeditions = models.Expedition.objects.filter(year=line[:4])
                 if expeditions:
                     assert len(expeditions) == 1
@@ -195,11 +167,9 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
                     survexblock.save()
                     
         elif re.match("team$(?i)", cmd):
-            pass
-            print('   - Team found: ')
-            mteammember = regex_team.match(line)
+            mteammember = re.match(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line)
             if mteammember:
-                for tm in regex_team_member.split(mteammember.group(2)):
+                for tm in re.split(r" and | / |, | & | \+ |^both$|^none$(?i)", mteammember.group(2)):
                     if tm:
                         personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
                         if (personexpedition, tm) not in teammembers:
@@ -211,7 +181,6 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
                             personrole.save()
                             
         elif cmd == "title":
-            #print('   - Title found: ')
             survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave)
             survextitle.save()
             
@@ -220,11 +189,8 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
             pass
 
         elif cmd == "data":
-            #print('   - Data found: ')
             ls = line.lower().split()
             stardata = { "type":ls[0] }
-            #print('    - Star data: ', stardata)
-            #print(ls)
             for i in range(0, len(ls)):
                 stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1
             if ls[0] in ["normal", "cartesian", "nosurvey"]:
@@ -235,32 +201,26 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
                 assert ls[0] == "passage", line
         
         elif cmd == "equate":
-            #print('   - Equate found: ')
             LoadSurvexEquate(survexblock, line)
 
         elif cmd == "fix":
-            #print('   - Fix found: ')
             survexblock.MakeSurvexStation(line.split()[0])
 
         else:
-            #print('   - Stuff')
             if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument",
                            "calibrate", "set", "infer", "alias", "ref", "cs", "declination", "case"]:
                 print("Unrecognised command in line:", cmd, line, survexblock, survexblock.survexfile.path)
-        endstamp = datetime.now()
-        timetaken = endstamp - stamp
-        print('   - Time to process: ' + str(timetaken))
 
 
 def ReloadSurvexCave(survex_cave, area):
-    print(' - Area: ' + str(area) + ' Cave: ' + str(survex_cave))
+    print(survex_cave, area)
     cave = models.Cave.objects.get(kataster_number=survex_cave, area__short_name=area)
-    print(' - ' + str(cave))
+    print(cave)
+    #cave = models.Cave.objects.get(kataster_number=survex_cave)
     cave.survexblock_set.all().delete()
     cave.survexfile_set.all().delete()
     cave.survexdirectory_set.all().delete()
     
-    file_stamp_start = datetime.now()
     survexfile = models.SurvexFile(path="caves-" + cave.kat_area() + "/" + survex_cave + "/" + survex_cave, cave=cave)
     survexfile.save()
     survexfile.SetDirectory()
@@ -268,9 +228,6 @@ def ReloadSurvexCave(survex_cave, area):
     survexblockroot = models.SurvexBlock(name="root", survexpath="caves-" + cave.kat_area(), begin_char=0, cave=cave, survexfile=survexfile, totalleglength=0.0)
     survexblockroot.save()
     fin = survexfile.OpenFile()
-    file_stamp_end = datetime.now()
-    file_time = file_stamp_end - file_stamp_start
-    print(' - Files time to process: ' + str(file_time))
     textlines = [ ]
     RecursiveLoad(survexblockroot, survexfile, fin, textlines)
     survexblockroot.text = "".join(textlines)
@@ -311,15 +268,10 @@ def LoadAllSurvexBlocks():
     print(" - Reloading all caves")
     caves = models.Cave.objects.all()
     for cave in caves:
-        rec_stamp_start = datetime.now()
         if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves-" + cave.kat_area(), cave.kataster_number)):
             if cave.kataster_number not in ['40']:
-                print(" - Loading " + str(cave) + " " + cave.kat_area())
+                print("loading", cave, cave.kat_area())
                 ReloadSurvexCave(cave.kataster_number, cave.kat_area())
-                rec_stamp_end = datetime.now()
-        timetaken = rec_stamp_end - rec_stamp_start
-        print(' - Time to process: ' + str(timetaken))
-        print('--------')
 
 
 poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")