recursive scan *import to make linear filelist

parsers/survex.py

@@ -53,6 +53,10 @@ class LoadSurvex():
     survexlegsalllength = 0.0
     survexlegsnumber = 0
     depthbegin = 0
+    depthimport = 0
+    stackbegin =[]
+    stackimport = []
+    svxfileslist =[]
     lineno = 0
     insp = ""
     callcount = 0
@@ -295,9 +299,10 @@ class LoadSurvex():
         self.lineno = 0
         sys.stderr.flush();
         self.callcount +=1
-        if self.callcount >=10:
+        if self.callcount % 10 ==0 :
-            self.callcount=0
             print(".", file=sys.stderr,end='')
+        if self.callcount % 500 ==0 :
+            print("\n", file=sys.stderr,end='')
         # Try to find the cave in the DB if not use the string as before
         path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", survexblock.survexfile.path)
         if path_match:
@@ -306,6 +311,79 @@ class LoadSurvex():
             if cave:
                 survexfile.cave = cave
 
+    def RecursiveScan(self, survexblock, survexfile, fin, flinear):
+        """Follows the *include links in all the survex files from the root file 1623.svx
+        and reads only the *import and *begin and *end statements. It produces a linearised
+        list of the import tree
+        """
+        indent = " " * self.depthimport
+        sys.stderr.flush();
+        self.callcount +=1
+        if self.callcount % 10 ==0 :
+            print(".", file=sys.stderr,end='')
+        if self.callcount % 500 ==0 :
+            print("\n", file=sys.stderr,end='')
+
+        self.svxfileslist.append(survexfile)
+        
+        svxlines = fin.read().splitlines()
+        for svxline in svxlines:
+            self.lineno += 1
+            sline, comment = self.rx_comment.match(svxline.strip()).groups()
+            mstar = self.rx_star.match(sline)
+            if mstar: # yes we are reading a *cmd
+                cmd, args = mstar.groups()
+                cmd = cmd.lower()
+                if re.match("include$(?i)", cmd):
+                    includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", args)))
+                    path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
+
+                    includesurvexfile = models_survex.SurvexFile(path=includepath)
+
+                    if includesurvexfile.exists():
+                        #--------------------------------------------------------
+                        self.depthimport += 1
+                        fininclude = includesurvexfile.OpenFile()
+                        flinear.write("{:2} {} *import {}\n".format(self.depthimport, indent, includesurvexfile.path))
+                        push = includesurvexfile.path.lower()
+                        self.stackimport.append(push)
+                        self.RecursiveScan(survexblock, includesurvexfile, fininclude, flinear)
+                        pop = self.stackimport.pop()
+                        if pop != push:
+                            print("!!!!!!!    ERROR pop != push {} != {} {}".format(pop, push, self.stackimport))
+                            print("!!!!!!!    ERROR pop != push {} != {} {}\n".format(pop, push, self.stackimport),file=flinear)
+                            print("!!!!!!!    ERROR pop != push {} != {} {}".format(pop, push, self.stackimport),file=sys.stderr)
+                        flinear.write("{:2} {} *tropmi {}\n".format(self.depthimport, indent, includesurvexfile.path))
+                        fininclude.close()
+                        self.depthimport -= 1
+                        #--------------------------------------------------------
+                    else:
+                        print("    ! ERROR *include file not found for {}".format(includesurvexfile))
+                elif re.match("begin$(?i)", cmd):
+                    self.depthbegin += 1
+                    depth = " " * self.depthbegin
+                    if args:
+                        pushargs = args
+                    else:
+                        pushargs = " "
+                    self.stackbegin.append(pushargs.lower())
+                    flinear.write("            {:2} {} *begin {}\n".format(self.depthbegin, depth, args))
+                    pass
+                elif re.match("end$(?i)", cmd):
+                    depth = " " * self.depthbegin
+                    flinear.write("            {:2} {} *end   {}\n".format(self.depthbegin, depth, args))
+                    if not args:
+                        args = " "
+                    popargs = self.stackbegin.pop()
+                    if popargs != args.lower():
+                        print("!!!!!!!    ERROR BEGIN/END pop != push {} != {}\n{}".format(popargs, args, self. stackbegin))
+                        print("!!!!!!!    ERROR BEGIN/END pop != push {} != {}\n{}\n".format(popargs, args, self. stackbegin), file=flinear)
+                        print(" !!!!!!!    ERROR BEGIN/END pop != push {} != {}\n{}".format(popargs, args,self. stackbegin), file=sys.stderr,)
+
+                    self.depthbegin -= 1
+                    pass
+
+
     def RecursiveLoad(self, survexblock, survexfile, fin):
         """Follows the *include links in all the survex files from the root file 1623.svx
         and reads in the survex blocks, other data and the wallet references (scansfolder) as it
@@ -453,15 +531,45 @@ def FindAndLoadAllSurvex(survexblockroot, survexfileroot):
     # Redirect sys.stdout to the file
     sys.stdout = open('svxblks.log', 'w')
 
-    svxl = LoadSurvex()
+    print(' - SCANNING All Survex Blocks...',file=sys.stderr)
+
+    svxl0 = LoadSurvex()
+    svxl0.callcount = 0
+    svxl0.depthimport = 0
+    indent=""
+    
+    mem0 = models.get_process_memory()
+    flinear = open('svxlinear.log', 'w')
+    flinear.write("    - MEM:{:.2f} MB START {}\n".format(mem0,survexfileroot.path))
 
     finroot = survexfileroot.OpenFile()
-    svxl.RecursiveLoad(survexblockroot, survexfileroot, finroot)
+    flinear.write("{:2} {} *import {}\n".format(svxl0.depthimport, indent, survexfileroot.path))
+    svxl0.RecursiveScan(survexblockroot, survexfileroot, finroot, flinear)
+    flinear.write("{:2} {} *tropmi {}\n".format(svxl0.depthimport, indent, survexfileroot.path))
+    mem1 = models.get_process_memory()
+    flinear.write("    - MEM:{:.2f} MB STOP {}\n".format(mem1,survexfileroot.path))
+    flinear.write("    - MEM:{:.3f} MB USED\n".format(mem1-mem0))
+    svxfileslist = svxl0.svxfileslist
+    flinear.write("    - {:,} survex files in linear import list \n".format(len(svxfileslist)))
+    flinear.close()
+    svxl0 = None
+    print("\n -  {:,} survex files in linear import list \n".format(len(svxfileslist)),file=sys.stderr)
+
+    # INSERT IN HERE linear, not recursive, wrt import loading of all the data using [svxfileslist] #
+    for f in svxfileslist:
+        # Load legs etc. recursive only in BEGIN / END 
+        pass
+    
+    print('\n - Loading All Survex Blocks...',file=sys.stderr)
+    svxlrl = LoadSurvex()
+
+    finroot = survexfileroot.OpenFile()
+    svxlrl.RecursiveLoad(survexblockroot, survexfileroot, finroot)
     finroot.close()
     
-    survexlegsnumber = svxl.survexlegsnumber
+    survexlegsnumber = svxlrl.survexlegsnumber
-    survexlegsalllength = svxl.survexlegsalllength
+    survexlegsalllength = svxlrl.survexlegsalllength
-    
+    svxlrl = None
     # Close the logging file, Restore sys.stdout to our old saved file handle
     sys.stdout.close()
     print("+", file=sys.stderr)