reduce mem use by 21.2MB by using a generator

parsers/survex.py

@@ -973,7 +973,7 @@ class LoadingSurvex():
             if cave:
                 survexfile.cave = cave
 
-    def LinearLoad(self, survexblock, path, svxlines):
+    def LinearLoad(self, survexblock, path, collatefilename):
         """Loads a single survex file. Usually used to import all the survex files which have been collated
         into a single file. Loads the begin/end blocks using a stack for labels.
         """
@@ -1003,7 +1003,7 @@ class LoadingSurvex():
             if blockcount % 800 ==0 :
                 print("\n", file=sys.stderr,end='')
                 mem=get_process_memory()
-                print("  - MEM:{:7.3f} MB in use".format(mem),file=sys.stderr)
+                print("  - MEM: {:7.2f} MB in use".format(mem),file=sys.stderr)
                 print("    ", file=sys.stderr,end='')
             sys.stderr.flush()
 
@@ -1190,7 +1190,11 @@ class LoadingSurvex():
                 self.LoadSurvexFallThrough(survexblock, args, cmd)
 
 
-        for svxline in svxlines:
+        # this is a python generator idiom. 
+        # see https://realpython.com/introduction-to-python-generators/
+        # this is the first use of generators in troggle (Oct.2022)
+        with open(collatefilename, "r") as fcollate:
+            for svxline in fcollate:
                 self.lineno += 1
                 sline, comment = self.rx_comment.match(svxline).groups()
                 if comment:
@@ -1620,17 +1624,13 @@ def FindAndLoadSurvex(survexblockroot):
     svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
     svx_load.svxdirs[""] = survexfileroot.survexdirectory
 
-    # This next should be rewritten to use a generator so that only one
-    # line is held in memory at a time:
-    with open(collatefilename, "r") as fcollate:
-            svxlines = fcollate.read().splitlines()
     #pr2 = cProfile.Profile()
     #pr2.enable()
     mem1 = get_process_memory()
-    print(f"  - MEM:{mem1:7.2f} MB immediately after reading '{collatefilename}' into memory.",file=sys.stderr)
+    print(f"  - MEM:{mem1:7.2f} MB NOT reading '{collatefilename}' into memory.",file=sys.stderr)
     print("    ", file=sys.stderr,end='')
     #----------------------------------------------------------------
-    svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines)
+    svx_load.LinearLoad(survexblockroot,survexfileroot.path, collatefilename)
     #----------------------------------------------------------------
     #pr2.disable()
     # with open('LinearLoad.prof', 'w') as f: