From b4c4f2aefcca362d28ef8d7c07fe65e5657c2b4c Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@klebos.com>
Date: Fri, 7 Oct 2022 10:57:30 +0300
Subject: [PATCH] reduce mem use by 21.2MB by using a generator

---
 parsers/survex.py | 68 +++++++++++++++++++++++------------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/parsers/survex.py b/parsers/survex.py
index 406f08a..222c676 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -973,7 +973,7 @@ class LoadingSurvex():
             if cave:
                 survexfile.cave = cave
 
-    def LinearLoad(self, survexblock, path, svxlines):
+    def LinearLoad(self, survexblock, path, collatefilename):
         """Loads a single survex file. Usually used to import all the survex files which have been collated
         into a single file. Loads the begin/end blocks using a stack for labels.
         """
@@ -1003,7 +1003,7 @@ class LoadingSurvex():
             if blockcount % 800 ==0 :
                 print("\n", file=sys.stderr,end='')
                 mem=get_process_memory()
-                print("  - MEM:{:7.3f} MB in use".format(mem),file=sys.stderr)
+                print("  - MEM: {:7.2f} MB in use".format(mem),file=sys.stderr)
                 print("    ", file=sys.stderr,end='')
             sys.stderr.flush()
 
@@ -1190,36 +1190,40 @@ class LoadingSurvex():
                 self.LoadSurvexFallThrough(survexblock, args, cmd)
 
 
-        for svxline in svxlines:
-            self.lineno += 1
-            sline, comment = self.rx_comment.match(svxline).groups()
-            if comment:
-                # this catches the ;*include NEWFILE and ;*edulcni ENDOFFILE lines too
-                self.LoadSurvexComment(survexblock, comment) 
+        # this is a python generator idiom. 
+        # see https://realpython.com/introduction-to-python-generators/
+        # this is the first use of generators in troggle (Oct.2022)
+        with open(collatefilename, "r") as fcollate:
+            for svxline in fcollate:
+                self.lineno += 1
+                sline, comment = self.rx_comment.match(svxline).groups()
+                if comment:
+                    # this catches the ;*include NEWFILE and ;*edulcni ENDOFFILE lines too
+                    self.LoadSurvexComment(survexblock, comment) 
 
-            if not sline:
-                continue # skip blank lines
+                if not sline:
+                    continue # skip blank lines
 
-            # detect a merge failure inserted by version control
-            mfail = self.rx_badmerge.match(sline)
-            if mfail: 
-                message = f"\n ! - ERROR version control merge failure\n   - '{sline}'\n"
-                message = message + f"   - line {self.lineno} in {blkid} in {survexblock}\n   - NERD++ needed to fix it"
-                print(message)
-                print(message,file=sys.stderr)
-                DataIssue.objects.create(parser='survex', message=message)
-                continue # skip this line
+                # detect a merge failure inserted by version control
+                mfail = self.rx_badmerge.match(sline)
+                if mfail: 
+                    message = f"\n ! - ERROR version control merge failure\n   - '{sline}'\n"
+                    message = message + f"   - line {self.lineno} in {blkid} in {survexblock}\n   - NERD++ needed to fix it"
+                    print(message)
+                    print(message,file=sys.stderr)
+                    DataIssue.objects.create(parser='survex', message=message)
+                    continue # skip this line
 
-            # detect a star command
-            star = self.rx_star.match(sline)
-            if star: 
-                # yes we are reading a *command
-                starstatement(star)
-            else: # not a *cmd so we are reading data OR a ";" rx_comment failed. We hope.
-                self.LoadSurvexLeg(survexblock, sline, comment, svxline)
+                # detect a star command
+                star = self.rx_star.match(sline)
+                if star: 
+                    # yes we are reading a *command
+                    starstatement(star)
+                else: # not a *cmd so we are reading data OR a ";" rx_comment failed. We hope.
+                    self.LoadSurvexLeg(survexblock, sline, comment, svxline)
 
-        self.legsnumber = nlegstotal
-        self.slength = slengthtotal  
+            self.legsnumber = nlegstotal
+            self.slength = slengthtotal  
                 
     def PushdownStackScan(self, survexblock, path, fin, flinear, fcollate):
         """Follows the *include links in all the survex files from the root file 1623.svx
@@ -1620,17 +1624,13 @@ def FindAndLoadSurvex(survexblockroot):
     svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
     svx_load.svxdirs[""] = survexfileroot.survexdirectory
 
-    # This next should be rewritten to use a generator so that only one
-    # line is held in memory at a time:
-    with open(collatefilename, "r") as fcollate:
-            svxlines = fcollate.read().splitlines()
     #pr2 = cProfile.Profile()
     #pr2.enable()
     mem1 = get_process_memory()
-    print(f"  - MEM:{mem1:7.2f} MB immediately after reading '{collatefilename}' into memory.",file=sys.stderr)
+    print(f"  - MEM:{mem1:7.2f} MB NOT reading '{collatefilename}' into memory.",file=sys.stderr)
     print("    ", file=sys.stderr,end='')
     #----------------------------------------------------------------
-    svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines)
+    svx_load.LinearLoad(survexblockroot,survexfileroot.path, collatefilename)
     #----------------------------------------------------------------
     #pr2.disable()
     # with open('LinearLoad.prof', 'w') as f: