expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-25 08:41:51 +00:00
Code Issues 19 Projects 3 Activity

reduce mem use by 21.2MB by using a generator

Browse Source
This commit is contained in:
Philip Sargent 2022-10-07 10:57:30 +03:00
parent d16226c879
commit b4c4f2aefc
1 changed files with 34 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
parsers/survex.py
View File

@ -973,7 +973,7 @@ class LoadingSurvex():
if cave:
survexfile.cave = cave
def LinearLoad(self, survexblock, path, svxlines):
def LinearLoad(self, survexblock, path, collatefilename):
"""Loads a single survex file. Usually used to import all the survex files which have been collated
into a single file. Loads the begin/end blocks using a stack for labels.
"""
@ -1003,7 +1003,7 @@ class LoadingSurvex():
if blockcount % 800 ==0 :
print("\n", file=sys.stderr,end='')
mem=get_process_memory()
print(" - MEM:{:7.3f} MB in use".format(mem),file=sys.stderr)
print(" - MEM: {:7.2f} MB in use".format(mem),file=sys.stderr)
print(" ", file=sys.stderr,end='')
sys.stderr.flush()
@ -1190,7 +1190,11 @@ class LoadingSurvex():
self.LoadSurvexFallThrough(survexblock, args, cmd)
for svxline in svxlines:
# this is a python generator idiom.
# see https://realpython.com/introduction-to-python-generators/
# this is the first use of generators in troggle (Oct.2022)
with open(collatefilename, "r") as fcollate:
for svxline in fcollate:
self.lineno += 1
sline, comment = self.rx_comment.match(svxline).groups()
if comment:
@ -1620,17 +1624,13 @@ def FindAndLoadSurvex(survexblockroot):
svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
svx_load.svxdirs[""] = survexfileroot.survexdirectory
# This next should be rewritten to use a generator so that only one
# line is held in memory at a time:
with open(collatefilename, "r") as fcollate:
svxlines = fcollate.read().splitlines()
#pr2 = cProfile.Profile()
#pr2.enable()
mem1 = get_process_memory()
print(f" - MEM:{mem1:7.2f} MB immediately after reading '{collatefilename}' into memory.",file=sys.stderr)
print(f" - MEM:{mem1:7.2f} MB NOT reading '{collatefilename}' into memory.",file=sys.stderr)
print(" ", file=sys.stderr,end='')
#----------------------------------------------------------------
svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines)
svx_load.LinearLoad(survexblockroot,survexfileroot.path, collatefilename)
#----------------------------------------------------------------
#pr2.disable()
# with open('LinearLoad.prof', 'w') as f: