expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-25 08:41:51 +00:00
Code Issues 19 Projects 3 Activity

reduce mem use by 21.2MB by using a generator

Browse Source
This commit is contained in:
Philip Sargent 2022-10-07 10:57:30 +03:00
parent d16226c879
commit b4c4f2aefc
1 changed files with 34 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
parsers/survex.py
View File

@ -973,7 +973,7 @@ class LoadingSurvex():
if cave:
survexfile.cave = cave
def LinearLoad(self, survexblock, path, svxlines):
def LinearLoad(self, survexblock, path, collatefilename):
"""Loads a single survex file. Usually used to import all the survex files which have been collated
into a single file. Loads the begin/end blocks using a stack for labels.
"""
@ -1003,7 +1003,7 @@ class LoadingSurvex():
if blockcount % 800 ==0 :
print("\n", file=sys.stderr,end='')
mem=get_process_memory()
print(" - MEM:{:7.3f} MB in use".format(mem),file=sys.stderr)
print(" - MEM: {:7.2f} MB in use".format(mem),file=sys.stderr)
print(" ", file=sys.stderr,end='')
sys.stderr.flush()
@ -1190,36 +1190,40 @@ class LoadingSurvex():
self.LoadSurvexFallThrough(survexblock, args, cmd)
for svxline in svxlines:
self.lineno += 1
sline, comment = self.rx_comment.match(svxline).groups()
if comment:
# this catches the ;*include NEWFILE and ;*edulcni ENDOFFILE lines too
self.LoadSurvexComment(survexblock, comment)
# this is a python generator idiom.
# see https://realpython.com/introduction-to-python-generators/
# this is the first use of generators in troggle (Oct.2022)
with open(collatefilename, "r") as fcollate:
for svxline in fcollate:
self.lineno += 1
sline, comment = self.rx_comment.match(svxline).groups()
if comment:
# this catches the ;*include NEWFILE and ;*edulcni ENDOFFILE lines too
self.LoadSurvexComment(survexblock, comment)
if not sline:
continue # skip blank lines
if not sline:
continue # skip blank lines
# detect a merge failure inserted by version control
mfail = self.rx_badmerge.match(sline)
if mfail:
message = f"\n ! - ERROR version control merge failure\n - '{sline}'\n"
message = message + f" - line {self.lineno} in {blkid} in {survexblock}\n - NERD++ needed to fix it"
print(message)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
continue # skip this line
# detect a merge failure inserted by version control
mfail = self.rx_badmerge.match(sline)
if mfail:
message = f"\n ! - ERROR version control merge failure\n - '{sline}'\n"
message = message + f" - line {self.lineno} in {blkid} in {survexblock}\n - NERD++ needed to fix it"
print(message)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
continue # skip this line
# detect a star command
star = self.rx_star.match(sline)
if star:
# yes we are reading a *command
starstatement(star)
else: # not a *cmd so we are reading data OR a ";" rx_comment failed. We hope.
self.LoadSurvexLeg(survexblock, sline, comment, svxline)
# detect a star command
star = self.rx_star.match(sline)
if star:
# yes we are reading a *command
starstatement(star)
else: # not a *cmd so we are reading data OR a ";" rx_comment failed. We hope.
self.LoadSurvexLeg(survexblock, sline, comment, svxline)
self.legsnumber = nlegstotal
self.slength = slengthtotal
self.legsnumber = nlegstotal
self.slength = slengthtotal
def PushdownStackScan(self, survexblock, path, fin, flinear, fcollate):
"""Follows the *include links in all the survex files from the root file 1623.svx
@ -1620,17 +1624,13 @@ def FindAndLoadSurvex(survexblockroot):
svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
svx_load.svxdirs[""] = survexfileroot.survexdirectory
# This next should be rewritten to use a generator so that only one
# line is held in memory at a time:
with open(collatefilename, "r") as fcollate:
svxlines = fcollate.read().splitlines()
#pr2 = cProfile.Profile()
#pr2.enable()
mem1 = get_process_memory()
print(f" - MEM:{mem1:7.2f} MB immediately after reading '{collatefilename}' into memory.",file=sys.stderr)
print(f" - MEM:{mem1:7.2f} MB NOT reading '{collatefilename}' into memory.",file=sys.stderr)
print(" ", file=sys.stderr,end='')
#----------------------------------------------------------------
svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines)
svx_load.LinearLoad(survexblockroot,survexfileroot.path, collatefilename)
#----------------------------------------------------------------
#pr2.disable()
# with open('LinearLoad.prof', 'w') as f: