expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-29 05:11:52 +00:00
Code Issues 19 Projects 3 Activity

reduce mem use by 21.2MB by using a generator

Browse Source
This commit is contained in:
Philip Sargent 2022-10-07 10:57:30 +03:00
parent d16226c879
commit b4c4f2aefc
1 changed files with 34 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
parsers/survex.py
View File

@ -973,7 +973,7 @@ class LoadingSurvex():
if cave: if cave:
survexfile.cave = cave survexfile.cave = cave
def LinearLoad(self, survexblock, path, svxlines): def LinearLoad(self, survexblock, path, collatefilename):
"""Loads a single survex file. Usually used to import all the survex files which have been collated """Loads a single survex file. Usually used to import all the survex files which have been collated
into a single file. Loads the begin/end blocks using a stack for labels. into a single file. Loads the begin/end blocks using a stack for labels.
""" """
@ -1003,7 +1003,7 @@ class LoadingSurvex():
if blockcount % 800 ==0 : if blockcount % 800 ==0 :
print("\n", file=sys.stderr,end='') print("\n", file=sys.stderr,end='')
mem=get_process_memory() mem=get_process_memory()
print(" - MEM:{:7.3f} MB in use".format(mem),file=sys.stderr) print(" - MEM: {:7.2f} MB in use".format(mem),file=sys.stderr)
print(" ", file=sys.stderr,end='') print(" ", file=sys.stderr,end='')
sys.stderr.flush() sys.stderr.flush()
@ -1190,36 +1190,40 @@ class LoadingSurvex():
self.LoadSurvexFallThrough(survexblock, args, cmd) self.LoadSurvexFallThrough(survexblock, args, cmd)
for svxline in svxlines: # this is a python generator idiom.
self.lineno += 1 # see https://realpython.com/introduction-to-python-generators/
sline, comment = self.rx_comment.match(svxline).groups() # this is the first use of generators in troggle (Oct.2022)
if comment: with open(collatefilename, "r") as fcollate:
# this catches the ;*include NEWFILE and ;*edulcni ENDOFFILE lines too for svxline in fcollate:
self.LoadSurvexComment(survexblock, comment) self.lineno += 1
sline, comment = self.rx_comment.match(svxline).groups()
if comment:
# this catches the ;*include NEWFILE and ;*edulcni ENDOFFILE lines too
self.LoadSurvexComment(survexblock, comment)
if not sline: if not sline:
continue # skip blank lines continue # skip blank lines
# detect a merge failure inserted by version control # detect a merge failure inserted by version control
mfail = self.rx_badmerge.match(sline) mfail = self.rx_badmerge.match(sline)
if mfail: if mfail:
message = f"\n ! - ERROR version control merge failure\n - '{sline}'\n" message = f"\n ! - ERROR version control merge failure\n - '{sline}'\n"
message = message + f" - line {self.lineno} in {blkid} in {survexblock}\n - NERD++ needed to fix it" message = message + f" - line {self.lineno} in {blkid} in {survexblock}\n - NERD++ needed to fix it"
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message)
continue # skip this line continue # skip this line
# detect a star command # detect a star command
star = self.rx_star.match(sline) star = self.rx_star.match(sline)
if star: if star:
# yes we are reading a *command # yes we are reading a *command
starstatement(star) starstatement(star)
else: # not a *cmd so we are reading data OR a ";" rx_comment failed. We hope. else: # not a *cmd so we are reading data OR a ";" rx_comment failed. We hope.
self.LoadSurvexLeg(survexblock, sline, comment, svxline) self.LoadSurvexLeg(survexblock, sline, comment, svxline)
self.legsnumber = nlegstotal self.legsnumber = nlegstotal
self.slength = slengthtotal self.slength = slengthtotal
def PushdownStackScan(self, survexblock, path, fin, flinear, fcollate): def PushdownStackScan(self, survexblock, path, fin, flinear, fcollate):
"""Follows the *include links in all the survex files from the root file 1623.svx """Follows the *include links in all the survex files from the root file 1623.svx
@ -1620,17 +1624,13 @@ def FindAndLoadSurvex(survexblockroot):
svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot) svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
svx_load.svxdirs[""] = survexfileroot.survexdirectory svx_load.svxdirs[""] = survexfileroot.survexdirectory
# This next should be rewritten to use a generator so that only one
# line is held in memory at a time:
with open(collatefilename, "r") as fcollate:
svxlines = fcollate.read().splitlines()
#pr2 = cProfile.Profile() #pr2 = cProfile.Profile()
#pr2.enable() #pr2.enable()
mem1 = get_process_memory() mem1 = get_process_memory()
print(f" - MEM:{mem1:7.2f} MB immediately after reading '{collatefilename}' into memory.",file=sys.stderr) print(f" - MEM:{mem1:7.2f} MB NOT reading '{collatefilename}' into memory.",file=sys.stderr)
print(" ", file=sys.stderr,end='') print(" ", file=sys.stderr,end='')
#---------------------------------------------------------------- #----------------------------------------------------------------
svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines) svx_load.LinearLoad(survexblockroot,survexfileroot.path, collatefilename)
#---------------------------------------------------------------- #----------------------------------------------------------------
#pr2.disable() #pr2.disable()
# with open('LinearLoad.prof', 'w') as f: # with open('LinearLoad.prof', 'w') as f: