diff --git a/parsers/survex.py b/parsers/survex.py index 406f08a..222c676 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -973,7 +973,7 @@ class LoadingSurvex(): if cave: survexfile.cave = cave - def LinearLoad(self, survexblock, path, svxlines): + def LinearLoad(self, survexblock, path, collatefilename): """Loads a single survex file. Usually used to import all the survex files which have been collated into a single file. Loads the begin/end blocks using a stack for labels. """ @@ -1003,7 +1003,7 @@ class LoadingSurvex(): if blockcount % 800 ==0 : print("\n", file=sys.stderr,end='') mem=get_process_memory() - print(" - MEM:{:7.3f} MB in use".format(mem),file=sys.stderr) + print(" - MEM: {:7.2f} MB in use".format(mem),file=sys.stderr) print(" ", file=sys.stderr,end='') sys.stderr.flush() @@ -1190,36 +1190,40 @@ class LoadingSurvex(): self.LoadSurvexFallThrough(survexblock, args, cmd) - for svxline in svxlines: - self.lineno += 1 - sline, comment = self.rx_comment.match(svxline).groups() - if comment: - # this catches the ;*include NEWFILE and ;*edulcni ENDOFFILE lines too - self.LoadSurvexComment(survexblock, comment) + # this is a python generator idiom. + # see https://realpython.com/introduction-to-python-generators/ + # this is the first use of generators in troggle (Oct.2022) + with open(collatefilename, "r") as fcollate: + for svxline in fcollate: + self.lineno += 1 + sline, comment = self.rx_comment.match(svxline).groups() + if comment: + # this catches the ;*include NEWFILE and ;*edulcni ENDOFFILE lines too + self.LoadSurvexComment(survexblock, comment) - if not sline: - continue # skip blank lines + if not sline: + continue # skip blank lines - # detect a merge failure inserted by version control - mfail = self.rx_badmerge.match(sline) - if mfail: - message = f"\n ! - ERROR version control merge failure\n - '{sline}'\n" - message = message + f" - line {self.lineno} in {blkid} in {survexblock}\n - NERD++ needed to fix it" - print(message) - print(message,file=sys.stderr) - DataIssue.objects.create(parser='survex', message=message) - continue # skip this line + # detect a merge failure inserted by version control + mfail = self.rx_badmerge.match(sline) + if mfail: + message = f"\n ! - ERROR version control merge failure\n - '{sline}'\n" + message = message + f" - line {self.lineno} in {blkid} in {survexblock}\n - NERD++ needed to fix it" + print(message) + print(message,file=sys.stderr) + DataIssue.objects.create(parser='survex', message=message) + continue # skip this line - # detect a star command - star = self.rx_star.match(sline) - if star: - # yes we are reading a *command - starstatement(star) - else: # not a *cmd so we are reading data OR a ";" rx_comment failed. We hope. - self.LoadSurvexLeg(survexblock, sline, comment, svxline) + # detect a star command + star = self.rx_star.match(sline) + if star: + # yes we are reading a *command + starstatement(star) + else: # not a *cmd so we are reading data OR a ";" rx_comment failed. We hope. + self.LoadSurvexLeg(survexblock, sline, comment, svxline) - self.legsnumber = nlegstotal - self.slength = slengthtotal + self.legsnumber = nlegstotal + self.slength = slengthtotal def PushdownStackScan(self, survexblock, path, fin, flinear, fcollate): """Follows the *include links in all the survex files from the root file 1623.svx @@ -1620,17 +1624,13 @@ def FindAndLoadSurvex(survexblockroot): svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot) svx_load.svxdirs[""] = survexfileroot.survexdirectory - # This next should be rewritten to use a generator so that only one - # line is held in memory at a time: - with open(collatefilename, "r") as fcollate: - svxlines = fcollate.read().splitlines() #pr2 = cProfile.Profile() #pr2.enable() mem1 = get_process_memory() - print(f" - MEM:{mem1:7.2f} MB immediately after reading '{collatefilename}' into memory.",file=sys.stderr) + print(f" - MEM:{mem1:7.2f} MB NOT reading '{collatefilename}' into memory.",file=sys.stderr) print(" ", file=sys.stderr,end='') #---------------------------------------------------------------- - svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines) + svx_load.LinearLoad(survexblockroot,survexfileroot.path, collatefilename) #---------------------------------------------------------------- #pr2.disable() # with open('LinearLoad.prof', 'w') as f: