From 13705244792e5f0a617125ba86cd56b245d13fda Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Thu, 25 Sep 2025 22:18:55 +0300 Subject: [PATCH] using io buffer to speedup, but it doesnt --- parsers/survex.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/parsers/survex.py b/parsers/survex.py index 00afad5..7e88afa 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1,4 +1,5 @@ import copy +import io import os import re import subprocess @@ -2010,7 +2011,7 @@ class LoadingSurvex: self.legsnumber = nlegstotal self.slength = slengthtotal - def PushdownStackScan(self, survexblock, path, finname, flinear, fcollate): + def PushdownStackScan(self, survexblock, path, finname, flinear, io_collate): """Follows the *include links in all the survex files from the root file (usually 1623.svx) and reads only the *include and *begin and *end statements. It produces a linearised list of the include tree and detects blocks included more than once. @@ -2034,7 +2035,7 @@ class LoadingSurvex: includestmt = self.rx_include.match(svxline) if not includestmt: - fcollate.write(f"{svxline.strip()}\n") + io_collate.write(f"{svxline.strip()}\n") sline, comment = self.rx_comment.match(svxline.strip()).groups() star = self.rx_star.match(sline) @@ -2053,12 +2054,12 @@ class LoadingSurvex: self.depthinclude += 1 # fininclude = open(fullpath,'r') finincludename = fullpath - fcollate.write(f";|*include {includepath}\n") + io_collate.write(f";|*include {includepath}\n") flinear.write(f"{self.depthinclude:2} {indent} *include {includepath}\n") push = includepath.lower() self.includestack.append(push) # ----------------- - self.PushdownStackScan(survexblock, includepath, finincludename, flinear, fcollate) + self.PushdownStackScan(survexblock, includepath, finincludename, flinear, io_collate) # ----------------- pop = self.includestack.pop() if pop != push: @@ -2068,7 +2069,7 @@ class LoadingSurvex: print(message, file=sys.stderr) stash_data_issue(parser="survex", message=message, url=None, sb=(path)) flinear.write(f"{self.depthinclude:2} {indent} *edulcni {pop}\n") - fcollate.write(f";|*edulcni {pop}\n") + io_collate.write(f";|*edulcni {pop}\n") # fininclude.close() self.depthinclude -= 1 # -------------------------------------------------------- @@ -2361,7 +2362,8 @@ def FindAndLoadSurvex(): svx_scan.uniquefile[str(survexfileroot)] = ["0"] indent = "" - fcollate = open(collatefilename, "w") + #fcollate = open(collatefilename, "w") + io_collate = io.StringIO() mem0 = get_process_memory() print(f" - MEM:{mem0:7.2f} MB START '{survexfileroot}'", file=sys.stderr) @@ -2370,7 +2372,7 @@ def FindAndLoadSurvex(): print(" ", file=sys.stderr, end="") finrootname = Path(settings.SURVEX_DATA, survexfileroot.path + ".svx") - fcollate.write(f";*include {survexfileroot.path}\n") + io_collate.write(f";*include {survexfileroot.path}\n") flinear.write(f"{svx_scan.depthinclude:2} {indent} *include {survexfileroot.path}\n") # import cProfile @@ -2381,7 +2383,7 @@ def FindAndLoadSurvex(): # pr.enable() svx_scan.svxpass = svx_scan.TREE # ---------------------------------------------------------------- - svx_scan.PushdownStackScan(survexblockroot, survexfileroot.path, finrootname, flinear, fcollate) + svx_scan.PushdownStackScan(survexblockroot, survexfileroot.path, finrootname, flinear, io_collate) # ---------------------------------------------------------------- svx_scan.svxpass = "" # pr.disable() @@ -2391,7 +2393,7 @@ def FindAndLoadSurvex(): # ps.print_stats() flinear.write(f"{svx_scan.depthinclude:2} {indent} *edulcni {survexfileroot.path}\n") - fcollate.write(f";*edulcni {survexfileroot.path}\n") + io_collate.write(f";*edulcni {survexfileroot.path}\n") mem1 = get_process_memory() flinear.write(f"\n - MEM:{mem1:.2f} MB STOP {survexfileroot.path}\n") flinear.write(f" - MEM:{mem1 - mem0:.3f} MB ADDITIONALLY USED\n") @@ -2496,16 +2498,16 @@ def FindAndLoadSurvex(): # this is a bit tricky as some unseen files will *include files we have already seen, which # we should not process again. finrootname = fullpathtotop - fcollate.write(f";*include {UNSEENS}\n") + io_collate.write(f";*include {UNSEENS}\n") flinear.write(f"{omit_scan.depthinclude:2} {indent} *include {unseensroot}\n") omit_scan.svxpass = omit_scan.ODDS # ---------------------------------------------------------------- - omit_scan.PushdownStackScan(survexblockroot, unseensroot, finrootname, flinear, fcollate) + omit_scan.PushdownStackScan(survexblockroot, unseensroot, finrootname, flinear, io_collate) # ---------------------------------------------------------------- omit_scan.svxpass = "" flinear.write(f"{omit_scan.depthinclude:2} {indent} *edulcni {unseensroot}\n") - fcollate.write(f";*edulcni {UNSEENS}\n") + io_collate.write(f";*edulcni {UNSEENS}\n") mem1 = get_process_memory() flinear.write(f"\n - MEM:{mem1:.2f} MB STOP {UNSEENS} Unseen Oddments\n") @@ -2513,7 +2515,10 @@ def FindAndLoadSurvex(): flinear.write(f" - {len(omit_scan.svxfileslist):,} survex files in linear include list Unseen Oddments \n") flinear.close() - fcollate.close() + # fcollate.close() + with open(collatefilename, "w", encoding="utf-8") as fcollate_f: + fcollate_f.write(io_collate.getvalue()) + io_collate.close() print( f"\n - {omit_scan.caverncount:,} runs of survex 'cavern' refreshing .3d files in the unseen list",