From 699c19245abe816bfde8cfe1a6f5ffe6af2fef60 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Sat, 11 Nov 2023 14:36:03 +0200 Subject: [PATCH] regenerating .pos file more carefully --- core/utils.py | 3 +-- parsers/locations.py | 61 ++++++++++++++++++++++---------------------- settings.py | 3 --- 3 files changed, 32 insertions(+), 35 deletions(-) diff --git a/core/utils.py b/core/utils.py index 88a4b1b..99acfb3 100644 --- a/core/utils.py +++ b/core/utils.py @@ -264,8 +264,7 @@ def writetrogglefile(filepath, filecontent): filename = filepath.name git = settings.GIT - # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod. - # do not trap exceptions, pass them up to the view that called this function + # do not trap exceptions, pass them up to the view that called this function print(f"WRITING{cwd}---{filename} ") with open(filepath, "w") as f: f.write(filecontent) diff --git a/parsers/locations.py b/parsers/locations.py index aeadfff..0aa674e 100644 --- a/parsers/locations.py +++ b/parsers/locations.py @@ -211,12 +211,10 @@ def LoadPositions(): SurvexStation.objects.all().delete() - def runcavern3d(): + def runcavern3d(msg=None): + if msg: + print(" - ", msg) outputdir = Path(str(f"{topdata}.svx")).parent - - # print(" - Regenerating stale cavern .log and .3d for '{}'\n days old: {:.1f} {:.1f} {:.1f}". - # format(topdata, (svx_t - d3d_t)/(24*3600), (cav_t - d3d_t)/(24*3600), (now - d3d_t)/(24*3600))) - file3d = Path(f"{topdata}.3d") try: sp = subprocess.run( @@ -230,7 +228,7 @@ def LoadPositions(): stash_data_issue(parser="positions", message=message) print(message) - # find the errors in the 1623.log file + # find the errors in the .log file sp = subprocess.run( ["grep", "error:", f"{topdata}.log"], capture_output=True, check=False, text=True ) # check=False means exception not raised @@ -248,9 +246,8 @@ def LoadPositions(): stash_data_issue(parser="positions", message=message) print(message) - if file3d.is_file(): # might be an old one though + if file3d.is_file(): # might be an old one though, if previous step failed try: - # print(" - Regenerating {} {}.3d in {}".format(settings.SURVEXPORT, topdata, settings.SURVEX_DATA)) sp = subprocess.run( [settings.SURVEXPORT, "--pos", f"{file3d}"], cwd=settings.SURVEX_DATA, @@ -274,7 +271,7 @@ def LoadPositions(): stash_data_issue(parser="positions", message=message) print(message) - topdata = os.fspath(Path(settings.SURVEX_DATA) / settings.SURVEX_TOPNAME) + topdata = Path(settings.SURVEX_DATA, settings.SURVEX_TOPNAME) print(f" - Generating a list of Pos from {topdata}.3d and then loading...") found = 0 @@ -282,29 +279,33 @@ def LoadPositions(): # cavern defaults to using same cwd as supplied input file completed_process = subprocess.run(["which", f"{settings.CAVERN}"], capture_output=True, check=True, text=True) - cav_t = os.path.getmtime(completed_process.stdout.strip()) + cav_t = Path(completed_process.stdout.strip()).stat().st_mtime + cav_d = datetime.fromtimestamp(cav_t).strftime('%d %b %Y %H:%M:%S') - svxpath = topdata + ".svx" - d3dpath = topdata + ".3d" - pospath = topdata + ".pos" + svxpath = topdata.with_suffix(".svx") + d3dpath = topdata.with_suffix(".3d") + pospath = topdata.with_suffix(".pos") + + if not settings.DEVSERVER: + runcavern3d(f"Regen - on server {settings.DEVSERVER=}") # always regenerate .3d and .pos on the server + else: + # These basic tests fail to capture the case where a *included svx file has changed, + # typically this is one of the fixedpts *fix files. + for p in [pospath, d3dpath]: + if not p.is_file(): + runcavern3d(f"Creating {p}.3d, .pos") + svx_t = svxpath.stat().st_mtime + d3d_t = d3dpath.stat().st_mtime # os.path.getmtime(d3dpath) + svx_d = datetime.fromtimestamp(svx_t).strftime('%d %b %Y %H:%M:%S') + d3d_d = datetime.fromtimestamp(d3d_t).strftime('%d %b %Y %H:%M:%S') - svx_t = os.path.getmtime(svxpath) - - if os.path.isfile(d3dpath): - # always fails to find log file if a double directory, e.g. caves-1623/B4/B4/B4.svx Why ? - d3d_t = os.path.getmtime(d3dpath) - - now = time.time() - if not os.path.isfile(pospath): - runcavern3d() - if not os.path.isfile(d3dpath): - runcavern3d() - elif d3d_t - svx_t > 0: # stale, 3d older than svx file - runcavern3d() - elif now - d3d_t > 24 * 60 * 60: # >1 days old, re-run anyway - runcavern3d() - elif cav_t - d3d_t > 0: # new version of cavern - runcavern3d() + now = time.time() + if d3d_t - svx_t < 0: # stale, 3d older than svx file . But .svx timestamp does not reflect *include timestamps + runcavern3d(f"Regen - stale {d3d_d} earlier than {svx_d}") + elif now - d3d_t > 24 * 60 * 60: # >1 days old, re-run anyway + runcavern3d(f"Regen - old") + elif d3d_t - cav_t < 0: # new version of cavern + runcavern3d(f"Regen - new survex version {d3d_d} earlier than {cav_d} ") mappoints = {} found_points = {} diff --git a/settings.py b/settings.py index 80a1de2..67c006a 100644 --- a/settings.py +++ b/settings.py @@ -42,9 +42,6 @@ MOGRIFY = "mogrify" # for rotating images # django/conf/global_settings.py which is automatically loaded first. # read https://docs.djangoproject.com/en/dev/topics/settings/ -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -# BASE_DIR = os.path.dirname(os.path.dirname(__file__)) - # Django settings for troggle project. ALLOWED_HOSTS = ["*", "expo.survex.com", ".survex.com", "localhost", "127.0.0.1", "192.168.0.5"]