regenerating .pos file more carefully

core/utils.py

@@ -264,8 +264,7 @@ def writetrogglefile(filepath, filecontent):
     filename = filepath.name
     git = settings.GIT
 
-    # as the wsgi process www-data, we have group write-access but are not owner, so cannot chmod.
+   # do not trap exceptions, pass them up to the view that called this function
-    # do not trap exceptions, pass them up to the view that called this function
     print(f"WRITING{cwd}---{filename} ")
     with open(filepath, "w") as f:
         f.write(filecontent)

parsers/locations.py

@@ -211,12 +211,10 @@ def LoadPositions():
     SurvexStation.objects.all().delete()
 
 
-    def runcavern3d():
+    def runcavern3d(msg=None):
+        if msg:
+            print("  - ", msg)
         outputdir = Path(str(f"{topdata}.svx")).parent
-
-        # print(" -  Regenerating stale cavern .log and .3d for '{}'\n    days old: {:.1f}    {:.1f}    {:.1f}".
-        #    format(topdata, (svx_t - d3d_t)/(24*3600), (cav_t - d3d_t)/(24*3600), (now - d3d_t)/(24*3600)))
-
         file3d = Path(f"{topdata}.3d")
         try:
             sp = subprocess.run(
@@ -230,7 +228,7 @@ def LoadPositions():
                 stash_data_issue(parser="positions", message=message)
                 print(message)
 
-                # find the errors in the 1623.log file
+                # find the errors in the .log file
                 sp = subprocess.run(
                     ["grep", "error:", f"{topdata}.log"], capture_output=True, check=False, text=True
                 )  # check=False means exception not raised
@@ -248,9 +246,8 @@ def LoadPositions():
                 stash_data_issue(parser="positions", message=message)
                 print(message)
 
-        if file3d.is_file():  # might be an old one though
+        if file3d.is_file():  # might be an old one though, if previous step failed
             try:
-                # print(" -  Regenerating {} {}.3d  in  {}".format(settings.SURVEXPORT, topdata, settings.SURVEX_DATA))
                 sp = subprocess.run(
                     [settings.SURVEXPORT, "--pos", f"{file3d}"],
                     cwd=settings.SURVEX_DATA,
@@ -274,7 +271,7 @@ def LoadPositions():
             stash_data_issue(parser="positions", message=message)
             print(message)
 
-    topdata = os.fspath(Path(settings.SURVEX_DATA) / settings.SURVEX_TOPNAME)
+    topdata = Path(settings.SURVEX_DATA, settings.SURVEX_TOPNAME)
     print(f" - Generating a list of Pos from {topdata}.3d and then loading...")
 
     found = 0
@@ -282,29 +279,33 @@ def LoadPositions():
     # cavern defaults to using same cwd as supplied input file
 
     completed_process = subprocess.run(["which", f"{settings.CAVERN}"], capture_output=True, check=True, text=True)
-    cav_t = os.path.getmtime(completed_process.stdout.strip())
+    cav_t = Path(completed_process.stdout.strip()).stat().st_mtime
+    cav_d = datetime.fromtimestamp(cav_t).strftime('%d %b %Y %H:%M:%S')
 
-    svxpath = topdata + ".svx"
+    svxpath = topdata.with_suffix(".svx")
-    d3dpath = topdata + ".3d"
+    d3dpath = topdata.with_suffix(".3d")
-    pospath = topdata + ".pos"
+    pospath = topdata.with_suffix(".pos")
     
-    svx_t = os.path.getmtime(svxpath)
+    if not settings.DEVSERVER:
+        runcavern3d(f"Regen - on server {settings.DEVSERVER=}") # always regenerate .3d and .pos on the server
+    else:
+        # These basic tests fail to capture the case where a *included svx file has changed,
+        # typically this is one of the fixedpts *fix files.
+        for p in [pospath, d3dpath]:
+            if not p.is_file():
+                runcavern3d(f"Creating {p}.3d, .pos")
+        svx_t = svxpath.stat().st_mtime
+        d3d_t = d3dpath.stat().st_mtime # os.path.getmtime(d3dpath)
+        svx_d = datetime.fromtimestamp(svx_t).strftime('%d %b %Y %H:%M:%S')
+        d3d_d = datetime.fromtimestamp(d3d_t).strftime('%d %b %Y %H:%M:%S')
 
-    if os.path.isfile(d3dpath):
+        now = time.time()
-        # always fails to find log file if a double directory, e.g. caves-1623/B4/B4/B4.svx Why ?
+        if d3d_t - svx_t < 0:  # stale, 3d older than svx file . But .svx timestamp does not reflect *include timestamps
-        d3d_t = os.path.getmtime(d3dpath)
+            runcavern3d(f"Regen - stale {d3d_d} earlier than {svx_d}")
-
+        elif now - d3d_t > 24 * 60 * 60:  # >1 days old, re-run anyway
-    now = time.time()
+            runcavern3d(f"Regen - old")
-    if not os.path.isfile(pospath):
+        elif d3d_t - cav_t < 0:  # new version of cavern
-        runcavern3d()
+            runcavern3d(f"Regen - new survex version {d3d_d} earlier than {cav_d} ")
-    if not os.path.isfile(d3dpath):
-        runcavern3d()
-    elif d3d_t - svx_t > 0:  # stale, 3d older than svx file 
-        runcavern3d()
-    elif now - d3d_t > 24 * 60 * 60:  # >1 days old, re-run anyway
-        runcavern3d()
-    elif cav_t - d3d_t > 0:  # new version of cavern
-        runcavern3d()
 
     mappoints = {}
     found_points = {}

settings.py

@@ -42,9 +42,6 @@ MOGRIFY = "mogrify" # for rotating images
 # django/conf/global_settings.py which is automatically loaded first.
 # read https://docs.djangoproject.com/en/dev/topics/settings/
 
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-# BASE_DIR = os.path.dirname(os.path.dirname(__file__))
-
 # Django settings for troggle project.
 
 ALLOWED_HOSTS = ["*", "expo.survex.com", ".survex.com", "localhost", "127.0.0.1", "192.168.0.5"]