running cavern on svx files improved

core/views/survex.py

@@ -102,6 +102,9 @@ survextemplatefile = """; *** THIS IS A TEMPLATE FILE NOT WHAT YOU MIGHT BE EXPE
 
 
 class SvxForm(forms.Form):
+    '''Two-pane form, upper half is the raw survex file, lower half (with green background)
+    is the output of running 'cavern' on the survex file.
+    '''
     dirname = forms.CharField(widget=forms.TextInput(attrs={"readonly":True}))
     filename = forms.CharField(widget=forms.TextInput(attrs={"readonly":True}))
     datetime = forms.DateTimeField(widget=forms.TextInput(attrs={"readonly":True}))
@@ -112,11 +115,11 @@ class SvxForm(forms.Form):
     
     def GetDiscCode(self):
         fname = survexdatasetpath / (self.data['filename'] + ".svx")
-        if not os.path.isfile(fname):
+        if not fname.is_file():
             print(">>> >>> WARNING - svx file not found, showing TEMPLATE SVX",fname, flush=True)
             self.template = True
             return survextemplatefile
-        fin = open(fname, "rt",encoding='utf8',newline='')
+        fin = open(fname, "r",encoding='utf8',newline='')
         svxtext = fin.read() 
         fin.close()
         return svxtext
@@ -140,13 +143,13 @@ class SvxForm(forms.Form):
 
         # Make this create new survex folders if needed
         try:
-            fout = open(fname, "wt", encoding='utf8',newline='\n')
+            fout = open(fname, "w", encoding='utf8',newline='\n')
         except FileNotFoundError:
             pth = os.path.dirname(self.data['filename'])
             newpath = survexdatasetpath / pth
             if not os.path.exists(newpath):
                 os.makedirs(newpath) 
-            fout = open(fname, "wt", encoding='utf8',newline='\n')
+            fout = open(fname, "w", encoding='utf8',newline='\n')
         except PermissionError:
             return "CANNOT  save this file.\nPERMISSIONS incorrectly set on server for this file. Ask a nerd to fix this."
 
@@ -155,22 +158,45 @@ class SvxForm(forms.Form):
         res = fout.write("\n")
         fout.close()
         
-        # INSERT code to do git add and commit here, to loser repo.
+        # INSERT code to do git add and commit here, to loser repo. When Wookey chnages :loser: to use git.
         
-        return "SAVED ."
+        return "SAVED"
 
     def Process(self):
-        print(">>>>....\n\n\n....Processing\n\n\n")
+        print(">>>>....\n....Processing\n")
         froox = os.fspath(survexdatasetpath / (self.data['filename'] + ".svx"))
         froog = os.fspath(survexdatasetpath / (self.data['filename'] + ".log"))
         cwd = os.getcwd()
         os.chdir(os.path.split(froox)[0])
         os.system(settings.CAVERN + " --log " + froox )
         os.chdir(cwd)
-        fin = open(froog, "rt",encoding='utf8')
+   
+    # Update this to use the new syntax..
+    # sp = subprocess.run([settings.CAVERN, "--log", f'--output={outputdir}', f'{fullpath}.svx'],
+                            # capture_output=True, check=False, text=True)
+    # if sp.returncode != 0:
+        # message = f' ! Error running {settings.CAVERN}: {fullpath}' 
+        # DataIssue.objects.create(parser='entrances', message=message)
+        # print(message)
+        # print(f'stderr:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout)  + '\n\nreturn code: ' + str(sp.returncode))
+
+        filepatherr = Path(survexdatasetpath / str(self.data['filename'] + ".err"))
+        if filepatherr.is_file():
+            if filepatherr.stat().st_size == 0:
+                filepatherr.unlink() # delete empty closure error file
+
+        fin = open(froog, "r",encoding='utf8')
         log = fin.read()
         fin.close()
-        log = re.sub("(?s).*?(Survey contains)", "\\1", log)
+        #log = re.sub("(?s).*?(Survey contains)", "\\1", log) # this omits any ERROR MESSAGES ! Don't do it.
+        for s in ["Removing trailing traverses...\n\n",
+            "Concatenating traverses...\n\n"
+            "Simplifying network...\n\n",
+            "Calculating network...\n\n",
+            "Calculating traverses...\n\n",
+            "Calculating trailing traverses...\n\n",
+            "Calculating statistics...\n\n"]:
+            log = log.replace(s,"")
         return log
 
 
@@ -178,7 +204,8 @@ class SvxForm(forms.Form):
 def svx(request, survex_file):
     '''Displays a single survex file in an textarea window (using a javascript online editor to enable 
     editing) with buttons which allow SAVE, check for DIFFerences from saved, and RUN (which runs the
-    cavern executable and displays the output below the main textarea). Requires CSRF to be set upcorrect;ly, and requires permission to write to  the filesystem.
+    cavern executable and displays the output below the main textarea). 
+    Requires CSRF to be set up correct;ly, and requires permission to write to  the filesystem.
     '''
     warning = False
     
@@ -206,9 +233,9 @@ def svx(request, survex_file):
                 pass
             if "process" in rform.data:
                 if not difflist:
-                    message = "OUTPUT FROM PROCESSING"
                     logmessage = form.Process()
-                    print(logmessage)
+                    if logmessage:
+                       message = f"OUTPUT FROM PROCESSING\n{logmessage}"
                 else:
                     message = "SAVE FILE FIRST"
                     form.data['code'] = rcode
@@ -220,6 +247,7 @@ def svx(request, survex_file):
                 if message != "SAVED":
                     form.data['code'] = rcode
             if "diff" in rform.data:
+                print("Differences: ")
                 form.data['code'] = rcode
     
     #process(survex_file)
@@ -228,7 +256,7 @@ def svx(request, survex_file):
         if form.template:
             warning = True
     if not difflist:
-        difflist.append("none")
+        difflist.append("No differences - file was saved")
     if message:
         difflist.insert(0, message)
     
@@ -237,7 +265,7 @@ def svx(request, survex_file):
     
     vmap = {'settings': settings,
             'warning': warning,
-            'has_3d': os.path.isfile(survexdatasetpath / survex_file / ".3d"),
+            'has_3d': (Path(survexdatasetpath) / Path(survex_file + ".3d")).is_file(),
             'title': survex_file,
             'svxincludes': svxincludes,
             'difflist': difflist,
@@ -250,40 +278,59 @@ def svx(request, survex_file):
         
     return render(request, 'svxfile.html', vmap)
 
-def svxraw(request, survex_file):
+# The cavern running function. This is NOT where it is run inside the form! see SvxForm.Process() for that
-    '''Used for rendering .log files from survex outputtype'''
-    svx = open(os.path.join(survexdatasetpath /  survex_file / ".svx"), "rt",encoding='utf8')
-    return HttpResponse(svx, content_type="text")
-
-
-# The cavern running function
 def process(survex_file):
+    '''This runs cavern only where a .3d, .log or .err file is requested.
+    '''
+    filepathsvx = survexdatasetpath /  str( survex_file + ".svx")
     cwd = os.getcwd()
     os.chdir(os.path.split(os.fspath(survexdatasetpath /  survex_file))[0])
-    os.system(settings.CAVERN + " --log " + survexdatasetpath /  survex_file / ".svx")
+    os.system(settings.CAVERN + " --log " + str(filepathsvx))
     os.chdir(cwd)
     
+    # Update this to use the new syntax..
+    # sp = subprocess.run([settings.CAVERN, "--log", f'--output={outputdir}', f'{fullpath}.svx'],
+                            # capture_output=True, check=False, text=True)
+    # if sp.returncode != 0:
+        # message = f' ! Error running {settings.CAVERN}: {fullpath}' 
+        # DataIssue.objects.create(parser='entrances', message=message)
+        # print(message)
+        # print(f'stderr:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout)  + '\n\nreturn code: ' + str(sp.returncode))
+
+    filepatherr = Path(survexdatasetpath / str(survex_file + ".err"))
+    if filepatherr.is_file():
+        if filepatherr.stat().st_size == 0:
+            filepatherr.unlink() # delete empty closure error file
         
 def threed(request, survex_file):
-    process(survex_file)
+    filepath3d  = survexdatasetpath /  str(survex_file + ".3d")
-    try:
+    filepathlog = survexdatasetpath /  str(survex_file + ".log")
-        threed = open(survexdatasetpath /  survex_file / ".3d", "rt",encoding='utf8')
+    if filepath3d.is_file():
-        return HttpResponse(threed, content_type="model/3d")
+        threed = open(filepath3d, "rb")
-    except:
+        return HttpResponse(threed, content_type="application/x-aven")
-        log = open(survexdatasetpath /  survex_file / ".log", "rt",encoding='utf8')
+    else:
+        process(survex_file) # should not need to do this if it already exists, as it should.
+        log = open(survexdatasetpath /  str(survex_file + ".log"), "r",encoding='utf-8')
         return HttpResponse(log, content_type="text")
 
-
+def svxlog(request, survex_file):
-def log(request, survex_file):
+    '''Used for rendering .log files from survex outputtype'''
-    process(survex_file)
+    filepathlog = survexdatasetpath /  str(survex_file + ".log")
-    log = open(survexdatasetpath /  survex_file / ".log", "rt",encoding='utf8')
+    if not filepathlog.is_file():
-    return HttpResponse(log, content_type="text")
+       process(survex_file)
-
+    log = open(filepathlog, "r")
+    return HttpResponse(log,content_type="text/plain; charset=utf-8") #default: "text/html; charset=utf-8"
 
 def err(request, survex_file):
+    filepatherr = survexdatasetpath /  str(survex_file + ".err")
+    if not filepatherr.is_file(): # probably not there because it was empty, but re-run anyway
+       process(survex_file)
     process(survex_file)
-    err = open(survexdatasetpath /  survex_file / ".err", "rt",encoding='utf8')
+    if filepatherr.is_file():
-    return HttpResponse(err, content_type="text")
+        err = open(filepatherr, "r")
+        return HttpResponse(err, content_type="text/plain; charset=utf-8")
+    else:
+        return HttpResponse(f'No closure errors. \nEmpty {filepatherr} file produced. \nSee the .log file.', content_type="text/plain; charset=utf-8")
 
 
 def identifycavedircontents(gcavedir):

parsers/survex.py

@@ -1221,9 +1221,15 @@ class LoadingSurvex():
             self.caverncount += 1
             
             # should also collect all the .err files too and create a DataIssue for each one which 
-            # - is nonzero in size
+            # - is nonzero in size AND
             # - has Error greater than 5% anywhere, or some other more serious error
             
+            errpath = Path(fullpath + ".err")
+            if errpath.is_file():
+                if errpath.stat().st_size == 0:
+                    errpath.unlink() # delete empty closure error file
+
+
         svxpath = Path(fullpath + ".svx")
         logpath = Path(fullpath + ".log")
         outputdir = Path(svxpath).parent

templates/svxfile.html

@@ -3,7 +3,6 @@
 {% block title %}{{ title }}{% endblock %}
 
 {% block head %}
-<script type="text/javascript" src="{{settings.MEDIA_URL }}js/base.js"></script>
 <!--  Should we wait until CodeMirror 6.0 is available https://codemirror.net/6/ -->
 <!--  <script type="text/javascript" src="{{settings.JSLIB_URL}}jquery-form/jquery.form.min.js"></script> <!-- INVALID-->
 <!--  <script type="text/javascript" src="{{settings.JSLIB_URL}}codemirror/codemirror.min.js"></script>   <!-- INVALID-->
@@ -69,14 +68,16 @@ $(document).ready(function()
 
 {% if logmessage %}
     {% if has_3d %}
-        <p><a href='{% url "threed" title %}'>3d file</a></p>
+        <p>Click here to view the <a href='{% url "threed" title %}'>3d file</a> produced as output</p>
     {% else %}
-        <p><b>No 3d file</b></p>
+        <p><b>No 3d file generated. There was an ERROR<br>Read the full <a href='{% url "svxlog" title %}'>.log file</a>
+        the <a href='{% url "err" title %}'>.err file</a> and fix this .svx file</b></p>
     {% endif %}
-    <pre>
+<!--    <pre>
-    LOGMESSAGES
+LOGMESSAGES
-    {{logmessage}}
+{{logmessage}}
     </pre>
+    -->
 {% endif %}
 </div>
 

templates/svxfiledifflistonly.html

@@ -8,7 +8,7 @@
     {% if has_3d %}
         <p><a href='{% url "threed" title %}'>3d file</a></p>
     {% else %}
-        <p><b>No 3d file</b></p>
+        <p><b>No 3d file generated. ERROR. Read the full .log file and fix the .svx file</b></p>
     {% endif %}
     <pre>
     LOGMESSAGES

urls.py

@@ -154,7 +154,7 @@ trogglepatterns = [
 
     path('survexfile/<path:survex_file>.svx', survex.svx,        name="svx"), 
     path('survexfile/<path:survex_file>.3d',  survex.threed,     name="threed"), 
-    path('survexfile/<path:survex_file>.log', survex.svxraw,     name="svxraw"),  
+    path('survexfile/<path:survex_file>.log', survex.svxlog,     name="svxlog"),  
     path('survexfile/<path:survex_file>.err', survex.err,        name="err"), 
     path('survexfile/<path:survex_cave>',     survex.survexcavesingle,    name="survexcavessingle"),