From f99ebf84e9da3a6f65a586977a43af992d6157a6 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Fri, 11 Mar 2022 16:22:37 +0000 Subject: [PATCH] running cavern on svx files improved --- core/views/survex.py | 121 ++++++++++++++++++++--------- parsers/survex.py | 8 +- templates/svxfile.html | 13 ++-- templates/svxfiledifflistonly.html | 2 +- urls.py | 2 +- 5 files changed, 100 insertions(+), 46 deletions(-) diff --git a/core/views/survex.py b/core/views/survex.py index 47ccad7..c0f63ef 100644 --- a/core/views/survex.py +++ b/core/views/survex.py @@ -102,6 +102,9 @@ survextemplatefile = """; *** THIS IS A TEMPLATE FILE NOT WHAT YOU MIGHT BE EXPE class SvxForm(forms.Form): + '''Two-pane form, upper half is the raw survex file, lower half (with green background) + is the output of running 'cavern' on the survex file. + ''' dirname = forms.CharField(widget=forms.TextInput(attrs={"readonly":True})) filename = forms.CharField(widget=forms.TextInput(attrs={"readonly":True})) datetime = forms.DateTimeField(widget=forms.TextInput(attrs={"readonly":True})) @@ -112,11 +115,11 @@ class SvxForm(forms.Form): def GetDiscCode(self): fname = survexdatasetpath / (self.data['filename'] + ".svx") - if not os.path.isfile(fname): + if not fname.is_file(): print(">>> >>> WARNING - svx file not found, showing TEMPLATE SVX",fname, flush=True) self.template = True return survextemplatefile - fin = open(fname, "rt",encoding='utf8',newline='') + fin = open(fname, "r",encoding='utf8',newline='') svxtext = fin.read() fin.close() return svxtext @@ -140,13 +143,13 @@ class SvxForm(forms.Form): # Make this create new survex folders if needed try: - fout = open(fname, "wt", encoding='utf8',newline='\n') + fout = open(fname, "w", encoding='utf8',newline='\n') except FileNotFoundError: pth = os.path.dirname(self.data['filename']) newpath = survexdatasetpath / pth if not os.path.exists(newpath): os.makedirs(newpath) - fout = open(fname, "wt", encoding='utf8',newline='\n') + fout = open(fname, "w", encoding='utf8',newline='\n') except PermissionError: return "CANNOT save this file.\nPERMISSIONS incorrectly set on server for this file. Ask a nerd to fix this." @@ -155,22 +158,45 @@ class SvxForm(forms.Form): res = fout.write("\n") fout.close() - # INSERT code to do git add and commit here, to loser repo. + # INSERT code to do git add and commit here, to loser repo. When Wookey chnages :loser: to use git. - return "SAVED ." + return "SAVED" def Process(self): - print(">>>>....\n\n\n....Processing\n\n\n") + print(">>>>....\n....Processing\n") froox = os.fspath(survexdatasetpath / (self.data['filename'] + ".svx")) froog = os.fspath(survexdatasetpath / (self.data['filename'] + ".log")) cwd = os.getcwd() os.chdir(os.path.split(froox)[0]) os.system(settings.CAVERN + " --log " + froox ) os.chdir(cwd) - fin = open(froog, "rt",encoding='utf8') + + # Update this to use the new syntax.. + # sp = subprocess.run([settings.CAVERN, "--log", f'--output={outputdir}', f'{fullpath}.svx'], + # capture_output=True, check=False, text=True) + # if sp.returncode != 0: + # message = f' ! Error running {settings.CAVERN}: {fullpath}' + # DataIssue.objects.create(parser='entrances', message=message) + # print(message) + # print(f'stderr:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode)) + + filepatherr = Path(survexdatasetpath / str(self.data['filename'] + ".err")) + if filepatherr.is_file(): + if filepatherr.stat().st_size == 0: + filepatherr.unlink() # delete empty closure error file + + fin = open(froog, "r",encoding='utf8') log = fin.read() fin.close() - log = re.sub("(?s).*?(Survey contains)", "\\1", log) + #log = re.sub("(?s).*?(Survey contains)", "\\1", log) # this omits any ERROR MESSAGES ! Don't do it. + for s in ["Removing trailing traverses...\n\n", + "Concatenating traverses...\n\n" + "Simplifying network...\n\n", + "Calculating network...\n\n", + "Calculating traverses...\n\n", + "Calculating trailing traverses...\n\n", + "Calculating statistics...\n\n"]: + log = log.replace(s,"") return log @@ -178,7 +204,8 @@ class SvxForm(forms.Form): def svx(request, survex_file): '''Displays a single survex file in an textarea window (using a javascript online editor to enable editing) with buttons which allow SAVE, check for DIFFerences from saved, and RUN (which runs the - cavern executable and displays the output below the main textarea). Requires CSRF to be set upcorrect;ly, and requires permission to write to the filesystem. + cavern executable and displays the output below the main textarea). + Requires CSRF to be set up correct;ly, and requires permission to write to the filesystem. ''' warning = False @@ -206,9 +233,9 @@ def svx(request, survex_file): pass if "process" in rform.data: if not difflist: - message = "OUTPUT FROM PROCESSING" logmessage = form.Process() - print(logmessage) + if logmessage: + message = f"OUTPUT FROM PROCESSING\n{logmessage}" else: message = "SAVE FILE FIRST" form.data['code'] = rcode @@ -220,6 +247,7 @@ def svx(request, survex_file): if message != "SAVED": form.data['code'] = rcode if "diff" in rform.data: + print("Differences: ") form.data['code'] = rcode #process(survex_file) @@ -228,7 +256,7 @@ def svx(request, survex_file): if form.template: warning = True if not difflist: - difflist.append("none") + difflist.append("No differences - file was saved") if message: difflist.insert(0, message) @@ -237,7 +265,7 @@ def svx(request, survex_file): vmap = {'settings': settings, 'warning': warning, - 'has_3d': os.path.isfile(survexdatasetpath / survex_file / ".3d"), + 'has_3d': (Path(survexdatasetpath) / Path(survex_file + ".3d")).is_file(), 'title': survex_file, 'svxincludes': svxincludes, 'difflist': difflist, @@ -250,40 +278,59 @@ def svx(request, survex_file): return render(request, 'svxfile.html', vmap) -def svxraw(request, survex_file): - '''Used for rendering .log files from survex outputtype''' - svx = open(os.path.join(survexdatasetpath / survex_file / ".svx"), "rt",encoding='utf8') - return HttpResponse(svx, content_type="text") - - -# The cavern running function +# The cavern running function. This is NOT where it is run inside the form! see SvxForm.Process() for that def process(survex_file): + '''This runs cavern only where a .3d, .log or .err file is requested. + ''' + filepathsvx = survexdatasetpath / str( survex_file + ".svx") cwd = os.getcwd() os.chdir(os.path.split(os.fspath(survexdatasetpath / survex_file))[0]) - os.system(settings.CAVERN + " --log " + survexdatasetpath / survex_file / ".svx") + os.system(settings.CAVERN + " --log " + str(filepathsvx)) os.chdir(cwd) + + # Update this to use the new syntax.. + # sp = subprocess.run([settings.CAVERN, "--log", f'--output={outputdir}', f'{fullpath}.svx'], + # capture_output=True, check=False, text=True) + # if sp.returncode != 0: + # message = f' ! Error running {settings.CAVERN}: {fullpath}' + # DataIssue.objects.create(parser='entrances', message=message) + # print(message) + # print(f'stderr:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode)) - + filepatherr = Path(survexdatasetpath / str(survex_file + ".err")) + if filepatherr.is_file(): + if filepatherr.stat().st_size == 0: + filepatherr.unlink() # delete empty closure error file + def threed(request, survex_file): - process(survex_file) - try: - threed = open(survexdatasetpath / survex_file / ".3d", "rt",encoding='utf8') - return HttpResponse(threed, content_type="model/3d") - except: - log = open(survexdatasetpath / survex_file / ".log", "rt",encoding='utf8') + filepath3d = survexdatasetpath / str(survex_file + ".3d") + filepathlog = survexdatasetpath / str(survex_file + ".log") + if filepath3d.is_file(): + threed = open(filepath3d, "rb") + return HttpResponse(threed, content_type="application/x-aven") + else: + process(survex_file) # should not need to do this if it already exists, as it should. + log = open(survexdatasetpath / str(survex_file + ".log"), "r",encoding='utf-8') return HttpResponse(log, content_type="text") - -def log(request, survex_file): - process(survex_file) - log = open(survexdatasetpath / survex_file / ".log", "rt",encoding='utf8') - return HttpResponse(log, content_type="text") - +def svxlog(request, survex_file): + '''Used for rendering .log files from survex outputtype''' + filepathlog = survexdatasetpath / str(survex_file + ".log") + if not filepathlog.is_file(): + process(survex_file) + log = open(filepathlog, "r") + return HttpResponse(log,content_type="text/plain; charset=utf-8") #default: "text/html; charset=utf-8" def err(request, survex_file): + filepatherr = survexdatasetpath / str(survex_file + ".err") + if not filepatherr.is_file(): # probably not there because it was empty, but re-run anyway + process(survex_file) process(survex_file) - err = open(survexdatasetpath / survex_file / ".err", "rt",encoding='utf8') - return HttpResponse(err, content_type="text") + if filepatherr.is_file(): + err = open(filepatherr, "r") + return HttpResponse(err, content_type="text/plain; charset=utf-8") + else: + return HttpResponse(f'No closure errors. \nEmpty {filepatherr} file produced. \nSee the .log file.', content_type="text/plain; charset=utf-8") def identifycavedircontents(gcavedir): diff --git a/parsers/survex.py b/parsers/survex.py index 8f3b65d..127bb70 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1221,8 +1221,14 @@ class LoadingSurvex(): self.caverncount += 1 # should also collect all the .err files too and create a DataIssue for each one which - # - is nonzero in size + # - is nonzero in size AND # - has Error greater than 5% anywhere, or some other more serious error + + errpath = Path(fullpath + ".err") + if errpath.is_file(): + if errpath.stat().st_size == 0: + errpath.unlink() # delete empty closure error file + svxpath = Path(fullpath + ".svx") logpath = Path(fullpath + ".log") diff --git a/templates/svxfile.html b/templates/svxfile.html index 76a27fa..3417859 100644 --- a/templates/svxfile.html +++ b/templates/svxfile.html @@ -3,7 +3,6 @@ {% block title %}{{ title }}{% endblock %} {% block head %} - @@ -69,14 +68,16 @@ $(document).ready(function() {% if logmessage %} {% if has_3d %} -

3d file

+

Click here to view the 3d file produced as output

{% else %} -

No 3d file

+

No 3d file generated. There was an ERROR
Read the full .log file + the .err file and fix this .svx file

{% endif %} - 
-    LOGMESSAGES
-    {{logmessage}}
+
 {% endif %}
 
 
diff --git a/templates/svxfiledifflistonly.html b/templates/svxfiledifflistonly.html
index b78c04d..7daecec 100644
--- a/templates/svxfiledifflistonly.html
+++ b/templates/svxfiledifflistonly.html
@@ -8,7 +8,7 @@
     {% if has_3d %}
         
3d file

     {% else %}
-        
No 3d file

+        
No 3d file generated. ERROR. Read the full .log file and fix the .svx file

     {% endif %}
     
     LOGMESSAGES
diff --git a/urls.py b/urls.py
index 0ee12a2..466cb78 100644
--- a/urls.py
+++ b/urls.py
@@ -154,7 +154,7 @@ trogglepatterns = [
 
     path('survexfile/.svx', survex.svx,        name="svx"), 
     path('survexfile/.3d',  survex.threed,     name="threed"), 
-    path('survexfile/.log', survex.svxraw,     name="svxraw"),  
+    path('survexfile/.log', survex.svxlog,     name="svxlog"),  
     path('survexfile/.err', survex.err,        name="err"), 
     path('survexfile/',     survex.survexcavesingle,    name="survexcavessingle"),