catch nonUTF8 survex files, DataIssues url editor

core/views/survex.py

@@ -119,9 +119,14 @@ class SvxForm(forms.Form):
             print(">>> >>> WARNING - svx file not found, showing TEMPLATE SVX",fname, flush=True)
             self.template = True
             return survextemplatefile
+        try:
             fin = open(fname, "r",encoding='utf8',newline='')
             svxtext = fin.read() 
             fin.close()
+        except:
+            fin = open(fname, "r",encoding='iso-8859-1',newline='')
+            svxtext = fin.read() 
+            fin.close()
         return svxtext
             
     def DiffCode(self, rcode):

parsers/survex.py

@@ -1140,7 +1140,18 @@ class LoadingSurvex():
                 return
         self.svxfileslist.append(path)
 
+        try:
             svxlines = fin.read().splitlines()
+        except UnicodeDecodeError:
+            # some bugger put an umlaut in a non-UTF survex file ?!
+            message = f"    ! ERROR *include file '{path}' in '{survexblock}' has UnicodeDecodeError"
+            print(message)
+            print(message,file=sys.stderr)
+            offendingfile = "/survexfile/" + path + ".svx"
+            DataIssue.objects.create(parser='survex', message=message,  url=offendingfile)
+            return # skip this survex file and all things *included in it
+       
+        
         for svxline in svxlines:
             self.lineno += 1
             thissvxline += 1
@@ -1153,7 +1164,7 @@ class LoadingSurvex():
                 print(message)
                 print(message,file=sys.stderr)
                 DataIssue.objects.create(parser='survex', message=message)
-                return # skip this survex file
+                return # skip this survex file and all things *included in it
 
             includestmt =self.rx_include.match(svxline)
             if not includestmt: