Merge branch 'python3-new' of ssh://expo.survex.com/home/expo/troggle into python3-new

_deploy/debian/wookey-exposerver-recipe.txt

@@ -56,6 +56,9 @@ mysql -p
 CREATE DATABASE troggle;
 GRANT ALL PRIVILEGES ON troggle.* TO 'expo'@'localhost' IDENTIFIED BY 'somepassword';  
 
+sudo service mariadb stop
+sudo service mariadb start
+
 install django:
 sudo apt install python-django python-django-registration python-django-imagekit python-django-tinymce fonts-freefont-ttf libapache2-mod-wsgi
 

core/views/survex.py

@@ -119,9 +119,14 @@ class SvxForm(forms.Form):
             print(">>> >>> WARNING - svx file not found, showing TEMPLATE SVX",fname, flush=True)
             self.template = True
             return survextemplatefile
-        fin = open(fname, "r",encoding='utf8',newline='')
-        svxtext = fin.read() 
-        fin.close()
+        try:
+            fin = open(fname, "r",encoding='utf8',newline='')
+            svxtext = fin.read() 
+            fin.close()
+        except:
+            fin = open(fname, "r",encoding='iso-8859-1',newline='')
+            svxtext = fin.read() 
+            fin.close()
         return svxtext
             
     def DiffCode(self, rcode):

parsers/logbooks.py

@@ -59,7 +59,7 @@ noncaveplaces = [ "QMplaceholder", "Journey", "Loser Plateau", "UNKNOWN", 'plate
 logdataissues = TROG['issues']['logdataissues']
 trips ={}
 
-entries = { "2019": 20, "2018": 74, "2017": 60, "2016": 81, "2015": 79, 
+entries = { "2019": 36, "2018": 74, "2017": 60, "2016": 81, "2015": 79, 
     "2014": 65, "2013": 51, "2012": 75, "2011": 68, "2010": 22, "2009": 52, 
     "2008": 49, "2007": 111, "2006": 60, "2005": 55, "2004": 76, "2003": 40, "2002": 31, 
     "2001": 48, "2000": 54, "1999": 79, "1998": 43, "1997": 53, "1996": 94, "1995": 41, 
@@ -174,6 +174,7 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_
 
 def ParseDate(tripdate, year):
     """ Interprets dates in the expo logbooks and returns a correct datetime.date object  """
+    dummydate = date(1970, 1, 1)
     try:
         mdatestandard = re.match(r"(\d\d\d\d)-(\d\d)-(\d\d)", tripdate)
         mdategoof = re.match(r"(\d\d?)/0?(\d)/(20|19)?(\d\d)", tripdate)
@@ -182,7 +183,7 @@ def ParseDate(tripdate, year):
                 message = f" ! - Bad date (year) in logbook: {tripdate} - {year}"
                 DataIssue.objects.create(parser='logbooks', message=message)
                 logdataissues["tripdate"]=message
-                return datetime.date(1970, 1, 1)
+                return dummydate
             else:
                 year, month, day = int(mdatestandard.group(1)), int(mdatestandard.group(2)), int(mdatestandard.group(3))
         elif mdategoof:
@@ -190,7 +191,7 @@ def ParseDate(tripdate, year):
                 message = " ! - Bad date mdategoof.group(3) in logbook: " + tripdate + " - " + mdategoof.group(3)
                 DataIssue.objects.create(parser='logbooks', message=message)
                 logdataissues["tripdate"]=message
-                return datetime.date(1970, 1, 1)
+                return dummydate
             else:
                 yadd = int(year[:2]) * 100
                 day, month, year = int(mdategoof.group(1)), int(mdategoof.group(2)), int(mdategoof.group(4)) + yadd

parsers/survex.py

@@ -835,7 +835,7 @@ class LoadingSurvex():
             else:
                 message = f' ! QM Unrecognised as a valid QM in "{survexblock.survexfile.path}" QM{qml.group(1)} {qml.group(2)}'
                 print(message)
-                DataIssue.objects.create(parser='survex', message=message)
+                DataIssue.objects.create(parser='survex', message=message, url=f'/survexfile/{survexblock.survexfile.path}.svx')
 
             
         included = self.rx_comminc.match(comment)
@@ -1140,7 +1140,18 @@ class LoadingSurvex():
                 return
         self.svxfileslist.append(path)
 
-        svxlines = fin.read().splitlines()
+        try:
+            svxlines = fin.read().splitlines()
+        except UnicodeDecodeError:
+            # some bugger put an umlaut in a non-UTF survex file ?!
+            message = f"    ! ERROR *include file '{path}' in '{survexblock}' has UnicodeDecodeError"
+            print(message)
+            print(message,file=sys.stderr)
+            offendingfile = "/survexfile/" + path + ".svx"
+            DataIssue.objects.create(parser='survex', message=message,  url=offendingfile)
+            return # skip this survex file and all things *included in it
+       
+        
         for svxline in svxlines:
             self.lineno += 1
             thissvxline += 1
@@ -1153,7 +1164,7 @@ class LoadingSurvex():
                 print(message)
                 print(message,file=sys.stderr)
                 DataIssue.objects.create(parser='survex', message=message)
-                return # skip this survex file
+                return # skip this survex file and all things *included in it
 
             includestmt =self.rx_include.match(svxline)
             if not includestmt: