Handling and fixing bad dates in JSON input

core/models/survex.py

@@ -2,6 +2,7 @@ import os
 import re
 import json
 import operator
+import datetime
 from urllib.parse import urljoin
 from pathlib import Path
 from functools import reduce
@@ -10,6 +11,7 @@ from django.db import models
 from django.conf import settings
 from django.urls import reverse
 
+# from troggle.core.models.troggle import DataIssue # circular import. Hmm
 
 class SurvexDirectory(models.Model):
     path = models.CharField(max_length=200)
@@ -185,6 +187,7 @@ class Wallet(models.Model):
         fp = Path(self.fpath)
         wname = fp.name
         wyear = fp.parent.name
+        wurl = f"/scanupload/{self.walletname}" # .replace('#', ':')     
         
         jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
         if not Path(jsonfile).is_file():
@@ -195,11 +198,30 @@ class Wallet(models.Model):
                 try:
                     waldata = json.load(json_f)
                 except:
-                    wurl = f"/scanupload/{self.walletname}" # .replace('#', ':')     
                     message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
                     #print(message)
                     raise
-
+                if waldata["date"]:
+                    datestr = waldata["date"].replace('.','-')
+                    try:
+                        thisdate = datetime.date.fromisoformat(datestr)
+                    except ValueError:
+                        # probably a single digit day number. HACKUS MAXIMUS.
+                        # clearly we need to fix this when we first import date strings..
+                        datestr = datestr[:-1] + '0' + datestr[-1]
+                        print(f' - {datestr=} ')
+                        try:
+                            thisdate = datetime.date.fromisoformat(datestr)
+                            try:
+                                waldata["date"] = thisdate.isoformat()
+                            except:
+                                message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file"
+                                from troggle.core.models.troggle import DataIssue
+                                DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) 
+                        except:
+                            message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load fro, {jsonfile} JSON file"
+                            from troggle.core.models.troggle import DataIssue
+                            DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) 
         return waldata
         
     def year(self): 

core/views/scans.py

@@ -82,12 +82,15 @@ def fillblankpeople(w):
         # this isn't working..? why? Because it needs a *ref and an import
         wp = w.people()
         w.persons = wp
-        if len(wp) == 1:
+        if not wp:
-            # print(f' - {wp=}')
+            populatewallet(w)
-            nobody = wp[0].lower()
+        else:
-            if  nobody == 'unknown' or nobody == 'nobody' or nobody == ' ' or nobody == '':
+            if len(wp) == 1:
-                print(f' - {wp=} {nobody=}')
+                # print(f' - {wp=}')
-                populatewallet(w)
+                nobody = wp[0].lower()
+                if  nobody == 'unknown' or nobody == 'nobody' or nobody == ' ' or nobody == '':
+                    print(f' - {wp=} {nobody=}')
+                    populatewallet(w)
         
 def fillblankothers(w):       
     earliest = datetime.datetime.now().date()

core/views/uploads.py

@@ -631,14 +631,24 @@ def scanupload(request, path=None):
                 # clearly we need to fix this when we first import date strings..
                 datestr = datestr[:-1] + '0' + datestr[-1]
                 print(f' - {datestr=} ')
-                samedate = datetime.date.fromisoformat(datestr)
+                try:
+                    samedate = datetime.date.fromisoformat(datestr)
+                except:
+                    try:
+                        samedate = datetime.date.fromisoformat(datestr[:10])
+                    except:
+                        samedate = None
                 
             thisexpo = Expedition.objects.get(year=int(year))
-            expeditionday = thisexpo.get_expedition_day(samedate)
+            if samedate:
-            #print(f' - {thisexpo=} {expeditionday=}')
+                expeditionday = thisexpo.get_expedition_day(samedate)
-            svxothers = SurvexBlock.objects.filter(expeditionday=expeditionday)
+                #print(f' - {thisexpo=} {expeditionday=}')
-            #print(f' - {thisexpo=} {expeditionday=} {svxothers=}')
+                svxothers = SurvexBlock.objects.filter(expeditionday=expeditionday)
-            trips = LogbookEntry.objects.filter(date=samedate)
+                #print(f' - {thisexpo=} {expeditionday=} {svxothers=}')
+                trips = LogbookEntry.objects.filter(date=samedate)
+            else:
+                svxothers = None
+                trips = None
  
         else:
             svxothers = None