detect non-ISO dates in JSON and from user and fix

core/models/wallets.py

@@ -15,6 +15,42 @@ from django.urls import reverse
 
 YEAR_RANGE = (1975, 2050)
 
+def make_valid_date(date):
+    """Take whatever garbage some fool has typed in and try to make it into a valid ISO-format date
+    """
+    datestr = date.replace(".", "-")
+    try:
+        samedate = datetime.date.fromisoformat(datestr)
+    except ValueError:
+        # Could be in std euro format e.g. 14/07/2023
+        match = re.search(r'(\d{1,2})/(\d{1,2})/(\d{2,4})', datestr)
+        if match:
+            d = int(match.group(1))
+            m = int(match.group(2))
+            y = int(match.group(3))
+            if y<2000:
+                y = y + 2000
+            try:
+                samedate = datetime.date(y, m, d)
+                print(f"- - Warning, not in ISO format. '{datestr=}' but we coped: {samedate.isoformat()} ")
+                return samedate
+            except:
+                print(f"! - Fail, tried to decompose date in dd/mm/yyyy format but failed: {datestr=}  ")
+                return None
+        # probably a single digit day number. HACKUS MAXIMUS.
+        datestr = datestr[:-1] + "0" + datestr[-1]
+        # datestr = f"{datestr:02d}"
+        print(f"! - ValueError, trying.. {datestr=} ")
+        try:
+            samedate = datetime.date.fromisoformat(datestr)
+        except:
+            try:
+                samedate = datetime.date.fromisoformat(datestr[:10])
+            except:
+                print(f"! - ValueError, FAILED {datestr=} ")
+                samedate = None
+    return samedate
+
 class Wallet(models.Model):
     """We do not keep the JSON values in the database, we query them afresh each time,
     but we will change this when we need to do a Django query on e.g. personame
@@ -33,7 +69,9 @@ class Wallet(models.Model):
 
     def get_json(self):
         """Read the JSON file for the wallet and do stuff
-        Do it every time it is queried, to be sure the result is fresh"""
+        Do it every time it is queried, to be sure the result is fresh
+        
+        import DataIssue locally to prevent import cycle problem"""
         # jsonfile = Path(self.fpath, 'contents.json')
 
         # Get from git repo instead
@@ -42,7 +80,7 @@ class Wallet(models.Model):
         fp = Path(self.fpath)
         wname = fp.name
         wyear = fp.parent.name
-        wurl = f"/walletedit/{self.walletname}"  # .replace('#', ':')
+        wurl = f"/walletedit/{self.walletname}".replace('#', ':')
 
         if len(wyear) != 4 or len(wname) !=6: 
             # no contents.json for old-style wallets
@@ -52,8 +90,10 @@ class Wallet(models.Model):
 
         jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
         if not Path(jsonfile).is_file():
-            print(f'{jsonfile} is not a file {wyear=} {wname=} ')
-            # Should this be a DataIssue?
+            message = f"! {jsonfile} is not a file {wyear=} {wname=} "
+            from troggle.core.models.troggle import DataIssue
+            print(message)
+            DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
             return None
         else:
             with open(jsonfile) as json_f:
@@ -62,33 +102,19 @@ class Wallet(models.Model):
                 except:
                     message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
                     print(message)
-                    # Should this be a DataIssue?
+                    DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
                     return None
+                    
                 if waldata["date"]:
-                    datestr = waldata["date"].replace(".", "-")
-                    try:
-                        thisdate = datetime.date.fromisoformat(datestr)
-                    except ValueError:
-                        # probably a single digit day number. HACKUS MAXIMUS.
-                        # clearly we need to fix this when we first import date strings..
-                        datestr = datestr[:-1] + "0" + datestr[-1]
-                        print(f" - {datestr=} ")
-                        try:
-                            thisdate = datetime.date.fromisoformat(datestr)
-                            self.walletdate = thisdate
-                            self.save()
-                            try:
-                                waldata["date"] = thisdate.isoformat()
-                            except:
-                                message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file"
-                                from troggle.core.models.troggle import DataIssue
-
-                                DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
-                        except:
-                            message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
-                            from troggle.core.models.troggle import DataIssue
-
-                            DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
+                    thisdate = make_valid_date(waldata["date"])
+                    if thisdate:
+                        self.walletdate = thisdate
+                        self.save()
+                        waldata["date"] = thisdate.isoformat()
+                    else:
+                        message = f"! {str(self.walletname)} Date format not ISO {waldata['date']}. Failed to load from {jsonfile} JSON file"
+                        from troggle.core.models.troggle import DataIssue
+                        DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
         return waldata
 
     def year(self):