detect non-ISO dates in JSON and from user and fix

core/models/wallets.py

@@ -15,6 +15,42 @@ from django.urls import reverse
 
 YEAR_RANGE = (1975, 2050)
 
+def make_valid_date(date):
+    """Take whatever garbage some fool has typed in and try to make it into a valid ISO-format date
+    """
+    datestr = date.replace(".", "-")
+    try:
+        samedate = datetime.date.fromisoformat(datestr)
+    except ValueError:
+        # Could be in std euro format e.g. 14/07/2023
+        match = re.search(r'(\d{1,2})/(\d{1,2})/(\d{2,4})', datestr)
+        if match:
+            d = int(match.group(1))
+            m = int(match.group(2))
+            y = int(match.group(3))
+            if y<2000:
+                y = y + 2000
+            try:
+                samedate = datetime.date(y, m, d)
+                print(f"- - Warning, not in ISO format. '{datestr=}' but we coped: {samedate.isoformat()} ")
+                return samedate
+            except:
+                print(f"! - Fail, tried to decompose date in dd/mm/yyyy format but failed: {datestr=}  ")
+                return None
+        # probably a single digit day number. HACKUS MAXIMUS.
+        datestr = datestr[:-1] + "0" + datestr[-1]
+        # datestr = f"{datestr:02d}"
+        print(f"! - ValueError, trying.. {datestr=} ")
+        try:
+            samedate = datetime.date.fromisoformat(datestr)
+        except:
+            try:
+                samedate = datetime.date.fromisoformat(datestr[:10])
+            except:
+                print(f"! - ValueError, FAILED {datestr=} ")
+                samedate = None
+    return samedate
+
 class Wallet(models.Model):
     """We do not keep the JSON values in the database, we query them afresh each time,
     but we will change this when we need to do a Django query on e.g. personame
@@ -33,7 +69,9 @@ class Wallet(models.Model):
 
     def get_json(self):
         """Read the JSON file for the wallet and do stuff
-        Do it every time it is queried, to be sure the result is fresh"""
+        Do it every time it is queried, to be sure the result is fresh
+        
+        import DataIssue locally to prevent import cycle problem"""
         # jsonfile = Path(self.fpath, 'contents.json')
 
         # Get from git repo instead
@@ -42,7 +80,7 @@ class Wallet(models.Model):
         fp = Path(self.fpath)
         wname = fp.name
         wyear = fp.parent.name
-        wurl = f"/walletedit/{self.walletname}"  # .replace('#', ':')
+        wurl = f"/walletedit/{self.walletname}".replace('#', ':')
 
         if len(wyear) != 4 or len(wname) !=6: 
             # no contents.json for old-style wallets
@@ -52,8 +90,10 @@ class Wallet(models.Model):
 
         jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
         if not Path(jsonfile).is_file():
-            print(f'{jsonfile} is not a file {wyear=} {wname=} ')
+            message = f"! {jsonfile} is not a file {wyear=} {wname=} "
-            # Should this be a DataIssue?
+            from troggle.core.models.troggle import DataIssue
+            print(message)
+            DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
             return None
         else:
             with open(jsonfile) as json_f:
@@ -62,33 +102,19 @@ class Wallet(models.Model):
                 except:
                     message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
                     print(message)
-                    # Should this be a DataIssue?
+                    DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
                     return None
+                    
                 if waldata["date"]:
-                    datestr = waldata["date"].replace(".", "-")
+                    thisdate = make_valid_date(waldata["date"])
-                    try:
+                    if thisdate:
-                        thisdate = datetime.date.fromisoformat(datestr)
+                        self.walletdate = thisdate
-                    except ValueError:
+                        self.save()
-                        # probably a single digit day number. HACKUS MAXIMUS.
+                        waldata["date"] = thisdate.isoformat()
-                        # clearly we need to fix this when we first import date strings..
+                    else:
-                        datestr = datestr[:-1] + "0" + datestr[-1]
+                        message = f"! {str(self.walletname)} Date format not ISO {waldata['date']}. Failed to load from {jsonfile} JSON file"
-                        print(f" - {datestr=} ")
+                        from troggle.core.models.troggle import DataIssue
-                        try:
+                        DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
-                            thisdate = datetime.date.fromisoformat(datestr)
-                            self.walletdate = thisdate
-                            self.save()
-                            try:
-                                waldata["date"] = thisdate.isoformat()
-                            except:
-                                message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file"
-                                from troggle.core.models.troggle import DataIssue
-
-                                DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
-                        except:
-                            message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
-                            from troggle.core.models.troggle import DataIssue
-
-                            DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
         return waldata
 
     def year(self):

core/views/wallets_edit.py

@@ -18,7 +18,7 @@ from troggle.core.models.caves import Cave
 from troggle.core.models.logbooks import LogbookEntry  # , PersonLogEntry
 from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
 from troggle.core.models.troggle import DataIssue, Expedition
-from troggle.core.models.wallets import Wallet, YEAR_RANGE
+from troggle.core.models.wallets import Wallet, YEAR_RANGE, make_valid_date
 
 from troggle.core.views.auth import login_required_if_public
 from troggle.core.views.caves import getCave
@@ -32,7 +32,7 @@ from troggle.parsers.scans import contentsjson
 """
 
 todo = """
-- Nasty bug in navingating to 'previous wallet' when we have a 2-year gap in expos
+- Nasty bug in navigating to 'previous wallet' when we have a 2-year gap in expos
   The xxxx#00 wallet is not getting edited correctly. Something is off by one somewhere..
 
 - Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
@@ -496,25 +496,7 @@ def walletedit(request, path=None):
                     or thing == "[]"
                     or thing is None)
 
-    def make_valid_date(date):
+
-        datestr = date.replace(".", "-")
-        try:
-            samedate = datetime.date.fromisoformat(datestr)
-        except ValueError:
-            # probably a single digit day number. HACKUS MAXIMUS.
-            # clearly we need to fix this when we first import date strings..
-            datestr = datestr[:-1] + "0" + datestr[-1]
-            # datestr = f"{datestr:02d}"
-            print(f"! - ValueError, trying.. {datestr=} ")
-            try:
-                samedate = datetime.date.fromisoformat(datestr)
-            except:
-                try:
-                    samedate = datetime.date.fromisoformat(datestr[:10])
-                except:
-                    print(f"! - ValueError, FAILED {datestr=} ")
-                    samedate = None
-        return samedate
         
     def scan_survexblocks(svxfile):
         """Scans for *team people attached to all the survex blocks in this svxfile

parsers/imports.py

@@ -27,7 +27,7 @@ def import_people():
         troggle.parsers.people.load_people_expos()
 
 def import_surveyscans():
-    print("-- Importing Survey Scans")
+    print("-- Importing Survey Scans and Wallets")
     with transaction.atomic():
         troggle.parsers.scans.load_all_scans()
 

parsers/scans.py

@@ -38,6 +38,7 @@ def load_all_scans():
     Wallet.objects.all().delete()
     print("  - deleting all Wallet and SingleScan objects")
     DataIssue.objects.filter(parser="scans").delete()
+    DataIssue.objects.filter(parser="wallets").delete()
 
     # These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet.
     valids = [