diff --git a/core/models/wallets.py b/core/models/wallets.py
index 9da5695..29ca9e2 100644
--- a/core/models/wallets.py
+++ b/core/models/wallets.py
@@ -15,6 +15,42 @@ from django.urls import reverse
YEAR_RANGE = (1975, 2050)
+def make_valid_date(date):
+ """Take whatever garbage some fool has typed in and try to make it into a valid ISO-format date
+ """
+ datestr = date.replace(".", "-")
+ try:
+ samedate = datetime.date.fromisoformat(datestr)
+ except ValueError:
+ # Could be in std euro format e.g. 14/07/2023
+ match = re.search(r'(\d{1,2})/(\d{1,2})/(\d{2,4})', datestr)
+ if match:
+ d = int(match.group(1))
+ m = int(match.group(2))
+ y = int(match.group(3))
+ if y<2000:
+ y = y + 2000
+ try:
+ samedate = datetime.date(y, m, d)
+ print(f"- - Warning, not in ISO format. '{datestr=}' but we coped: {samedate.isoformat()} ")
+ return samedate
+ except:
+ print(f"! - Fail, tried to decompose date in dd/mm/yyyy format but failed: {datestr=} ")
+ return None
+ # probably a single digit day number. HACKUS MAXIMUS.
+ datestr = datestr[:-1] + "0" + datestr[-1]
+ # datestr = f"{datestr:02d}"
+ print(f"! - ValueError, trying.. {datestr=} ")
+ try:
+ samedate = datetime.date.fromisoformat(datestr)
+ except:
+ try:
+ samedate = datetime.date.fromisoformat(datestr[:10])
+ except:
+ print(f"! - ValueError, FAILED {datestr=} ")
+ samedate = None
+ return samedate
+
class Wallet(models.Model):
"""We do not keep the JSON values in the database, we query them afresh each time,
but we will change this when we need to do a Django query on e.g. personame
@@ -33,7 +69,9 @@ class Wallet(models.Model):
def get_json(self):
"""Read the JSON file for the wallet and do stuff
- Do it every time it is queried, to be sure the result is fresh"""
+ Do it every time it is queried, to be sure the result is fresh
+
+ import DataIssue locally to prevent import cycle problem"""
# jsonfile = Path(self.fpath, 'contents.json')
# Get from git repo instead
@@ -42,7 +80,7 @@ class Wallet(models.Model):
fp = Path(self.fpath)
wname = fp.name
wyear = fp.parent.name
- wurl = f"/walletedit/{self.walletname}" # .replace('#', ':')
+ wurl = f"/walletedit/{self.walletname}".replace('#', ':')
if len(wyear) != 4 or len(wname) !=6:
# no contents.json for old-style wallets
@@ -52,8 +90,10 @@ class Wallet(models.Model):
jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
if not Path(jsonfile).is_file():
- print(f'{jsonfile} is not a file {wyear=} {wname=} ')
- # Should this be a DataIssue?
+ message = f"! {jsonfile} is not a file {wyear=} {wname=} "
+ from troggle.core.models.troggle import DataIssue
+ print(message)
+ DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
return None
else:
with open(jsonfile) as json_f:
@@ -62,33 +102,19 @@ class Wallet(models.Model):
except:
message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
print(message)
- # Should this be a DataIssue?
+ DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
return None
+
if waldata["date"]:
- datestr = waldata["date"].replace(".", "-")
- try:
- thisdate = datetime.date.fromisoformat(datestr)
- except ValueError:
- # probably a single digit day number. HACKUS MAXIMUS.
- # clearly we need to fix this when we first import date strings..
- datestr = datestr[:-1] + "0" + datestr[-1]
- print(f" - {datestr=} ")
- try:
- thisdate = datetime.date.fromisoformat(datestr)
- self.walletdate = thisdate
- self.save()
- try:
- waldata["date"] = thisdate.isoformat()
- except:
- message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file"
- from troggle.core.models.troggle import DataIssue
-
- DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
- except:
- message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
- from troggle.core.models.troggle import DataIssue
-
- DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
+ thisdate = make_valid_date(waldata["date"])
+ if thisdate:
+ self.walletdate = thisdate
+ self.save()
+ waldata["date"] = thisdate.isoformat()
+ else:
+ message = f"! {str(self.walletname)} Date format not ISO {waldata['date']}. Failed to load from {jsonfile} JSON file"
+ from troggle.core.models.troggle import DataIssue
+ DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
return waldata
def year(self):
diff --git a/core/views/wallets_edit.py b/core/views/wallets_edit.py
index 7cf2251..9273fed 100644
--- a/core/views/wallets_edit.py
+++ b/core/views/wallets_edit.py
@@ -18,7 +18,7 @@ from troggle.core.models.caves import Cave
from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
from troggle.core.models.troggle import DataIssue, Expedition
-from troggle.core.models.wallets import Wallet, YEAR_RANGE
+from troggle.core.models.wallets import Wallet, YEAR_RANGE, make_valid_date
from troggle.core.views.auth import login_required_if_public
from troggle.core.views.caves import getCave
@@ -32,7 +32,7 @@ from troggle.parsers.scans import contentsjson
"""
todo = """
-- Nasty bug in navingating to 'previous wallet' when we have a 2-year gap in expos
+- Nasty bug in navigating to 'previous wallet' when we have a 2-year gap in expos
The xxxx#00 wallet is not getting edited correctly. Something is off by one somewhere..
- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
@@ -496,25 +496,7 @@ def walletedit(request, path=None):
or thing == "[]"
or thing is None)
- def make_valid_date(date):
- datestr = date.replace(".", "-")
- try:
- samedate = datetime.date.fromisoformat(datestr)
- except ValueError:
- # probably a single digit day number. HACKUS MAXIMUS.
- # clearly we need to fix this when we first import date strings..
- datestr = datestr[:-1] + "0" + datestr[-1]
- # datestr = f"{datestr:02d}"
- print(f"! - ValueError, trying.. {datestr=} ")
- try:
- samedate = datetime.date.fromisoformat(datestr)
- except:
- try:
- samedate = datetime.date.fromisoformat(datestr[:10])
- except:
- print(f"! - ValueError, FAILED {datestr=} ")
- samedate = None
- return samedate
+
def scan_survexblocks(svxfile):
"""Scans for *team people attached to all the survex blocks in this svxfile
diff --git a/parsers/imports.py b/parsers/imports.py
index 50b45cf..9191194 100644
--- a/parsers/imports.py
+++ b/parsers/imports.py
@@ -27,7 +27,7 @@ def import_people():
troggle.parsers.people.load_people_expos()
def import_surveyscans():
- print("-- Importing Survey Scans")
+ print("-- Importing Survey Scans and Wallets")
with transaction.atomic():
troggle.parsers.scans.load_all_scans()
diff --git a/parsers/scans.py b/parsers/scans.py
index 3f042da..9929209 100644
--- a/parsers/scans.py
+++ b/parsers/scans.py
@@ -38,6 +38,7 @@ def load_all_scans():
Wallet.objects.all().delete()
print(" - deleting all Wallet and SingleScan objects")
DataIssue.objects.filter(parser="scans").delete()
+ DataIssue.objects.filter(parser="wallets").delete()
# These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet.
valids = [