xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

detect non-ISO dates in JSON and from user and fix

Browse Source
This commit is contained in:
Philip Sargent 2023-07-23 23:30:19 +03:00
parent 3ea014ec57
commit 9473b22bd9
4 changed files with 60 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
core/models/wallets.py
View File

@ -15,6 +15,42 @@ from django.urls import reverse
YEAR_RANGE = (1975, 2050)
def make_valid_date(date):
"""Take whatever garbage some fool has typed in and try to make it into a valid ISO-format date
"""
datestr = date.replace(".", "-")
try:
samedate = datetime.date.fromisoformat(datestr)
except ValueError:
# Could be in std euro format e.g. 14/07/2023
match = re.search(r'(\d{1,2})/(\d{1,2})/(\d{2,4})', datestr)
if match:
d = int(match.group(1))
m = int(match.group(2))
y = int(match.group(3))
if y<2000:
y = y + 2000
try:
samedate = datetime.date(y, m, d)
print(f"- - Warning, not in ISO format. '{datestr=}' but we coped: {samedate.isoformat()} ")
return samedate
except:
print(f"! - Fail, tried to decompose date in dd/mm/yyyy format but failed: {datestr=} ")
return None
# probably a single digit day number. HACKUS MAXIMUS.
datestr = datestr[:-1] + "0" + datestr[-1]
# datestr = f"{datestr:02d}"
print(f"! - ValueError, trying.. {datestr=} ")
try:
samedate = datetime.date.fromisoformat(datestr)
except:
try:
samedate = datetime.date.fromisoformat(datestr[:10])
except:
print(f"! - ValueError, FAILED {datestr=} ")
samedate = None
return samedate
class Wallet(models.Model):
"""We do not keep the JSON values in the database, we query them afresh each time,
but we will change this when we need to do a Django query on e.g. personame
@ -33,7 +69,9 @@ class Wallet(models.Model):
def get_json(self):
"""Read the JSON file for the wallet and do stuff
Do it every time it is queried, to be sure the result is fresh"""
Do it every time it is queried, to be sure the result is fresh
import DataIssue locally to prevent import cycle problem"""
# jsonfile = Path(self.fpath, 'contents.json')
# Get from git repo instead
@ -42,7 +80,7 @@ class Wallet(models.Model):
fp = Path(self.fpath)
wname = fp.name
wyear = fp.parent.name
wurl = f"/walletedit/{self.walletname}" # .replace('#', ':')
wurl = f"/walletedit/{self.walletname}".replace('#', ':')
if len(wyear) != 4 or len(wname) !=6:
# no contents.json for old-style wallets
@ -52,8 +90,10 @@ class Wallet(models.Model):
jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
if not Path(jsonfile).is_file():
print(f'{jsonfile} is not a file {wyear=} {wname=} ')
# Should this be a DataIssue?
message = f"! {jsonfile} is not a file {wyear=} {wname=} "
from troggle.core.models.troggle import DataIssue
print(message)
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
return None
else:
with open(jsonfile) as json_f:
@ -62,33 +102,19 @@ class Wallet(models.Model):
except:
message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
print(message)
# Should this be a DataIssue?
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
return None
if waldata["date"]:
datestr = waldata["date"].replace(".", "-")
try:
thisdate = datetime.date.fromisoformat(datestr)
except ValueError:
# probably a single digit day number. HACKUS MAXIMUS.
# clearly we need to fix this when we first import date strings..
datestr = datestr[:-1] + "0" + datestr[-1]
print(f" - {datestr=} ")
try:
thisdate = datetime.date.fromisoformat(datestr)
self.walletdate = thisdate
self.save()
try:
waldata["date"] = thisdate.isoformat()
except:
message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file"
from troggle.core.models.troggle import DataIssue
DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
except:
message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
from troggle.core.models.troggle import DataIssue
DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
thisdate = make_valid_date(waldata["date"])
if thisdate:
self.walletdate = thisdate
self.save()
waldata["date"] = thisdate.isoformat()
else:
message = f"! {str(self.walletname)} Date format not ISO {waldata['date']}. Failed to load from {jsonfile} JSON file"
from troggle.core.models.troggle import DataIssue
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
return waldata
def year(self):

24
core/views/wallets_edit.py
View File

@ -18,7 +18,7 @@ from troggle.core.models.caves import Cave
from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
from troggle.core.models.troggle import DataIssue, Expedition
from troggle.core.models.wallets import Wallet, YEAR_RANGE
from troggle.core.models.wallets import Wallet, YEAR_RANGE, make_valid_date
from troggle.core.views.auth import login_required_if_public
from troggle.core.views.caves import getCave
@ -32,7 +32,7 @@ from troggle.parsers.scans import contentsjson
"""
todo = """
- Nasty bug in navingating to 'previous wallet' when we have a 2-year gap in expos
- Nasty bug in navigating to 'previous wallet' when we have a 2-year gap in expos
The xxxx#00 wallet is not getting edited correctly. Something is off by one somewhere..
- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
@ -496,25 +496,7 @@ def walletedit(request, path=None):
or thing == "[]"
or thing is None)
def make_valid_date(date):
datestr = date.replace(".", "-")
try:
samedate = datetime.date.fromisoformat(datestr)
except ValueError:
# probably a single digit day number. HACKUS MAXIMUS.
# clearly we need to fix this when we first import date strings..
datestr = datestr[:-1] + "0" + datestr[-1]
# datestr = f"{datestr:02d}"
print(f"! - ValueError, trying.. {datestr=} ")
try:
samedate = datetime.date.fromisoformat(datestr)
except:
try:
samedate = datetime.date.fromisoformat(datestr[:10])
except:
print(f"! - ValueError, FAILED {datestr=} ")
samedate = None
return samedate
def scan_survexblocks(svxfile):
"""Scans for *team people attached to all the survex blocks in this svxfile

2
parsers/imports.py
View File

@ -27,7 +27,7 @@ def import_people():
troggle.parsers.people.load_people_expos()
def import_surveyscans():
print("-- Importing Survey Scans")
print("-- Importing Survey Scans and Wallets")
with transaction.atomic():
troggle.parsers.scans.load_all_scans()

1
parsers/scans.py
View File

@ -38,6 +38,7 @@ def load_all_scans():
Wallet.objects.all().delete()
print(" - deleting all Wallet and SingleScan objects")
DataIssue.objects.filter(parser="scans").delete()
DataIssue.objects.filter(parser="wallets").delete()
# These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet.
valids = [