forked from expo/troggle
detect non-ISO dates in JSON and from user and fix
This commit is contained in:
parent
3ea014ec57
commit
9473b22bd9
@ -15,6 +15,42 @@ from django.urls import reverse
|
||||
|
||||
YEAR_RANGE = (1975, 2050)
|
||||
|
||||
def make_valid_date(date):
|
||||
"""Take whatever garbage some fool has typed in and try to make it into a valid ISO-format date
|
||||
"""
|
||||
datestr = date.replace(".", "-")
|
||||
try:
|
||||
samedate = datetime.date.fromisoformat(datestr)
|
||||
except ValueError:
|
||||
# Could be in std euro format e.g. 14/07/2023
|
||||
match = re.search(r'(\d{1,2})/(\d{1,2})/(\d{2,4})', datestr)
|
||||
if match:
|
||||
d = int(match.group(1))
|
||||
m = int(match.group(2))
|
||||
y = int(match.group(3))
|
||||
if y<2000:
|
||||
y = y + 2000
|
||||
try:
|
||||
samedate = datetime.date(y, m, d)
|
||||
print(f"- - Warning, not in ISO format. '{datestr=}' but we coped: {samedate.isoformat()} ")
|
||||
return samedate
|
||||
except:
|
||||
print(f"! - Fail, tried to decompose date in dd/mm/yyyy format but failed: {datestr=} ")
|
||||
return None
|
||||
# probably a single digit day number. HACKUS MAXIMUS.
|
||||
datestr = datestr[:-1] + "0" + datestr[-1]
|
||||
# datestr = f"{datestr:02d}"
|
||||
print(f"! - ValueError, trying.. {datestr=} ")
|
||||
try:
|
||||
samedate = datetime.date.fromisoformat(datestr)
|
||||
except:
|
||||
try:
|
||||
samedate = datetime.date.fromisoformat(datestr[:10])
|
||||
except:
|
||||
print(f"! - ValueError, FAILED {datestr=} ")
|
||||
samedate = None
|
||||
return samedate
|
||||
|
||||
class Wallet(models.Model):
|
||||
"""We do not keep the JSON values in the database, we query them afresh each time,
|
||||
but we will change this when we need to do a Django query on e.g. personame
|
||||
@ -33,7 +69,9 @@ class Wallet(models.Model):
|
||||
|
||||
def get_json(self):
|
||||
"""Read the JSON file for the wallet and do stuff
|
||||
Do it every time it is queried, to be sure the result is fresh"""
|
||||
Do it every time it is queried, to be sure the result is fresh
|
||||
|
||||
import DataIssue locally to prevent import cycle problem"""
|
||||
# jsonfile = Path(self.fpath, 'contents.json')
|
||||
|
||||
# Get from git repo instead
|
||||
@ -42,7 +80,7 @@ class Wallet(models.Model):
|
||||
fp = Path(self.fpath)
|
||||
wname = fp.name
|
||||
wyear = fp.parent.name
|
||||
wurl = f"/walletedit/{self.walletname}" # .replace('#', ':')
|
||||
wurl = f"/walletedit/{self.walletname}".replace('#', ':')
|
||||
|
||||
if len(wyear) != 4 or len(wname) !=6:
|
||||
# no contents.json for old-style wallets
|
||||
@ -52,8 +90,10 @@ class Wallet(models.Model):
|
||||
|
||||
jsonfile = Path(settings.DRAWINGS_DATA, "walletjson") / wyear / wname / "contents.json"
|
||||
if not Path(jsonfile).is_file():
|
||||
print(f'{jsonfile} is not a file {wyear=} {wname=} ')
|
||||
# Should this be a DataIssue?
|
||||
message = f"! {jsonfile} is not a file {wyear=} {wname=} "
|
||||
from troggle.core.models.troggle import DataIssue
|
||||
print(message)
|
||||
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
|
||||
return None
|
||||
else:
|
||||
with open(jsonfile) as json_f:
|
||||
@ -62,33 +102,19 @@ class Wallet(models.Model):
|
||||
except:
|
||||
message = f"! {str(self.walletname)} Failed to load {jsonfile} JSON file"
|
||||
print(message)
|
||||
# Should this be a DataIssue?
|
||||
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
|
||||
return None
|
||||
|
||||
if waldata["date"]:
|
||||
datestr = waldata["date"].replace(".", "-")
|
||||
try:
|
||||
thisdate = datetime.date.fromisoformat(datestr)
|
||||
except ValueError:
|
||||
# probably a single digit day number. HACKUS MAXIMUS.
|
||||
# clearly we need to fix this when we first import date strings..
|
||||
datestr = datestr[:-1] + "0" + datestr[-1]
|
||||
print(f" - {datestr=} ")
|
||||
try:
|
||||
thisdate = datetime.date.fromisoformat(datestr)
|
||||
thisdate = make_valid_date(waldata["date"])
|
||||
if thisdate:
|
||||
self.walletdate = thisdate
|
||||
self.save()
|
||||
try:
|
||||
waldata["date"] = thisdate.isoformat()
|
||||
except:
|
||||
message = f"! {str(self.walletname)} Date formatting failure {thisdate}. Failed to load from {jsonfile} JSON file"
|
||||
else:
|
||||
message = f"! {str(self.walletname)} Date format not ISO {waldata['date']}. Failed to load from {jsonfile} JSON file"
|
||||
from troggle.core.models.troggle import DataIssue
|
||||
|
||||
DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
|
||||
except:
|
||||
message = f"! {str(self.walletname)} Date format not ISO {datestr}. Failed to load from {jsonfile} JSON file"
|
||||
from troggle.core.models.troggle import DataIssue
|
||||
|
||||
DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl)
|
||||
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
|
||||
return waldata
|
||||
|
||||
def year(self):
|
||||
|
||||
@ -18,7 +18,7 @@ from troggle.core.models.caves import Cave
|
||||
from troggle.core.models.logbooks import LogbookEntry # , PersonLogEntry
|
||||
from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole
|
||||
from troggle.core.models.troggle import DataIssue, Expedition
|
||||
from troggle.core.models.wallets import Wallet, YEAR_RANGE
|
||||
from troggle.core.models.wallets import Wallet, YEAR_RANGE, make_valid_date
|
||||
|
||||
from troggle.core.views.auth import login_required_if_public
|
||||
from troggle.core.views.caves import getCave
|
||||
@ -32,7 +32,7 @@ from troggle.parsers.scans import contentsjson
|
||||
"""
|
||||
|
||||
todo = """
|
||||
- Nasty bug in navingating to 'previous wallet' when we have a 2-year gap in expos
|
||||
- Nasty bug in navigating to 'previous wallet' when we have a 2-year gap in expos
|
||||
The xxxx#00 wallet is not getting edited correctly. Something is off by one somewhere..
|
||||
|
||||
- Register uploaded filenames in the Django db without needing to wait for a reset & bulk file import
|
||||
@ -496,25 +496,7 @@ def walletedit(request, path=None):
|
||||
or thing == "[]"
|
||||
or thing is None)
|
||||
|
||||
def make_valid_date(date):
|
||||
datestr = date.replace(".", "-")
|
||||
try:
|
||||
samedate = datetime.date.fromisoformat(datestr)
|
||||
except ValueError:
|
||||
# probably a single digit day number. HACKUS MAXIMUS.
|
||||
# clearly we need to fix this when we first import date strings..
|
||||
datestr = datestr[:-1] + "0" + datestr[-1]
|
||||
# datestr = f"{datestr:02d}"
|
||||
print(f"! - ValueError, trying.. {datestr=} ")
|
||||
try:
|
||||
samedate = datetime.date.fromisoformat(datestr)
|
||||
except:
|
||||
try:
|
||||
samedate = datetime.date.fromisoformat(datestr[:10])
|
||||
except:
|
||||
print(f"! - ValueError, FAILED {datestr=} ")
|
||||
samedate = None
|
||||
return samedate
|
||||
|
||||
|
||||
def scan_survexblocks(svxfile):
|
||||
"""Scans for *team people attached to all the survex blocks in this svxfile
|
||||
|
||||
@ -27,7 +27,7 @@ def import_people():
|
||||
troggle.parsers.people.load_people_expos()
|
||||
|
||||
def import_surveyscans():
|
||||
print("-- Importing Survey Scans")
|
||||
print("-- Importing Survey Scans and Wallets")
|
||||
with transaction.atomic():
|
||||
troggle.parsers.scans.load_all_scans()
|
||||
|
||||
|
||||
@ -38,6 +38,7 @@ def load_all_scans():
|
||||
Wallet.objects.all().delete()
|
||||
print(" - deleting all Wallet and SingleScan objects")
|
||||
DataIssue.objects.filter(parser="scans").delete()
|
||||
DataIssue.objects.filter(parser="wallets").delete()
|
||||
|
||||
# These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet.
|
||||
valids = [
|
||||
|
||||
Loading…
Reference in New Issue
Block a user