From 68865a80efd95f30afc04f6cb83d3ac57000b98c Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@klebos.com>
Date: Fri, 16 Sep 2022 22:54:22 +0300
Subject: [PATCH] Fixing bad date parsing, better warning msgs
---
parsers/survex.py | 28 +++++++++++++++++++++-------
1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/parsers/survex.py b/parsers/survex.py
index 5e67f91..c08faee 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -215,6 +215,7 @@ class LoadingSurvex():
GetPersonExpeditionNameLookup() needs to be fixed.
personrole is used to record that a person was on a trip, NOT the role they played.
+ (NB PersonTrip is a logbook thing)
"""
teammembers = [ ]
mteammember = self.rx_teammem.match(line)
@@ -248,6 +249,7 @@ class LoadingSurvex():
def LoadSurvexUnits(self, survexblock, line):
# all for 4 survex files with measurements in feet. bugger.
+ # Won't need this once we move to using cavern output for lengths
tapeunits = self.rx_tapelng.match(line) # tape|length
if not tapeunits:
return
@@ -272,12 +274,12 @@ class LoadingSurvex():
DataIssue.objects.create(parser='survexunits', message=message)
def LoadSurvexDate(self, survexblock, line):
- # we should make this a date RANGE for everything
+ # we should make this a date RANGE for everything?
def findexpedition(year):
return Expedition.objects.filter(year=year)
def setdate(year):
- # cacheing to save DB query on every block and to prepare for django-less troggle in future
+ # cacheing to save DB query on every block
if year in self.expos:
expo = self.expos[year]
else:
@@ -294,25 +296,35 @@ class LoadingSurvex():
survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date)
survexblock.save()
+ oline = line
if len(line) > 10:
- if line[10] == "-":
+ # message = "! DATE Warning LONG DATE '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
+ # print(self.insp+message)
+ # DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
+ if line[10] == "-": # ie a range, just look at first date
line = line[0:10]
if len(line) == 10:
year = line[:4]
- # TO DO set to correct Austrian timezone Europe/Vienna
+ # TO DO set to correct Austrian timezone Europe/Vienna ?
# %m and %d need leading zeros. Source svx files require them.
- survexblock.date = datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d')
+ survexblock.date = datetime.strptime(line.replace('.','-'), '%Y-%m-%d')
setdate(year)
elif len(line) == 7:
year = line[:4]
- survexblock.date = datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m') # sets to first of month
+ message = "! DATE Warning only accurate to the month, setting to 1st '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
+ print(self.insp+message)
+ DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
+ survexblock.date = datetime.strptime(line.replace('.','-'), '%Y-%m') # sets to first of month
setdate(year)
elif len(line) == 4:
year = line[:4]
+ message = "! DATE WARNING only accurate to the YEAR, setting to 1st January '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
+ print(self.insp+message)
+ DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexblock.date = datetime.strptime(line, '%Y') # sets to January 1st
setdate(year)
else:
- message = "! Error DATE unrecognised '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
+ message = "! DATE Error unrecognised '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
@@ -320,6 +332,8 @@ class LoadingSurvex():
"""This reads compass, clino and tape data but only keeps the tape lengths,
the rest is discarded after error-checking.
Now skipping the error checking - returns as soon as the leg is not one we count.
+
+ REPLACE ALL THIS by reading the .log output of cavern for the file
"""
invalid_clino = 180.0
invalid_compass = 720.0