xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Fixing bad date parsing, better warning msgs

Browse Source
This commit is contained in:
Philip Sargent 2022-09-16 22:54:22 +03:00
parent ddfc677a1e
commit 68865a80ef
1 changed files with 21 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
parsers/survex.py
View File

@ -215,6 +215,7 @@ class LoadingSurvex():
GetPersonExpeditionNameLookup() needs to be fixed. GetPersonExpeditionNameLookup() needs to be fixed.
personrole is used to record that a person was on a trip, NOT the role they played. personrole is used to record that a person was on a trip, NOT the role they played.
(NB PersonTrip is a logbook thing)
""" """
teammembers = [ ] teammembers = [ ]
mteammember = self.rx_teammem.match(line) mteammember = self.rx_teammem.match(line)
@ -248,6 +249,7 @@ class LoadingSurvex():
def LoadSurvexUnits(self, survexblock, line): def LoadSurvexUnits(self, survexblock, line):
# all for 4 survex files with measurements in feet. bugger. # all for 4 survex files with measurements in feet. bugger.
# Won't need this once we move to using cavern output for lengths
tapeunits = self.rx_tapelng.match(line) # tape|length tapeunits = self.rx_tapelng.match(line) # tape|length
if not tapeunits: if not tapeunits:
return return
@ -272,12 +274,12 @@ class LoadingSurvex():
DataIssue.objects.create(parser='survexunits', message=message) DataIssue.objects.create(parser='survexunits', message=message)
def LoadSurvexDate(self, survexblock, line): def LoadSurvexDate(self, survexblock, line):
# we should make this a date RANGE for everything # we should make this a date RANGE for everything?
def findexpedition(year): def findexpedition(year):
return Expedition.objects.filter(year=year) return Expedition.objects.filter(year=year)
def setdate(year): def setdate(year):
# cacheing to save DB query on every block and to prepare for django-less troggle in future # cacheing to save DB query on every block
if year in self.expos: if year in self.expos:
expo = self.expos[year] expo = self.expos[year]
else: else:
@ -294,25 +296,35 @@ class LoadingSurvex():
survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date) survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date)
survexblock.save() survexblock.save()
oline = line
if len(line) > 10: if len(line) > 10:
if line[10] == "-": # message = "! DATE Warning LONG DATE '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
# print(self.insp+message)
# DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
if line[10] == "-": # ie a range, just look at first date
line = line[0:10] line = line[0:10]
if len(line) == 10: if len(line) == 10:
year = line[:4] year = line[:4]
# TO DO set to correct Austrian timezone Europe/Vienna # TO DO set to correct Austrian timezone Europe/Vienna ?
# %m and %d need leading zeros. Source svx files require them. # %m and %d need leading zeros. Source svx files require them.
survexblock.date = datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d') survexblock.date = datetime.strptime(line.replace('.','-'), '%Y-%m-%d')
setdate(year) setdate(year)
elif len(line) == 7: elif len(line) == 7:
year = line[:4] year = line[:4]
survexblock.date = datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m') # sets to first of month message = "! DATE Warning only accurate to the month, setting to 1st '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexblock.date = datetime.strptime(line.replace('.','-'), '%Y-%m') # sets to first of month
setdate(year) setdate(year)
elif len(line) == 4: elif len(line) == 4:
year = line[:4] year = line[:4]
message = "! DATE WARNING only accurate to the YEAR, setting to 1st January '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexblock.date = datetime.strptime(line, '%Y') # sets to January 1st survexblock.date = datetime.strptime(line, '%Y') # sets to January 1st
setdate(year) setdate(year)
else: else:
message = "! Error DATE unrecognised '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) message = "! DATE Error unrecognised '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
print(self.insp+message) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path)) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
@ -320,6 +332,8 @@ class LoadingSurvex():
"""This reads compass, clino and tape data but only keeps the tape lengths, """This reads compass, clino and tape data but only keeps the tape lengths,
the rest is discarded after error-checking. the rest is discarded after error-checking.
Now skipping the error checking - returns as soon as the leg is not one we count. Now skipping the error checking - returns as soon as the leg is not one we count.
REPLACE ALL THIS by reading the .log output of cavern for the file
""" """
invalid_clino = 180.0 invalid_clino = 180.0
invalid_compass = 720.0 invalid_compass = 720.0