expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-25 00:31:55 +00:00
Code Issues 19 Projects 3 Activity

making it work with 1626 no schonberg

Browse Source
This commit is contained in:
Philip Sargent 2022-07-23 19:26:47 +03:00
parent 7288bd9da3
commit a1e538e93a
6 changed files with 159494 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -128,3 +128,4 @@ media/jslib/openlayers/Lang/zh-TW.js
_test_response.html _test_response.html
_deploy/wsl/localsettingsWSL.py.bak _deploy/wsl/localsettingsWSL.py.bak
therionrefs.log therionrefs.log
_1623-and-1626.svx

159299
_1623-and-1626-no-schoenberg-hs.svx Normal file
View File

File diff suppressed because it is too large Load Diff

163
parsers/caves.py
View File

@ -87,8 +87,10 @@ def do_pending_cave(k, url, area_1623):
''' '''
default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists
in expoweb/cave_data/1623-"k".html in expoweb/cave_data/1623-"k".html
oops. Now need to do for 1626 area too
''' '''
slug = "1623-" + k slug = k
default_note = f"_Survex file found in loser repo but no description in expoweb <br><br><br>\n" default_note = f"_Survex file found in loser repo but no description in expoweb <br><br><br>\n"
default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then " default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then "
@ -151,78 +153,6 @@ def do_pending_cave(k, url, area_1623):
print(message) print(message)
def readcaves():
'''Reads the xml-format HTML files in the EXPOWEB repo, not from the loser repo.
'''
# For those caves which do not have cave_data/1623-xxx.html XML files even though they exist and have surveys
# should put this in a simple list which can be edited using 'Edit this file'
pending = set()
fpending = Path(CAVEDESCRIPTIONS, "pendingcaves.txt")
if fpending.is_file():
with open(fpending, "r") as fo:
cids = fo.readlines()
for cid in cids:
pending.add(cid.rstrip('\n'))
with transaction.atomic():
print(" - Deleting Caves and Entrances")
# attempting to avoid MariaDB crash when doing this
try:
Area.objects.all().delete()
except:
pass
try:
Cave.objects.all().delete()
except:
pass
try:
Entrance.objects.all().delete()
except:
pass
# Clear the cave data issues and the caves as we are reloading
DataIssue.objects.filter(parser='areas').delete()
DataIssue.objects.filter(parser='caves').delete()
DataIssue.objects.filter(parser='caves ok').delete()
DataIssue.objects.filter(parser='entrances').delete()
print(" - Creating Areas 1623 and 1626")
# This crashes on the server with MariaDB even though a null parent is explicitly allowed.
area_1623= Area.objects.create(short_name = "1623", super=None)
print(" - Saving Area 1623")
area_1623.save()
area_1626= Area.objects.create(short_name = "1626", super=None)
print(" - Saving Area 1626")
area_1626.save()
print (" - Setting pending caves")
# Do this first, so that these empty entries are overwritten as they get properly created.
for k in pending:
url = "1623/" + k # Note we are not appending the .htm as we are modern folks now.
try:
do_pending_cave(k, url, area_1623)
except:
message = " ! Error. Cannot create pending cave and entrance, pending-id:{}".format(k)
DataIssue.objects.create(parser='caves', message=message)
print(message)
raise
with transaction.atomic():
print(" - settings.CAVEDESCRIPTIONS: ", CAVEDESCRIPTIONS)
print(" - Reading Entrances from entrance descriptions xml files")
for filename in next(os.walk(ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
# if filename.endswith('.html'):
# if Path(filename).stem[5:] in pending:
# print(f'Skipping pending entrance dummy file <{filename}>')
# else:
# readentrance(filename)
readentrance(filename)
print(" - Reading Caves from cave descriptions xml files")
for filename in next(os.walk(CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
if filename.endswith('.html'):
readcave(filename)
def readentrance(filename): def readentrance(filename):
global entrances_xslug global entrances_xslug
@ -484,4 +414,89 @@ def getXML(text, itemname, minItems = 1, maxItems = None, printwarnings = True,
"max": maxItems} + " in file " + context "max": maxItems} + " in file " + context
DataIssue.objects.create(parser='caves', message=message) DataIssue.objects.create(parser='caves', message=message)
print(message) print(message)
return items return items
def readcaves():
'''Reads the xml-format HTML files in the EXPOWEB repo, not from the loser repo.
'''
# For those caves which do not have cave_data/1623-xxx.html XML files even though they exist and have surveys
# should put this in a simple list which can be edited using 'Edit this file'
pending = set()
fpending = Path(CAVEDESCRIPTIONS, "pendingcaves.txt")
if fpending.is_file():
with open(fpending, "r") as fo:
cids = fo.readlines()
for cid in cids:
pending.add(cid.rstrip('\n'))
with transaction.atomic():
print(" - Deleting Caves and Entrances")
# attempting to avoid MariaDB crash when doing this
try:
Area.objects.all().delete()
except:
pass
try:
Cave.objects.all().delete()
except:
pass
try:
Entrance.objects.all().delete()
except:
pass
# Clear the cave data issues and the caves as we are reloading
DataIssue.objects.filter(parser='areas').delete()
DataIssue.objects.filter(parser='caves').delete()
DataIssue.objects.filter(parser='caves ok').delete()
DataIssue.objects.filter(parser='entrances').delete()
print(" - Creating Areas 1623 and 1626")
# This crashes on the server with MariaDB even though a null parent is explicitly allowed.
area_1623= Area.objects.create(short_name = "1623", super=None)
print(" - Saving Area 1623")
area_1623.save()
area_1624= Area.objects.create(short_name = "1624", super=None)
print(" - Saving Area 1624")
area_1624.save()
area_1626= Area.objects.create(short_name = "1626", super=None)
print(" - Saving Area 1626")
area_1626.save()
print (" - Setting pending caves")
# Do this first, so that these empty entries are overwritten as they get properly created.
for k in pending:
url = k.replace("-","/") # Note we are not appending the .htm as we are modern folks now.
area = area_1623
areanum = k[0:3]
if areanum == "1623":
area = area_1623
if areanum == "1624":
area = area_1624
if areanum == "1626":
area = area_1626
try:
do_pending_cave(k, url, area)
except:
message = f" ! Error. Cannot create pending cave and entrance, pending-id:{k} in area {areanum}"
DataIssue.objects.create(parser='caves', message=message)
print(message)
raise
with transaction.atomic():
print(" - settings.CAVEDESCRIPTIONS: ", CAVEDESCRIPTIONS)
print(" - Reading Entrances from entrance descriptions xml files")
for filename in next(os.walk(ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
# if filename.endswith('.html'):
# if Path(filename).stem[5:] in pending:
# print(f'Skipping pending entrance dummy file <{filename}>')
# else:
# readentrance(filename)
readentrance(filename)
print(" - Reading Caves from cave descriptions xml files")
for filename in next(os.walk(CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files
if filename.endswith('.html'):
readcave(filename)

2
parsers/logbooks.py
View File

@ -59,7 +59,7 @@ noncaveplaces = [ "QMplaceholder", "Journey", "Loser Plateau", "UNKNOWN", 'plate
logdataissues = TROG['issues']['logdataissues'] logdataissues = TROG['issues']['logdataissues']
trips ={} trips ={}
entries = { "2019": 36, "2018": 74, "2017": 60, "2016": 81, "2015": 79, entries = { "2019": 44, "2018": 74, "2017": 60, "2016": 81, "2015": 79,
"2014": 65, "2013": 51, "2012": 75, "2011": 68, "2010": 22, "2009": 52, "2014": 65, "2013": 51, "2012": 75, "2011": 68, "2010": 22, "2009": 52,
"2008": 49, "2007": 111, "2006": 60, "2005": 55, "2004": 76, "2003": 40, "2002": 31, "2008": 49, "2007": 111, "2006": 60, "2005": 55, "2004": 76, "2003": 40, "2002": 31,
"2001": 48, "2000": 54, "1999": 79, "1998": 43, "1997": 53, "1996": 94, "1995": 41, "2001": 48, "2000": 54, "1999": 79, "1998": 43, "1997": 53, "1996": 94, "1995": 41,

174
parsers/survex.py
View File

@ -86,6 +86,9 @@ class MapLocations(object):
def __str__(self): def __str__(self):
return "{} map locations".format(len(self.p)) return "{} map locations".format(len(self.p))
def get_offending_filename(path):
return "/survexfile/" + path + ".svx"
class SurvexLeg(): class SurvexLeg():
"""No longer a models.Model subclass, so no longer a database table """No longer a models.Model subclass, so no longer a database table
@ -186,7 +189,7 @@ class LoadingSurvex():
def __init__(self): def __init__(self):
self.caveslist = GetCaveLookup() self.caveslist = GetCaveLookup()
pass pass
def LoadSurvexFallThrough(self, survexblock, line, cmd): def LoadSurvexFallThrough(self, survexblock, line, cmd):
if cmd == "require": if cmd == "require":
pass # should we check survex version available for processing? pass # should we check survex version available for processing?
@ -195,13 +198,13 @@ class LoadingSurvex():
pass # we ignore all these, which is fine. pass # we ignore all these, which is fine.
else: else:
if cmd in ["include", "data", "flags", "title", "entrance","set", "units", "alias", "ref"]: if cmd in ["include", "data", "flags", "title", "entrance","set", "units", "alias", "ref"]:
message = "! Unparsed [*{}]: '{}' {} - not an error (probably)".format(cmd, line, survexblock.survexfile.path) message = "! Warning. Unparsed [*{}]: '{}' {} - not an error (probably)".format(cmd, line, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
else: else:
message = "! Bad svx command: [*{}] {} ({}) {}".format(cmd, line, survexblock, survexblock.survexfile.path) message = "! Bad svx command: [*{}] {} ({}) {}".format(cmd, line, survexblock, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
def LoadSurvexTeam(self, survexblock, line): def LoadSurvexTeam(self, survexblock, line):
"""Interpeting the *team fields means interpreting older style survex as well as current survex standard, """Interpeting the *team fields means interpreting older style survex as well as current survex standard,
@ -241,7 +244,7 @@ class LoadingSurvex():
self.flagsstar["splayalias"] = True self.flagsstar["splayalias"] = True
else: else:
message = "! Bad *ALIAS: '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) message = "! Bad *ALIAS: '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message)
def LoadSurvexUnits(self, survexblock, line): def LoadSurvexUnits(self, survexblock, line):
@ -255,7 +258,7 @@ class LoadingSurvex():
self.unitsfactor = float(factor) self.unitsfactor = float(factor)
if debugprint: if debugprint:
message = "! *UNITS NUMERICAL conversion [{}x] '{}' ({}) {}".format(factor, line, survexblock, survexblock.survexfile.path) message = "! *UNITS NUMERICAL conversion [{}x] '{}' ({}) {}".format(factor, line, survexblock, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survexunits', message=message) DataIssue.objects.create(parser='survexunits', message=message)
feet = re.match("(?i).*feet$",line) feet = re.match("(?i).*feet$",line)
@ -266,7 +269,7 @@ class LoadingSurvex():
self.units = "metres" self.units = "metres"
else: else:
message = "! *UNITS in YARDS!? - not converted '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) message = "! *UNITS in YARDS!? - not converted '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survexunits', message=message) DataIssue.objects.create(parser='survexunits', message=message)
def LoadSurvexDate(self, survexblock, line): def LoadSurvexDate(self, survexblock, line):
@ -282,7 +285,7 @@ class LoadingSurvex():
expeditions = findexpedition(year) expeditions = findexpedition(year)
if len(expeditions) != 1 : if len(expeditions) != 1 :
message = f"! More than one expedition in year {year} '{line}' ({survexblock}) {survexblock.survexfile.path}" message = f"! More than one expedition in year {year} '{line}' ({survexblock}) {survexblock.survexfile.path}"
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survexunits', message=message) DataIssue.objects.create(parser='survexunits', message=message)
expo= expeditions[0] expo= expeditions[0]
@ -311,8 +314,8 @@ class LoadingSurvex():
setdate(year) setdate(year)
else: else:
message = "! DATE unrecognised '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path) message = "! DATE unrecognised '{}' ({}) {}".format(line, survexblock, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
def LoadSurvexLeg(self, survexblock, sline, comment): def LoadSurvexLeg(self, survexblock, sline, comment):
"""This reads compass, clino and tape data but only keeps the tape lengths, """This reads compass, clino and tape data but only keeps the tape lengths,
@ -352,30 +355,36 @@ class LoadingSurvex():
ls = sline.lower().split() ls = sline.lower().split()
# skip all splay legs # skip all splay legs
if ls[datastar["from"]] == ".." or ls[datastar["from"]] == ".": try:
if debugprint: if ls[datastar["from"]] == ".." or ls[datastar["from"]] == ".":
print("Splay in ", survexblock.survexfile.path)
return
if ls[datastar["to"]] == ".." or ls[datastar["to"]] == ".":
if debugprint:
print("Splay in ", survexblock.survexfile.path)
return
if self.flagsstar["splayalias"]:
if ls[datastar["from"]] == "-":
if debugprint: if debugprint:
print("Aliased splay in ", survexblock.survexfile.path) print("Splay in ", survexblock.survexfile.path)
return return
if ls[datastar["to"]] == "-": if ls[datastar["to"]] == ".." or ls[datastar["to"]] == ".":
if debugprint: if debugprint:
print("Aliased splay in ", survexblock.survexfile.path) print("Splay in ", survexblock.survexfile.path)
return return
if self.flagsstar["splayalias"]:
if ls[datastar["from"]] == "-":
if debugprint:
print("Aliased splay in ", survexblock.survexfile.path)
return
if ls[datastar["to"]] == "-":
if debugprint:
print("Aliased splay in ", survexblock.survexfile.path)
return
except:
message = ' ! datastar parsing from/to incorrect in line %s in %s' % (ls, survexblock.survexfile.path)
print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
return
try: try:
tape = ls[datastar["tape"]] tape = ls[datastar["tape"]]
except: except:
message = ' ! datastar parsing incorrect in line %s in %s' % (ls, survexblock.survexfile.path) message = ' ! datastar parsing incorrect in line %s in %s' % (ls, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message) DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexleg.tape = invalid_tape survexleg.tape = invalid_tape
return return
# e.g. '29/09' or '(06.05)' in the tape measurement # e.g. '29/09' or '(06.05)' in the tape measurement
@ -387,34 +396,34 @@ class LoadingSurvex():
tape = float(tape) * self.unitsfactor tape = float(tape) * self.unitsfactor
if debugprint: if debugprint:
message = " ! Units: Length scaled {}m '{}' in ({}) units:{} factor:{}x".format(tape, ls, survexblock.survexfile.path, self.units, self.unitsfactor) message = " ! Units: Length scaled {}m '{}' in ({}) units:{} factor:{}x".format(tape, ls, survexblock.survexfile.path, self.units, self.unitsfactor)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message) DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
if self.units =="feet": if self.units =="feet":
tape = float(tape) / METRESINFEET tape = float(tape) / METRESINFEET
if debugprint: if debugprint:
message = " ! Units: converted to {:.3f}m from {} '{}' in ({})".format(tape, self.units, ls, survexblock.survexfile.path) message = " ! Units: converted to {:.3f}m from {} '{}' in ({})".format(tape, self.units, ls, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message) DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexleg.tape = float(tape) survexleg.tape = float(tape)
self.legsnumber += 1 self.legsnumber += 1
except ValueError: except ValueError:
message = " ! Value Error: Tape misread in line'{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units) message = " ! Value Error: Tape misread in line'{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message) DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexleg.tape = invalid_tape survexleg.tape = invalid_tape
try: try:
survexblock.legslength += survexleg.tape survexblock.legslength += survexleg.tape
self.slength += survexleg.tape self.slength += survexleg.tape
except ValueError: except ValueError:
message = " ! Value Error: Tape length not added '{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units) message = " ! Value Error: Tape length not added '{}' in {} units:{}".format(ls, survexblock.survexfile.path, self.units)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message) DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
try: try:
lcompass = ls[datastar["compass"]] lcompass = ls[datastar["compass"]]
except: except:
message = ' ! Value Error: Compass not found in line %s in %s' % (ls, survexblock.survexfile.path) message = ' ! Value Error: Compass not found in line %s in %s' % (ls, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survexleg', message=message) DataIssue.objects.create(parser='survexleg', message=message)
lcompass = invalid_compass lcompass = invalid_compass
@ -425,7 +434,7 @@ class LoadingSurvex():
print((" datastar:", datastar)) print((" datastar:", datastar))
print((" Line:", ls)) print((" Line:", ls))
message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path) message = ' ! Value Error: Clino misread in line %s in %s' % (ls, survexblock.survexfile.path)
DataIssue.objects.create(parser='survexleg', message=message) DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
lclino = invalid_clino lclino = invalid_clino
if lclino == "up": if lclino == "up":
@ -445,7 +454,7 @@ class LoadingSurvex():
print((" Line:", ls)) print((" Line:", ls))
message = " ! Value Error: lcompass:'{}' line {} in '{}'".format(lcompass, message = " ! Value Error: lcompass:'{}' line {} in '{}'".format(lcompass,
ls, survexblock.survexfile.path) ls, survexblock.survexfile.path)
DataIssue.objects.create(parser='survexleg', message=message) DataIssue.objects.create(parser='survexleg', message=message, url=get_offending_filename(survexblock.survexfile.path))
survexleg.compass = invalid_compass survexleg.compass = invalid_compass
# delete the object to save memory # delete the object to save memory
@ -453,17 +462,17 @@ class LoadingSurvex():
def LoadSurvexRef(self, survexblock, args): def LoadSurvexRef(self, survexblock, args):
#print(self.insp+ "*REF ---- '"+ args +"'") #print(self.insp+ "*REF ---- '"+ args +"'")
url=f'/survexfile/{survexblock.survexfile.path}' url= get_offending_filename(survexblock.survexfile.path)
# *REF but also ; Ref years from 1960 to 2039 # *REF but also ; Ref years from 1960 to 2039
refline = self.rx_ref_text.match(args) refline = self.rx_ref_text.match(args)
if refline: if refline:
# a textual reference such as "1996-1999 Not-KH survey book pp 92-95" # a textual reference such as "1996-1999 Not-KH survey book pp 92-95"
print(self.insp+ "*REF quoted text so ignored:"+ args) print(f'{self.insp} *REF quoted text so ignored:{args} in {survexblock.survexfile.path}')
return return
if len(args)< 4: if len(args)< 4:
message = " ! Empty or BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path) message = " ! Empty or BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url) DataIssue.objects.create(parser='survex', message=message, url=url)
return return
@ -484,18 +493,18 @@ class LoadingSurvex():
wallet = "0" + wallet wallet = "0" + wallet
if not (int(yr)>1960 and int(yr)<2039): if not (int(yr)>1960 and int(yr)<2039):
message = " ! Wallet year out of bounds {yr} '{refscan}' {survexblock.survexfile.path}" message = " ! Wallet year out of bounds {yr} '{refscan}' {survexblock.survexfile.path}"
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url) DataIssue.objects.create(parser='survex', message=message, url=url)
refscan = "%s#%s%s" % (yr, letterx, wallet) refscan = "%s#%s%s" % (yr, letterx, wallet)
try: try:
if int(wallet)>99: if int(wallet)>99:
message = " ! Wallet *REF {} - too big in '{}'".format(refscan, survexblock.survexfile.path) message = " ! Wallet *REF {} - very big (more than 99) so probably wrong in '{}'".format(refscan, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url) DataIssue.objects.create(parser='survex', message=message, url=url)
except: except:
message = " ! Wallet *REF {} - not numeric in '{}' -- parsing continues".format(refscan, survexblock.survexfile.path) message = " ! Wallet *REF {} - not numeric in '{}' -- parsing continues".format(refscan, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url) DataIssue.objects.create(parser='survex', message=message, url=url)
manywallets = Wallet.objects.filter(walletname=refscan) manywallets = Wallet.objects.filter(walletname=refscan)
if manywallets: if manywallets:
@ -504,11 +513,11 @@ class LoadingSurvex():
survexblock.save() survexblock.save()
if len(manywallets) > 1: if len(manywallets) > 1:
message = " ! Wallet *REF {} - more than one found {} scan folders in {}".format(refscan, len(manywallets), survexblock.survexfile.path) message = " ! Wallet *REF {} - more than one found {} scan folders in {}".format(refscan, len(manywallets), survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url) DataIssue.objects.create(parser='survex', message=message, url=url)
else: else:
message = " ! Wallet *REF '{}' - NOT found '{}'".format(refscan, survexblock.survexfile.path) message = " ! Wallet *REF '{}' - NOT found '{}'".format(refscan, survexblock.survexfile.path)
print((self.insp+message)) print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url) DataIssue.objects.create(parser='survex', message=message, url=url)
def TickSurvexQM(self, survexblock, qmtick): def TickSurvexQM(self, survexblock, qmtick):
@ -520,11 +529,11 @@ class LoadingSurvex():
#raise #raise
message = f' ! QM TICK find FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" qmlist:"{qm}" in "{survexblock.survexfile.path}" + comment:"{qmtick.group(3)}" ' message = f' ! QM TICK find FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" qmlist:"{qm}" in "{survexblock.survexfile.path}" + comment:"{qmtick.group(3)}" '
print(message) print(message)
DataIssue.objects.create(parser='survex', message=message, url=f'/survexfile/{survexblock.survexfile.path}.svx') DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
if len(qm)>1: if len(qm)>1:
message = f' ! QM TICK MULTIPLE found FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" in "{survexblock.survexfile.path}" + comment:"{qmtick.group(3)}" ' message = f' ! QM TICK MULTIPLE found FAIL QM{qmtick.group(1)} date:"{qmtick.group(2)}" in "{survexblock.survexfile.path}" + comment:"{qmtick.group(3)}" '
print(message) print(message)
DataIssue.objects.create(parser='survex', message=message, url=f'/survexfile/{survexblock.survexfile.path}.svx') DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
qm[0].ticked = True qm[0].ticked = True
qm[0].save() qm[0].save()
@ -578,14 +587,10 @@ class LoadingSurvex():
expoyear = str(survexblock.date.year), expoyear = str(survexblock.date.year),
cave = survexblock.survexfile.cave) cave = survexblock.survexfile.cave)
qm.save qm.save
if survexblock.survexfile.cave.kataster_number == "359":
message = " ! QM{} '{}' CREATED in DB in '{}'".format(qm_no, qm_nearest,survexblock.survexfile.path)
print(insp+message)
DataIssue.objects.create(parser='survex', message=message)
except: except:
message = " ! QM{} FAIL to create {} in'{}'".format(qm_no, qm_nearest,survexblock.survexfile.path) message = " ! QM{} FAIL to create {} in'{}'".format(qm_no, qm_nearest,survexblock.survexfile.path)
print(insp+message) print(insp+message)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
def LoadSurvexDataNormal(self,survexblock,args): def LoadSurvexDataNormal(self,survexblock,args):
"""Sets the order for data elements in this and following blocks, e.g. """Sets the order for data elements in this and following blocks, e.g.
@ -614,12 +619,19 @@ class LoadingSurvex():
message = " ! - Unrecognised *data normal statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath) message = " ! - Unrecognised *data normal statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath)
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
return return
else: else:
datastar = self.datastardefault datastar = self.datastardefault
# ls = ["normal", "from", "to", "tape", "compass", "clino" ] # ls = ["normal", "from", "to", "tape", "compass", "clino" ]
for i in range(1, len(ls)): # len[0] is "normal" for i in range(1, len(ls)): # len[0] is "normal"
if ls[i].lower() =="newline":
message = f" ! - ABORT *data statement has NEWLINE in it in {survexblock.survexfile.path}. Not parsed by torggle. '{args}'"
print(message)
print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
return False
if ls[i] in ["bearing","compass"]: if ls[i] in ["bearing","compass"]:
datastar["compass"] = i-1 datastar["compass"] = i-1
if ls[i] in ["clino","gradient"]: if ls[i] in ["clino","gradient"]:
@ -638,7 +650,7 @@ class LoadingSurvex():
message = " ! - Unrecognised *data statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath) message = " ! - Unrecognised *data statement '{}' {}|{}".format(args, survexblock.name, survexblock.survexpath)
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
def LoadSurvexFlags(self, args): def LoadSurvexFlags(self, args):
# Valid flags are DUPLICATE, SPLAY, and SURFACE, and a flag may be preceded with NOT to turn it off. # Valid flags are DUPLICATE, SPLAY, and SURFACE, and a flag may be preceded with NOT to turn it off.
@ -709,7 +721,7 @@ class LoadingSurvex():
self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory self.survexdict[self.svxdirs[headpath.lower()]] = [] # list of the files in the directory
return self.svxdirs[headpath.lower()] return self.svxdirs[headpath.lower()]
def ReportNonCaveIncludes(self, headpath, includelabel): def ReportNonCaveIncludes(self, headpath, includelabel, depth):
"""Ignore surface, kataser and gpx *include survex files """Ignore surface, kataser and gpx *include survex files
""" """
if headpath in self.ignorenoncave: if headpath in self.ignorenoncave:
@ -723,11 +735,11 @@ class LoadingSurvex():
#print("\n"+message) #print("\n"+message)
#print("\n"+message,file=sys.stderr) #print("\n"+message,file=sys.stderr)
return return
message = f" ! Error: '{headpath}' FAILURE (while creating '{includelabel}' in db - not a cave or in the ignore list of surface surveys. )" message = f" ! Error: FAILURE '{headpath}' while creating '{includelabel}' at depth:'{depth}'. Not a cave or in the ignore list:'{self.ignoreprefix}'"
# getting this triggered for gpx/2018 (cavern error) but not for gpx/2017 (no content). # getting this triggered for gpx/2018 (cavern error) but not for gpx/2017 (no content).
print("\n"+message) print("\n"+message)
print("\n"+message,file=sys.stderr) print("\n"+message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(headpath))
print(f' # datastack in LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr) print(f' # datastack in LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr)
for dict in self.datastack: for dict in self.datastack:
print(f'{dict["type"].upper()} ', end="",file=sys.stderr) print(f'{dict["type"].upper()} ', end="",file=sys.stderr)
@ -763,14 +775,14 @@ class LoadingSurvex():
message = " ! 'None' SurvexDirectory returned from GetSurvexDirectory({})".format(headpath) message = " ! 'None' SurvexDirectory returned from GetSurvexDirectory({})".format(headpath)
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url = f'/survexfile/{svxid}')
if cave: if cave:
newdirectory.cave = cave newdirectory.cave = cave
newfile.cave = cave newfile.cave = cave
# print(f"\n - New directory {newdirectory} for cave {newdirectory.cave}",file=sys.stderr) # print(f"\n - New directory {newdirectory} for cave {newdirectory.cave}",file=sys.stderr)
else: # probably a surface survey else: # probably a surface survey, or a cave in a new area e.g. 1624 not previously managed, and not in the pending list
self.ReportNonCaveIncludes(headpath, svxid) self.ReportNonCaveIncludes(headpath, svxid, depth)
if not newfile.survexdirectory: if not newfile.survexdirectory:
message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(svxid) message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(svxid)
@ -845,7 +857,7 @@ class LoadingSurvex():
else: else:
message = f' ! QM Unrecognised as valid in "{survexblock.survexfile.path}" QM{qml.group(1)} "{qml.group(2)}" : regex failure, typo?' message = f' ! QM Unrecognised as valid in "{survexblock.survexfile.path}" QM{qml.group(1)} "{qml.group(2)}" : regex failure, typo?'
print(message) print(message)
DataIssue.objects.create(parser='survex', message=message, url=f'/survexfile/{survexblock.survexfile.path}.svx') DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(survexblock.survexfile.path))
included = self.rx_comminc.match(comment) included = self.rx_comminc.match(comment)
@ -1068,7 +1080,11 @@ class LoadingSurvex():
print(" # CHANGE 'any' flag now:'{}' was:{} ".format(self.flagsstar["skiplegs"], oldflags["skiplegs"])) print(" # CHANGE 'any' flag now:'{}' was:{} ".format(self.flagsstar["skiplegs"], oldflags["skiplegs"]))
elif self.rx_data.match(cmd): elif self.rx_data.match(cmd):
self.LoadSurvexDataNormal(survexblock, args) if self.LoadSurvexDataNormal(survexblock, args):
pass
else:
# Abort, we do not cope with this *data format
return
elif self.rx_alias.match(cmd): elif self.rx_alias.match(cmd):
self.LoadSurvexAlias(survexblock, args) self.LoadSurvexAlias(survexblock, args)
elif self.rx_entrance.match(cmd): elif self.rx_entrance.match(cmd):
@ -1085,7 +1101,7 @@ class LoadingSurvex():
message = " ! -ERROR *include command not expected here {}. Re-run a full Survex import.".format(path) message = " ! -ERROR *include command not expected here {}. Re-run a full Survex import.".format(path)
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, )
else: else:
self.LoadSurvexFallThrough(survexblock, args, cmd) self.LoadSurvexFallThrough(survexblock, args, cmd)
@ -1120,7 +1136,7 @@ class LoadingSurvex():
self.legsnumber = nlegstotal self.legsnumber = nlegstotal
self.slength = slengthtotal self.slength = slengthtotal
def PushdownStackScan(self, survexblock, path, fin, flinear, fcollate): def PushdownStackScan(self, survexblock, path, fin, flinear, fcollate):
"""Follows the *include links in all the survex files from the root file 1623.svx """Follows the *include links in all the survex files from the root file 1623.svx
and reads only the *include and *begin and *end statements. It produces a linearised and reads only the *include and *begin and *end statements. It produces a linearised
@ -1130,23 +1146,24 @@ class LoadingSurvex():
indent = " " * self.depthinclude indent = " " * self.depthinclude
sys.stderr.flush(); sys.stderr.flush();
self.callcount +=1 self.callcount +=1
if self.callcount % 10 ==0 : if self.callcount % 10 ==0 :
print(".", file=sys.stderr,end='') print(".", file=sys.stderr,end='')
if self.callcount % 500 ==0 : if self.callcount % 500 ==0 :
print("\n ", file=sys.stderr,end='') print("\n ", file=sys.stderr,end='')
if path in self.svxfileslist: if path in self.svxfileslist:
message = " * Warning. Duplicate detected in *include list at callcount:{} depth:{} file:{}".format(self.callcount, self.depthinclude, path) message = f" * Warning. Duplicate detected. We have already seen this *include '{path}' from another survex file. Detected at callcount:{self.callcount} depth:{self.depthinclude}"
print(message) print(message)
print(message,file=flinear) print(message,file=flinear)
print("\n"+message,file=sys.stderr) print("\n"+message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
if self.svxfileslist.count(path) > 20: if self.svxfileslist.count(path) > 20:
message = " ! ERROR. Survex file already seen 20x. Probably an infinite loop so fix your *include statements that include this. Aborting. {}".format(path) message = " ! ERROR. Survex file already *included 20x. Probably an infinite loop so fix your *include statements that include this. Aborting. {}".format(path)
print(message) print(message)
print(message,file=flinear) print(message,file=flinear)
print(message,file=sys.stderr) print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
return return
self.svxfileslist.append(path) self.svxfileslist.append(path)
@ -1157,8 +1174,7 @@ class LoadingSurvex():
message = f" ! ERROR *include file '{path}' in '{survexblock}' has UnicodeDecodeError" message = f" ! ERROR *include file '{path}' in '{survexblock}' has UnicodeDecodeError"
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
offendingfile = "/survexfile/" + path + ".svx" DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
DataIssue.objects.create(parser='survex', message=message, url=offendingfile)
return # skip this survex file and all things *included in it return # skip this survex file and all things *included in it
@ -1173,7 +1189,7 @@ class LoadingSurvex():
message = message + f" - line {self.lineno} {survexblock}\n - Parsing aborted. NERD++ needed to fix it" message = message + f" - line {self.lineno} {survexblock}\n - Parsing aborted. NERD++ needed to fix it"
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
return # skip this survex file and all things *included in it return # skip this survex file and all things *included in it
includestmt =self.rx_include.match(svxline) includestmt =self.rx_include.match(svxline)
@ -1208,7 +1224,7 @@ class LoadingSurvex():
print(message) print(message)
print(message,file=flinear) print(message,file=flinear)
print(message,file=sys.stderr) print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop)) flinear.write("{:2} {} *edulcni {}\n".format(self.depthinclude, indent, pop))
fcollate.write(";*edulcni {}\n".format(pop)) fcollate.write(";*edulcni {}\n".format(pop))
fininclude.close() fininclude.close()
@ -1218,7 +1234,7 @@ class LoadingSurvex():
message = " ! ERROR *include file not found for:'{}'".format(includepath) message = " ! ERROR *include file not found for:'{}'".format(includepath)
print(message) print(message)
print(message,file=sys.stderr) print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
elif re.match("(?i)begin$", cmd): elif re.match("(?i)begin$", cmd):
self.depthbegin += 1 self.depthbegin += 1
depth = " " * self.depthbegin depth = " " * self.depthbegin
@ -1240,7 +1256,7 @@ class LoadingSurvex():
print(message) print(message)
print(message,file=flinear) print(message,file=flinear)
print(message,file=sys.stderr) print(message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message) DataIssue.objects.create(parser='survex', message=message, url=get_offending_filename(path))
self.depthbegin -= 1 self.depthbegin -= 1
pass pass
@ -1302,6 +1318,12 @@ class LoadingSurvex():
logpath = Path(fullpath + ".log") logpath = Path(fullpath + ".log")
outputdir = Path(svxpath).parent outputdir = Path(svxpath).parent
if not svxpath.is_file():
message = f' ! BAD survex file "{fullpath}" specified in *include (somewhere).. '
DataIssue.objects.create(parser='entrances', message=message)
print(message)
return
if not logpath.is_file(): # always run if logfile not there if not logpath.is_file(): # always run if logfile not there
runcavern() runcavern()
return return

11
settings.py
View File

@ -67,7 +67,7 @@ USE_L10N = True
FIX_PERMISSIONS = [] FIX_PERMISSIONS = []
# top-level survex file basename (without .svx) # top-level survex file basename (without .svx)
SURVEX_TOPNAME = "1623" SURVEX_TOPNAME = "1623-and-1626-no-schoenberg-hs"
MAX_LOGBOOK_ENTRY_TITLE_LENGTH = 200 MAX_LOGBOOK_ENTRY_TITLE_LENGTH = 200
DEFAULT_LOGBOOK_PARSER = "Parseloghtmltxt" DEFAULT_LOGBOOK_PARSER = "Parseloghtmltxt"
@ -108,10 +108,11 @@ LOGBOOK_PARSER_SETTINGS = {
} }
# Caves for which survex files exist, but are not otherwise registered # Caves for which survex files exist, but are not otherwise registered
PENDING = ["2007-06", "2009-02", # replaced (?) by expoweb/cave_data/pendingcaves.txt
"2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06", # PENDING = ["1626-361", "2007-06", "2009-02",
"2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888", # "2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06",
"2018-pf-01", "2018-pf-02"] # "2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888",
# "2018-pf-01", "2018-pf-02"]
APPEND_SLASH = False # never relevant because we have urls that match unknown files and produce an 'edit this page' response APPEND_SLASH = False # never relevant because we have urls that match unknown files and produce an 'edit this page' response
SMART_APPEND_SLASH = True #not eorking as middleware different after Dj2.0 SMART_APPEND_SLASH = True #not eorking as middleware different after Dj2.0