xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Fixing bugs and better error msgs

Browse Source
This commit is contained in:
Philip Sargent
2022-10-06 21:02:15 +03:00
parent 8d08a67302
commit 70709c505c
4 changed files with 167 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
parsers/survex.py
View File

@@ -523,7 +523,7 @@ class LoadingSurvex():
yr, letterx, wallet = argsgps.groups()
else:
perps = get_people_on_trip(survexblock)
message = f" ! Wallet *REF '{args}' malformed id in '{survexblock.survexfile.path}' {perps}"
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' malformed id '{args}' {perps}"
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
return
@@ -549,19 +549,25 @@ class LoadingSurvex():
message = " ! Wallet *REF {} - not numeric in '{}'".format(refscan, survexblock.survexfile.path)
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
manywallets = Wallet.objects.filter(walletname=refscan) # assumes all wallets found in earlier pass of data import
if manywallets:
survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
message = f' - Wallet {manywallets[0]=} successfully found in db. *ref in {survexblock.survexfile.path}'
# print(self.insp+message)
survexblock.save()
if len(manywallets) > 1:
message = " ! Wallet *REF {} - more than one found {} wallets in block {}".format(refscan, len(manywallets), survexblock.survexfile.path)
message = " ! Wallet *REF {} - more than one found {} wallets in db with same id {}".format(refscan, len(manywallets), survexblock.survexfile.path)
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
if survexblock.scanswallet:
if survexblock.scanswallet.walletname != refscan:
message = f" ! Wallet *REF {refscan} in {survexblock.survexfile.path} - Already a DIFFERENT wallet is set for this block '{survexblock.scanswallet.walletname}'"
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
else:
survexblock.scanswallet = manywallets[0] # this is a ForeignKey field
survexblock.save()
else:
perps = get_people_on_trip(survexblock)
message = f" ! Wallet *REF '{refscan}' in '{survexblock.survexfile.path}' {perps} NOT in database i.e. wallet does not exist."
message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}."
print(self.insp+message)
DataIssue.objects.create(parser='survex', message=message, url=url)
@@ -994,7 +1000,7 @@ class LoadingSurvex():
blockcount +=1
if blockcount % 20 ==0 :
print(".", file=sys.stderr,end='')
if blockcount % 400 ==0 :
if blockcount % 800 ==0 :
print("\n", file=sys.stderr,end='')
mem=get_process_memory()
print(" - MEM:{:7.3f} MB in use".format(mem),file=sys.stderr)
@@ -1603,16 +1609,12 @@ def FindAndLoadSurvex(survexblockroot):
# Before doing this, it would be good to identify the *equate and *entrance we need that are relevant to the
# entrance locations currently loaded after this by LoadPos(), but could better be done before ?
# look in MapLocations() for how we find the entrances
print('\n - Loading All Survex Blocks (LinearLoad)',file=sys.stderr)
svx_load = LoadingSurvex()
mem1 = get_process_memory()
print(" - MEM:{:7.2f} MB after creating empty loading object.".format(mem1),file=sys.stderr)
svx_load.survexdict[survexfileroot.survexdirectory] = []
svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot)
@@ -1624,6 +1626,8 @@ def FindAndLoadSurvex(survexblockroot):
svxlines = fcollate.read().splitlines()
#pr2 = cProfile.Profile()
#pr2.enable()
mem1 = get_process_memory()
print(f" - MEM:{mem1:7.2f} MB immediately after reading '{collatefilename}' into memory.",file=sys.stderr)
print(" ", file=sys.stderr,end='')
#----------------------------------------------------------------
svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines)
@@ -1633,6 +1637,7 @@ def FindAndLoadSurvex(survexblockroot):
# ps = pstats.Stats(pr2, stream=f)
# ps.sort_stats(SortKey.CUMULATIVE)
# ps.print_stats()
svxlines = [] # empty 30MB of stashed file
mem1 = get_process_memory()
print("\n - MEM:{:7.2f} MB STOP".format(mem1),file=sys.stderr)
print(" - MEM:{:7.3f} MB ADDITIONALLY USED".format(mem1-mem0),file=sys.stderr)
@@ -1748,24 +1753,24 @@ def LoadPositions():
capture_output=True, check=False, text=True) #check=False means exception not raised
if sp.returncode != 0:
message = f' ! Error: cavern: creating {file3d} in runcavern3()'
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='entrances', message=message)
print(message)
# find the errors in the 1623.log file
sp = subprocess.run(["grep", "error:", f"{topdata}.log"],
capture_output=True, check=False, text=True) #check=False means exception not raised
message = f' ! Error: cavern: {sp.stdout}'
DataIssue.objects.create(parser='survex', message=message)
message = f' ! Error: cavern: {sp.stdout} creating {file3d} '
DataIssue.objects.create(parser='entrances', message=message)
print(message)
except:
message = " ! CalledProcessError 'cavern' in runcavern3() at {topdata}."
DataIssue.objects.create(parser='survex', message=message)
message = f" ! CalledProcessError 'cavern' in runcavern3() at {topdata}."
DataIssue.objects.create(parser='entrances', message=message)
print(message)
if file3d.is_file():
message = " ! CalledProcessError. File permissions {file3d.stat().st_mode} on {str(file3d)}"
DataIssue.objects.create(parser='survex', message=message)
message = f" ! CalledProcessError. File permissions {file3d.stat().st_mode} on {str(file3d)}"
DataIssue.objects.create(parser='entrances', message=message)
print(message)
if file3d.is_file(): # might be an old one though
@@ -1776,7 +1781,7 @@ def LoadPositions():
if sp.returncode != 0:
print(f' ! Error: survexport creating {topdata}.pos in runcavern3().\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode))
except:
message = " ! CalledProcessError 'survexport' in runcavern3() at {file3d}."
message = f" ! CalledProcessError 'survexport' in runcavern3() at {file3d}."
DataIssue.objects.create(parser='entrances', message=message)
print(message)
else:
@@ -1837,9 +1842,9 @@ def LoadPositions():
try:
survexblockroot = SurvexBlock.objects.get(id=1)
except:
message = ' ! FAILED to find root SurvexBlock'
message = f' ! FAILED to find root SurvexBlock'
print(message)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='entrances', message=message)
raise
for line in posfile.readlines():
r = poslineregex.match(line)
@@ -1859,17 +1864,17 @@ def LoadPositions():
if len(sbqs)>1:
message = " ! MULTIPLE SurvexBlocks {:3} matching Entrance point {} {} '{}'".format(len(sbqs), blockpath, sid, id)
print(message)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='entrances', message=message)
sb = sbqs[0]
elif len(sbqs)<=0:
message = " ! ZERO SurvexBlocks matching Entrance point {} {} '{}'".format(blockpath, sid, id)
print(message)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='entrances', message=message)
sb = survexblockroot
except:
message = ' ! FAIL in getting SurvexBlock matching Entrance point {} {}'.format(blockpath, sid)
print(message)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='entrances', message=message)
try:
ss = SurvexStation(name=id, block=survexblockroot)
ss.x = float(x)
@@ -1880,7 +1885,7 @@ def LoadPositions():
except:
message = ' ! FAIL to create SurvexStation Entrance point {} {}'.format(blockpath, sid)
print(message)
DataIssue.objects.create(parser='survex', message=message)
DataIssue.objects.create(parser='entrances', message=message)
raise
print(" - {} SurvexStation entrances found.".format(found))