diff --git a/parsers/survex.py b/parsers/survex.py
index fc234d0..8f3b65d 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -6,7 +6,7 @@ import copy
import subprocess
from pathlib import Path
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, date
from django.utils.timezone import get_current_timezone
from django.utils.timezone import make_aware
@@ -32,7 +32,7 @@ todo = '''Also walk the entire tree in the :loser: repo looking for unconnected
wider troggle system (the name is not a hyperlink) - apparently randomly.
GetPersonExpeditionNameLookup() needs to be fixed.
-- fix THREEDCACHEDIR and put .3d files in same folder as .svx and fix CaveView
+-#BUG, if *date comes after *team, the person's date is not set at all. It needs re-setting at the endof the block.
- LoadSurvexFile() Creates a new current survexfile and valid .survexdirectory
The survexblock passed-in is not necessarily the parent. FIX THIS.
@@ -219,7 +219,7 @@ class LoadingSurvex():
teammembers.append((personexpedition, tm))
personrole = SurvexPersonRole(survexblock=survexblock, personexpedition=personexpedition, personname=tm)
personrole.save()
- personrole.expeditionday = survexblock.expeditionday
+ personrole.expeditionday = survexblock.expeditionday #BUG, if *date comes after *team, this is NOT SET.
if personexpedition:
personrole.person=personexpedition.person
self.currentpersonexped.append(personexpedition)
@@ -681,10 +681,14 @@ class LoadingSurvex():
#print("\n"+message)
#print("\n"+message,file=sys.stderr)
return
- message = f" ! {headpath} is not a fully-registered cave. (while creating '{includelabel}' sfile & sdirectory in survex parsing)"
+ message = f" ! Error: '{headpath}' not a cave or in the ignore list of surface surveys. (while creating '{includelabel}')"
print("\n"+message)
print("\n"+message,file=sys.stderr)
DataIssue.objects.create(parser='survex', message=message)
+ print(f' # datastack in LoadSurvexFile:{includelabel} type:', end="",file=sys.stderr)
+ for dict in self.datastack:
+ print(f'{dict["type"].upper()} ', end="",file=sys.stderr)
+
def LoadSurvexFile(self, svxid):
"""Creates SurvexFile in the database, and SurvexDirectory if needed
@@ -722,9 +726,9 @@ class LoadingSurvex():
newdirectory.cave = cave
newfile.cave = cave
#print("\n"+str(newdirectory.cave),file=sys.stderr)
- else:
+ else: # probably a surface survey
self.ReportNonCaveIncludes(headpath, svxid)
-
+
if not newfile.survexdirectory:
message = " ! SurvexDirectory NOT SET in new SurvexFile {} ".format(svxid)
print(message)
@@ -1200,36 +1204,47 @@ class LoadingSurvex():
svx_t = now - 365*24*3600
def runcavern():
- '''This assumes all survex files have unique names and they are taken from many folders but the output is all put
- into the same folder. A serious potential bug. Import parser checks uniqueness but much better not to do it like this.
- # see design docum in troggle/templates/cave.html
- # and views/caves.py rendercave()
+ '''regenerates the .3d file from the .svx if it is older than the svx file, or older than the software,
+ or randomly using chaosmonkey() just to keep things ticking over.
'''
print(f" - Regenerating stale (or chaos-monkeyed) cavern .log and .3d for '{fullpath}'\n at '{logpath}'\n")
print(f"days svx old: {(svx_t - log_t)/(24*3600):.1f} cav:{(cav_t - log_t)/(24*3600):.1f} log old: { (now - log_t)/(24*3600):.1f}")
outputdir = Path(str(f'{fullpath}.svx')).parent
- sp = subprocess.run([settings.CAVERN, "--log", f'--output={outputdir}', f'{fullpath}.svx'])
+ sp = subprocess.run([settings.CAVERN, "--log", f'--output={outputdir}', f'{fullpath}.svx'],
+ capture_output=True, check=False, text=True)
if sp.returncode != 0:
- print(f'fullpath: {fullpath}:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode))
+ message = f' ! Error running {settings.CAVERN}: {fullpath}'
+ DataIssue.objects.create(parser='entrances', message=message)
+ print(message)
+ print(f'stderr:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode))
self.caverncount += 1
# should also collect all the .err files too and create a DataIssue for each one which
# - is nonzero in size
# - has Error greater than 5% anywhere, or some other more serious error
- svxpath = fullpath + ".svx"
- logpath = fullpath + ".log"
+ svxpath = Path(fullpath + ".svx")
+ logpath = Path(fullpath + ".log")
outputdir = Path(svxpath).parent
- if not os.path.isfile(logpath):
+ if not logpath.is_file(): # always run if logfile not there
runcavern()
return
+ self.caverndate = now - 2*365*24*3600
+
if not self.caverndate:
- completed_process = subprocess.run(["which", "{}".format(settings.CAVERN)],
- capture_output=True, check=True, text=True)
- self.caverndate = os.path.getmtime(completed_process.stdout.strip())
+ sp = subprocess.run(["which", "{}".format(settings.CAVERN)],
+ capture_output=True, check=False, text=True)
+ if sp.returncode != 0:
+ message = f' ! Error running "which" on {settings.CAVERN}'
+ DataIssue.objects.create(parser='entrances', message=message)
+ print(message)
+ print(f'stderr:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode))
+ self.caverndate = os.path.getmtime(sp.stdout.strip())
+ else:
+ self.caverndate = now - 2*365*24*3600
cav_t = self.caverndate
log_t = os.path.getmtime(logpath)
svx_t = os.path.getmtime(svxpath)
@@ -1244,7 +1259,7 @@ class LoadingSurvex():
if cav_t - log_t > 0: # new version of cavern
runcavern()
return
- if chaosmonkey(400): # one in every 400 runs
+ if chaosmonkey(350): # one in every 350 runs
runcavern()
def FindAndLoadSurvex(survexblockroot):
@@ -1332,6 +1347,7 @@ def FindAndLoadSurvex(survexblockroot):
svxlines = fcollate.read().splitlines()
#pr2 = cProfile.Profile()
#pr2.enable()
+ print(" ", file=sys.stderr,end='')
#----------------------------------------------------------------
svx_load.LinearLoad(survexblockroot,survexfileroot.path, svxlines)
#----------------------------------------------------------------
@@ -1385,6 +1401,7 @@ def LoadSurvexBlocks():
DataIssue.objects.filter(parser='survex').delete()
DataIssue.objects.filter(parser='survexleg').delete()
DataIssue.objects.filter(parser='survexunits').delete()
+ DataIssue.objects.filter(parser='entrances').delete()
survexfileroot = MakeSurvexFileRoot()
# this next makes a block_object assciated with a file_object.path = SURVEX_TOPNAME
@@ -1421,21 +1438,48 @@ def LoadPositions():
# print(" - Regenerating stale cavern .log and .3d for '{}'\n days old: {:.1f} {:.1f} {:.1f}".
# format(topdata, (svx_t - d3d_t)/(24*3600), (cav_t - d3d_t)/(24*3600), (now - d3d_t)/(24*3600)))
- # subprocess.call([settings.CAVERN, "--log", f"--output={topdata}", f"{topdata}.svx"])
+
+ file3d = Path(f'{topdata}.3d')
try:
sp = subprocess.run([settings.CAVERN, "--log", f"--output={outputdir}", f"{topdata}.svx"],
- capture_output=True, check=True, text=True)
+ capture_output=True, check=False, text=True) #check=False means exception not raised
if sp.returncode != 0:
- print(f'topdata: {topdata}:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode))
-
- # print(" - Regenerating {} {}.3d in {}".format(settings.SURVEXPORT, topdata, settings.SURVEX_DATA))
- sp = subprocess.run([settings.SURVEXPORT, '--pos', f'{topdata}.3d'], cwd = settings.SURVEX_DATA,
- capture_output=True, check=True, text=True)
- if sp.returncode != 0:
- print(f'topdata: {topdata}:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode))
- except:
- print(f'topdata: {topdata}:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode))
+ message = f' ! Error: cavern: creating {file3d} in runcavern3()'
+ DataIssue.objects.create(parser='survex', message=message)
+ print(message)
+
+ # find the errors in the 1623.log file
+ sp = subprocess.run(["grep", "error:", f"{topdata}.log"],
+ capture_output=True, check=False, text=True) #check=False means exception not raised
+ message = f' ! Error: cavern: {sp.stdout}'
+ DataIssue.objects.create(parser='survex', message=message)
+ print(message)
+ except:
+ message = " ! CalledProcessError 'cavern' in runcavern3() at {topdata}."
+ DataIssue.objects.create(parser='survex', message=message)
+ print(message)
+
+ if file3d.is_file():
+ message = " ! CalledProcessError. File permissions {file3d.stat().st_mode} on {str(file3d)}"
+ DataIssue.objects.create(parser='survex', message=message)
+ print(message)
+
+ if file3d.is_file(): # might be an old one though
+ try:
+ # print(" - Regenerating {} {}.3d in {}".format(settings.SURVEXPORT, topdata, settings.SURVEX_DATA))
+ sp = subprocess.run([settings.SURVEXPORT, '--pos', f'{file3d}'], cwd = settings.SURVEX_DATA,
+ capture_output=True, check=False, text=True)
+ if sp.returncode != 0:
+ print(f' ! Error: survexport creating {topdata}.pos in runcavern3().\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode))
+ except:
+ message = " ! CalledProcessError 'survexport' in runcavern3() at {file3d}."
+ DataIssue.objects.create(parser='entrances', message=message)
+ print(message)
+ else:
+ message = f" ! Failed to find {file3d} so aborting generation of new .pos, using old one if present"
+ DataIssue.objects.create(parser='entrances', message=message)
+ print(message)
topdata = os.fspath(Path(settings.SURVEX_DATA) / settings.SURVEX_TOPNAME)
print(' - Generating a list of Pos from %s.svx and then loading...' % (topdata))
@@ -1476,7 +1520,13 @@ def LoadPositions():
svxid, number, point_type, label = pt
mappoints[svxid]=True
- posfile = open("%s.pos" % (topdata))
+ if not Path(pospath).is_file():
+ message = f" ! Failed to find {pospath} so aborting generation of entrance locations. "
+ DataIssue.objects.create(parser='entrances', message=message)
+ print(message)
+ return
+
+ posfile = open(pospath)
posfile.readline() #Drop header
try:
survexblockroot = SurvexBlock.objects.get(name=ROOTBLOCK)