From 931c33cfdb498deaca00b4293f4290e5edb6a8a6 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@klebos.com>
Date: Thu, 21 Jul 2022 19:01:04 +0300
Subject: [PATCH] Clean DataIssues output, new therionrefs.log file
---
.gitignore | 1 +
parsers/caves.py | 36 ++++++++++++++++++++++++------------
parsers/drawings.py | 13 ++++++++++---
3 files changed, 35 insertions(+), 15 deletions(-)
diff --git a/.gitignore b/.gitignore
index 4228331..e304964 100644
--- a/.gitignore
+++ b/.gitignore
@@ -127,3 +127,4 @@ media/jslib/openlayers/Lang/zh-CN.js
media/jslib/openlayers/Lang/zh-TW.js
_test_response.html
_deploy/wsl/localsettingsWSL.py.bak
+therionrefs.log
diff --git a/parsers/caves.py b/parsers/caves.py
index 96497c4..91f353a 100644
--- a/parsers/caves.py
+++ b/parsers/caves.py
@@ -26,6 +26,8 @@ todo='''- db Update does not work when a cave id is in the pending list but a pr
So we will need a separate file-editing capability just for this configuration file ?!
- crashes on MariaDB on server when deleting Caves and complains Area needs a non null parent, But this is not true.
+ The only solution we have found is to let it crash, then stop and restart MariaDB (requires a logon able to sudo)
+ and then restart the databasereset.py again. (status as of July 2022)
'''
entrances_xslug = {}
caves_xslug = {}
@@ -47,7 +49,7 @@ def dummy_entrance(k, slug, msg="DUMMY"):
slug = slug, primary = False)
except:
message = f" ! {k:11s} {msg} entrance create failure"
- DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}')
+ DataIssue.objects.create(parser='caves', message=message, url=f'{slug}')
print(message)
ent.cached_primary_slug = slug
@@ -56,7 +58,7 @@ def dummy_entrance(k, slug, msg="DUMMY"):
return ent
else:
message = f" ! {k:11s} {msg} cave SLUG '{slug}' create failure"
- DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}')
+ DataIssue.objects.create(parser='caves', message=message, url=f'{slug}')
print(message)
raise
@@ -69,12 +71,13 @@ def set_dummy_entrance(id, slug, cave, msg="DUMMY"):
letter = ""
entrances_xslug[slug] = entrance
ce = CaveAndEntrance.objects.update_or_create(cave = cave, entrance_letter = "", entrance = entrance)
- message = f' ! Warning: Dummy Entrance created for {id}, slug:"{slug}" '
- DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}')
+ message = f' ! Warning: Dummy Entrance created for {id}'
+
+ DataIssue.objects.create(parser='caves', message=message, url=f'{cave.url}')
print(message)
except:
message = f' ! Entrance Dummy setting failure, slug:"{slug}" cave id :"{id}" '
- DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}')
+ DataIssue.objects.create(parser='caves', message=message, url=f'{cave.url}')
print(message)
def do_pending_cave(k, url, area_1623):
@@ -82,19 +85,28 @@ def do_pending_cave(k, url, area_1623):
default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists
in expoweb/cave_data/1623-"k".html
'''
+ slug = "1623-" + k
+
default_note = f"_Survex file found in loser repo but no description in expoweb <br><br><br>\n"
default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then "
default_note += f"<br><br>\n\n - (1) search in the survex file for the *ref to find a "
default_note += f"relevant wallet, e.g.<a href='/survey_scans/2009%252311/'>2009#11</a> and read the notes image files <br>\n - "
default_note += f"<br><br>\n\n - (2) search in the Expo for that year e.g. <a href='/expedition/{k[0:4]}'>{k[0:4]}</a> to find a "
- default_note += f"relevant logbook entry, then <br>\n - "
+ default_note += f"relevant logbook entry, remember that the date may have been recorded incorrectly, "
+ default_note += f"so check for trips i.e. logbook entries involving the same people as were listed in the survex file, "
+ default_note += f"and you should also check the scanned copy of the logbook (linked from each logbook entry page) "
+ default_note += f"just in case a vital trip was not transcribed, then <br>\n - "
default_note += f"click on 'Edit this cave' and copy the information you find in the survex file and the logbook"
+ default_note += f"and delete all the text in the 'Notes' section - which is the text you are reading now."
+ default_note += f"<br><br>\n\n - Only two fields on this form are essential. "
+ default_note += f"Documentation of all the fields on 'Edit this cave' form is in <a href='/handbook/survey/caveentryfields.html'>handbook/survey/caveentryfields</a>"
default_note += f"<br><br>\n\n - "
- default_note += f"When you Submit it will create a file file in expoweb/cave_data/ "
+ default_note += f"When you Submit it will create a new file in expoweb/cave_data/ "
default_note += f"<br><br>\n\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. "
default_note += f"and then Submit to save it (if you forget to do this, a dummy entrance will be created for your new cave description)."
+ default_note += f"<br><br>\n\n - Finally, you need to find a nerd to edit the file '<var>expoweb/cave_data/pending.txt</var>' "
+ default_note += f"to remove the line <br><var>{slug}</var><br> as it is no longer 'pending' but 'done. Well Done."
- slug = "1623-" + k
cave = Cave(
unofficial_number = k,
@@ -424,7 +436,7 @@ def readcave(filename):
ce = CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance)
except:
message = f' ! Entrance setting failure, slug:"{slug}" #entrances:{len(entrances)} {entrance} letter:"{letter}" cave:"{c}" filename:"cave_data/{filename}"'
- DataIssue.objects.create(parser='caves', message=message)
+ DataIssue.objects.create(parser='caves', message=message, url=f'{c.url}_edit/')
print(message)
if survex_file[0]:
@@ -436,18 +448,18 @@ def readcave(filename):
if description_file[0]: # if not an empty string
message = f' - {slug:12} complex description filename "{description_file[0]}" inside "{CAVEDESCRIPTIONS}/{filename}"'
- DataIssue.objects.create(parser='caves ok', message=message, url=f'{slug}/edit/')
+ DataIssue.objects.create(parser='caves ok', message=message, url=f'/{slug}_cave_edit/')
print(message)
if not (Path(EXPOWEB) / description_file[0]).is_file():
message = f' ! {slug:12} description filename "{EXPOWEB}/{description_file[0]}" does not refer to a real file'
- DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/')
+ DataIssue.objects.create(parser='caves', message=message, url=f'/{slug}_cave_edit/')
print(message)
#c.description_file="" # done only once, to clear out cruft.
#c.save()
else: # more than one item in long list
message = f' ! ABORT loading this cave. in "{filename}"'
- DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slugs}/edit/')
+ DataIssue.objects.create(parser='caves', message=message, url=f'/{slug}_cave_edit/')
print(message)
def getXML(text, itemname, minItems = 1, maxItems = None, printwarnings = True, context = ""):
diff --git a/parsers/drawings.py b/parsers/drawings.py
index 2f4e4a9..a6c16be 100644
--- a/parsers/drawings.py
+++ b/parsers/drawings.py
@@ -128,15 +128,19 @@ def settherionfileinfo(filetuple):
# scrap blownout -projection plan -scale [-81.0 -42.0 216.0 -42.0 0.0 0.0 7.5438 0.0 m]
for xth_me in rx_xth_me.findall(ttext):
+ # WORK IN PROGRESS. Do not clutter up the DataIssues list with this
message = f'! Un-parsed image filename: {therionfile.dwgname} : {xth_me.split()[-3]} - {therionfile.dwgpath}'
- #print(message)
- DataIssue.objects.create(parser='xTherion', message=message, url=f'/dwgdataraw/{therionfile.dwgpath}')
+ # print(message)
+ # DataIssue.objects.create(parser='xTherion', message=message, url=f'/dwgdataraw/{therionfile.dwgpath}')
+ with open('therionrefs.log', 'a') as lg:
+ lg.write(message + '\n')
+
findimageinsert(therionfile, xth_me)
for inp in rx_input.findall(ttext):
# if this 'input' is a .th2 file we have already seen, then we can assign this as a sub-file
# but we would need to disentangle to get the current path properly
- message = f'! Un-set Therion .th2 input: - {therionfile.dwgname} : {inp} - {therionfile.dwgpath}'
+ message = f'! Un-set (?) Therion .th2 input: - {therionfile.dwgname} : {inp} - {therionfile.dwgpath}'
#print(message)
DataIssue.objects.create(parser='xTherion', message=message, url=f'/dwgdataraw/{therionfile.dwgpath}')
findimportinsert(therionfile, inp)
@@ -207,6 +211,9 @@ def load_drawings_files():
DataIssue.objects.filter(parser='Therion').delete()
DataIssue.objects.filter(parser='xTherion').delete()
DataIssue.objects.filter(parser='Tunnel').delete()
+ if(os.path.isfile('therionrefs.log')):
+ os.remove('therionrefs.log')
+
drawingsdirs = [ "" ]
while drawingsdirs: