From 251e3bf84410018afd87a5f7da552c363fa3e465 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@klebos.com>
Date: Tue, 15 Mar 2022 20:53:55 +0000
Subject: [PATCH] Bug fixes
---
core/views/uploads.py | 22 ++++++++++++++++++----
parsers/QMs.py | 19 +++++++++++++++----
parsers/scans.py | 1 +
3 files changed, 34 insertions(+), 8 deletions(-)
diff --git a/core/views/uploads.py b/core/views/uploads.py
index aa10c7f..038f2d0 100644
--- a/core/views/uploads.py
+++ b/core/views/uploads.py
@@ -1,6 +1,8 @@
import re, os
import subprocess
import json
+import settings
+
from pathlib import Path
from django import forms
@@ -18,6 +20,7 @@ from troggle.parsers.imports import import_caves, import_people, import_surveysc
from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
+from troggle.core.models.troggle import DataIssue
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
from troggle.core.models.survex import DrawingFile
@@ -77,7 +80,8 @@ def scanupload(request, wallet=None):
if int(wnumber) == 0:
prev = f'{int(wnumber):02d}'
-
+
+ wurl = f"/scanupload/{wallet}"
wallet = wallet.replace(':','#')
dirpath = Path(settings.SURVEY_SCANS, year, wallet)
contents_path = dirpath / contentsjson
@@ -87,7 +91,7 @@ def scanupload(request, wallet=None):
if request.method == 'POST':
form = FilesForm(request.POST,request.FILES)
- print(f'! - FilesForm POSTED')
+ #print(f'! - FilesForm POSTED')
if form.is_valid():
f = request.FILES["uploadfiles"]
@@ -144,10 +148,13 @@ def scanupload(request, wallet=None):
except:
message = f"! Failed to load {contents_path} JSON file"
print(message)
- DataIssue.objects.create(parser='Scans', message=message)
+ DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
raise
if not waldata["people"]:
waldata["people"]=["NOBODY"]
+
+ cave =""
+ psg = ""
if waldata["cave"]:
cave = waldata["cave"]
if waldata["name"]:
@@ -155,6 +162,13 @@ def scanupload(request, wallet=None):
if waldata["survex file"]:
if not isinstance(waldata["survex file"], list):
waldata["survex file"] = [waldata["survex file"]]
+
+ for svx in waldata["survex file"]:
+ print(f'{svx}')
+ if not (Path(settings.SURVEX_DATA) / svx).is_file():
+ message = f"! Incorrect survex file in wallet data: {svx} not found in LOSER repo"
+ print(message)
+ DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty,
@@ -265,7 +279,7 @@ def dwgupload(request, folder=None, gitdisable='no'):
'''
if name in [ '.gitignore', ]:
return False
- if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt', '.jpg', '.jpeg', '.png', '.pdf']:
+ if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt', '.jpg', '.jpeg', '.png', '.pdf', '.topo']:
return True # dangerous, we should check the actual file binary signature
return False
diff --git a/parsers/QMs.py b/parsers/QMs.py
index be180b2..c97fb51 100644
--- a/parsers/QMs.py
+++ b/parsers/QMs.py
@@ -33,19 +33,25 @@ def parseCaveQMs(cave,inputFile):
try:
steinBr=Cave.objects.get(official_name="Steinbrückenhöhle")
except Cave.DoesNotExist:
- print("Steinbruckenhoehle is not in the database. Please run parsers.")
+ message = f' ! - {qmPath} Steinbruckenhoehle is not in the database. Please run cave parser'
+ print(message)
+ DataIssue.objects.create(parser='QMs', message=message)
return
elif cave=='234-Hauch':
try:
hauchHl=Cave.objects.get(official_name="Hauchhöhle")
except Cave.DoesNotExist:
- print("Hauchhoehle is not in the database. Please run parsers.")
+ message = f' ! - {qmPath} Hauchhoehle is not in the database. Please run cave parser'
+ print(message)
+ DataIssue.objects.create(parser='QMs', message=message)
return
elif cave =='161-KH':
try:
kh=Cave.objects.get(official_name="Kaninchenhöhle")
except Cave.DoesNotExist:
- print("KH is not in the database. Please run parsers.")
+ message = f' ! - {qmPath} KH is not in the database. Please run cave parser'
+ print(message)
+ DataIssue.objects.create(parser='QMs', message=message)
parse_KH_QMs(kh, inputFile=inputFile)
return
@@ -103,9 +109,14 @@ def parseCaveQMs(cave,inputFile):
newQM.save()
except KeyError: #check on this one
+ message = f' ! - {qmPath} KeyError {str(line)} '
+ print(message)
+ DataIssue.objects.create(parser='QMs', message=message)
continue
except IndexError:
- print("Index error in " + str(line))
+ message = f' ! - {qmPath} IndexError {str(line)} '
+ print(message)
+ DataIssue.objects.create(parser='QMs', message=message)
continue
def parse_KH_QMs(kh, inputFile):
diff --git a/parsers/scans.py b/parsers/scans.py
index 351441c..6ac37fa 100644
--- a/parsers/scans.py
+++ b/parsers/scans.py
@@ -105,6 +105,7 @@ def load_all_scans():
SingleScan.objects.all().delete()
Wallet.objects.all().delete()
print(' - deleting all scansFolder and scansSingle objects')
+ DataIssue.objects.filter(parser='scans').delete()
# first do the smkhs (large kh survey scans) directory
manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs")