Bug fixes

core/views/uploads.py

@@ -1,6 +1,8 @@
 import re, os
 import subprocess
 import json
+import settings
+
 from pathlib import Path
 
 from django import forms
@@ -18,6 +20,7 @@ from troggle.parsers.imports import import_caves, import_people, import_surveysc
 from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
 from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml
 # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
+from troggle.core.models.troggle import DataIssue
 from troggle.core.models.troggle import Expedition, Person, PersonExpedition
 from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
 from troggle.core.models.survex import DrawingFile
@@ -77,7 +80,8 @@ def scanupload(request, wallet=None):
     
     if int(wnumber) == 0:
         prev = f'{int(wnumber):02d}'
-            
+
+    wurl = f"/scanupload/{wallet}"     
     wallet = wallet.replace(':','#')
     dirpath = Path(settings.SURVEY_SCANS, year, wallet)
     contents_path = dirpath / contentsjson
@@ -87,7 +91,7 @@ def scanupload(request, wallet=None):
    
     if request.method == 'POST':
         form = FilesForm(request.POST,request.FILES)
-        print(f'! - FilesForm POSTED')
+        #print(f'! - FilesForm POSTED')
  
         if form.is_valid():
             f = request.FILES["uploadfiles"]
@@ -144,10 +148,13 @@ def scanupload(request, wallet=None):
             except:
                 message = f"! Failed to load {contents_path} JSON file"
                 print(message)
-                DataIssue.objects.create(parser='Scans', message=message)
+                DataIssue.objects.create(parser='scans', message=message, url=wurl)  # set URL to this wallet folder
                 raise
     if not waldata["people"]:
         waldata["people"]=["NOBODY"]
+        
+    cave =""
+    psg = ""
     if waldata["cave"]:
         cave =  waldata["cave"]
     if waldata["name"]:
@@ -155,6 +162,13 @@ def scanupload(request, wallet=None):
     if waldata["survex file"]:
         if not isinstance(waldata["survex file"], list):
             waldata["survex file"] = [waldata["survex file"]]
+             
+    for svx in waldata["survex file"]:
+        print(f'{svx}')
+        if not (Path(settings.SURVEX_DATA) / svx).is_file():
+                message = f"! Incorrect survex file in wallet data: {svx} not found in LOSER repo"
+                print(message)
+                DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
 
     
     context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty,
@@ -265,7 +279,7 @@ def dwgupload(request, folder=None, gitdisable='no'):
         '''
         if name in [ '.gitignore', ]:
             return False
-        if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt', '.jpg', '.jpeg', '.png', '.pdf']:
+        if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.txt', '.jpg', '.jpeg', '.png', '.pdf', '.topo']:
             return True # dangerous, we should check the actual file binary signature
         return False
 

parsers/QMs.py

@@ -33,19 +33,25 @@ def parseCaveQMs(cave,inputFile):
         try:
             steinBr=Cave.objects.get(official_name="Steinbrückenhöhle")
         except Cave.DoesNotExist:
-            print("Steinbruckenhoehle is not in the database. Please run parsers.")
+            message = f' ! - {qmPath} Steinbruckenhoehle is not in the database. Please run cave parser' 
+            print(message)
+            DataIssue.objects.create(parser='QMs', message=message)
             return
     elif cave=='234-Hauch':
         try:
             hauchHl=Cave.objects.get(official_name="Hauchhöhle")
         except Cave.DoesNotExist:
-            print("Hauchhoehle is not in the database. Please run parsers.")
+            message = f' ! - {qmPath} Hauchhoehle is not in the database. Please run cave parser' 
+            print(message)
+            DataIssue.objects.create(parser='QMs', message=message)
             return
     elif cave =='161-KH':
         try:
             kh=Cave.objects.get(official_name="Kaninchenhöhle")
         except Cave.DoesNotExist:
-            print("KH is not in the database. Please run parsers.")
+            message = f' ! - {qmPath} KH is not in the database. Please run cave parser' 
+            print(message)
+            DataIssue.objects.create(parser='QMs', message=message)
         parse_KH_QMs(kh, inputFile=inputFile) 
         return
 
@@ -103,9 +109,14 @@ def parseCaveQMs(cave,inputFile):
                 newQM.save() 
                 
         except KeyError: #check on this one
+            message = f' ! - {qmPath} KeyError {str(line)} ' 
+            print(message)
+            DataIssue.objects.create(parser='QMs', message=message)
             continue
         except IndexError:
-            print("Index error in " + str(line))
+            message = f' ! - {qmPath} IndexError {str(line)} ' 
+            print(message)
+            DataIssue.objects.create(parser='QMs', message=message)
             continue
 
 def parse_KH_QMs(kh, inputFile):

parsers/scans.py

@@ -105,6 +105,7 @@ def load_all_scans():
     SingleScan.objects.all().delete()
     Wallet.objects.all().delete()
     print(' - deleting all scansFolder and scansSingle objects')
+    DataIssue.objects.filter(parser='scans').delete()
 
     # first do the smkhs (large kh survey scans) directory
     manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs")