From 5e478c7eb0a2bcd23ca46d1c4bd200750fbc9231 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@klebos.com>
Date: Tue, 27 Apr 2021 20:44:24 +0100
Subject: [PATCH] Imports in control panel work again
---
core/views/other.py | 63 +++++++++++++++++++++++++--------
parsers/imports.py | 5 +--
parsers/people.py | 1 +
parsers/surveys.py | 21 ++++++-----
templates/controlPanel.html | 70 ++++++++++++++++++++++---------------
5 files changed, 106 insertions(+), 54 deletions(-)
diff --git a/core/views/other.py b/core/views/other.py
index 8a4dd37..552bf6f 100644
--- a/core/views/other.py
+++ b/core/views/other.py
@@ -9,7 +9,9 @@ from django.shortcuts import render
from django.template import Context, loader
from django.core.files.storage import FileSystemStorage, default_storage
-import troggle.parsers.imports
+from troggle.parsers.imports import import_caves, import_people, import_surveyscans
+from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
+# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
from .login import login_required_if_public
@@ -82,27 +84,60 @@ def frontpage(request):
return render(request,'frontpage.html', locals())
+@login_required_if_public
def controlpanel(request):
+ '''This should be re-written to use ajax so that the user can see the progress
+ of the actions.
+
+ Also need to add reinit_db option
+ '''
jobs_completed=[]
- if request.method=='POST':
- if request.user.is_superuser: # expoadmin is both .is_staff and ._is_superuser
- # NONE of this works now that databaseReset (now parsers.imports) has been so extensively rewritten
- reinit_db()
+
+ def process_imports():
+ '''databaseReset.py
+ jq.enq("reinit",reinit_db)
+ jq.enq("caves",import_caves)
+ jq.enq("people",import_people)
+ jq.enq("scans",import_surveyscans)
+ jq.enq("logbooks",import_logbooks)
+ jq.enq("QMs",import_QMs)
+ jq.enq("drawings",import_drawingsfiles)
+ jq.enq("survex",import_survex)
+ '''
+ if request.POST.get("import_caves", False):
import_caves()
+ jobs_completed.append('Caves')
+ if request.POST.get("import_people", False):
import_people()
+ jobs_completed.append('People')
+ if request.POST.get("import_surveyscans", False):
import_surveyscans()
+ jobs_completed.append('Scans')
+ if request.POST.get("import_logbooks", False):
import_logbooks()
+ jobs_completed.append('Logbooks')
+ if request.POST.get("import_QMs", False):
import_QMs()
- import_dwgfiles()
- import_survexblks()
- import_survexpos()
- else:
- if request.user.is_authenticated: #The user is logged in, but is not a superuser.
- return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'error':'You must be a superuser to use that feature.'})
- else:
- return HttpResponseRedirect(reverse('auth_login'))
+ jobs_completed.append('QMs')
+ if request.POST.get("import_drawingsfiles", False):
+ import_drawingsfiles()
+ jobs_completed.append('Drawings')
+ if request.POST.get("import_survex", False):
+ import_survex()
+ jobs_completed.append('Survex')
+
+ print("", flush=True)
+
+ if not request.user.is_superuser: # expoadmin is both .is_staff and ._is_superuser
+ return render(request,'controlPanel.html', {'error': 'You are logged in, but not logged in as "expoadmin". \nLogout and login again to contnue.'})
+ else:
+ if request.method=='POST':
+ #reinit_db()
+ process_imports()
+ return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
+ else:
+ return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
- return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
def downloadlogbook(request,year=None,extension=None,queryset=None):
diff --git a/parsers/imports.py b/parsers/imports.py
index 01c1d7b..6eca632 100644
--- a/parsers/imports.py
+++ b/parsers/imports.py
@@ -15,7 +15,8 @@ import troggle.parsers.surveys
import troggle.parsers.logbooks
import troggle.parsers.QMs
-'''Master data importUsed only by databaseReset.py currently
+'''Master data import.
+Used only by databaseReset.py and online controlpanel.
'''
def import_caves():
@@ -46,9 +47,9 @@ def import_QMs():
def import_survex():
# when this import is moved to the top with the rest it all crashes horribly
+ print("-- Importing Survex and Entrance Positions")
with transaction.atomic():
import troggle.parsers.survex
- print("-- Importing Survex and Entrance Positions")
print(" - Survex Blocks")
with transaction.atomic():
troggle.parsers.survex.LoadSurvexBlocks()
diff --git a/parsers/people.py b/parsers/people.py
index 3f7c02a..3b5efef 100644
--- a/parsers/people.py
+++ b/parsers/people.py
@@ -124,6 +124,7 @@ def load_people_expos():
lookupAttribs = {'person':person, 'expedition':expedition}
nonLookupAttribs = {'nickname':nickname, 'is_guest':(personline[header["Guest"]] == "1")}
save_carefully(PersonExpedition, lookupAttribs, nonLookupAttribs)
+ print("", flush=True)
# used in other referencing parser functions
diff --git a/parsers/surveys.py b/parsers/surveys.py
index f9ca56a..5b7bc1e 100644
--- a/parsers/surveys.py
+++ b/parsers/surveys.py
@@ -14,8 +14,10 @@ from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
from troggle.core.models.troggle import DataIssue
from troggle.core.utils import save_carefully
-'''Scans through all the :drawings: repository looking
+'''Searches through all the :drawings: repository looking
for tunnel and therion files
+
+Searches through all the survey scans directories in expofiles, looking for images to be referenced.
'''
@@ -41,7 +43,7 @@ def listdir(*directories):
return [folder.rstrip(r"/") for folder in folders]
-# handles url or file, so we can refer to a set of scans on another server
+# handles url or file, so we can refer to a set of scans (not drawings) on another server
def GetListDir(sdir):
res = [ ]
if sdir[:7] == "http://":
@@ -80,7 +82,7 @@ def LoadListScansFile(wallet):
# this iterates through the scans directories (either here or on the remote server)
# and builds up the models we can access later
-def LoadListScans():
+def load_all_scans():
print(' - Loading Survey Scans')
@@ -116,17 +118,18 @@ def LoadListScans():
wallet = Wallet(fpath=ff, walletname=f)
wallet.save()
LoadListScansFile(wallet)
-
-def find_tunnel_scan(dwgfile, path):
+ print("", flush=True)
+
+def find_tunnel_file(dwgfile, path):
'''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file
which we have already seen when we imported all the files we could find in the surveyscans direstories
'''
wallet, scansfile = None, None
- mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg))$", path)
+ mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg|gif))$", path)
if mscansdir:
scanswalletl = Wallet.objects.filter(walletname=mscansdir.group(1))
- # This should properly detect if a list of folders is returned and do something sensible, not just pick the first.
+ # This should be chnaged to properly detect if a list of folders is returned and do something sensible, not just pick the first.
if len(scanswalletl):
wallet = scanswalletl[0]
if len(scanswalletl) > 1:
@@ -148,7 +151,7 @@ def find_tunnel_scan(dwgfile, path):
if scansfile:
dwgfile.scans.add(scansfile)
- elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif|pdf)$(?i)", path):
+ elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif)$(?i)", path):
name = os.path.split(path)[1]
rdwgfilel = DrawingFile.objects.filter(dwgname=name)
if len(rdwgfilel):
@@ -248,7 +251,7 @@ def setdwgfileinfo(dwgfile):
# <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0">
for path, style in rx_pcpath.findall(ttext):
- find_tunnel_scan(dwgfile, path.decode())
+ find_tunnel_file(dwgfile, path.decode())
# should also scan and look for survex blocks that might have been included, and image scans
# which would populate dwgfile.survexfile
diff --git a/templates/controlPanel.html b/templates/controlPanel.html
index 0668055..bab7bf9 100644
--- a/templates/controlPanel.html
+++ b/templates/controlPanel.html
@@ -3,28 +3,27 @@
{% if jobs_completed %}
<div class="noticeBox">
- Just finished running:
+ Just finished running:
<ul>
{% for job in jobs_completed %}
<li>{{ job }}</li>
{% endfor %}
</ul>
- {% if settings.LOGFILE %}See the logfile at {{settings.LOGFILE}} for more information.{% endif %}
- <a href="#" class="closeDiv">dismiss this message</a>
+ See <a href="/dataissues">Data Issues</a> report
+ <br>
</div>
{% endif %}
{% if error %}
<div class="noticeBox">
{{ error }}
- <a href="#" class="closeDiv">dismiss this message</a>
</div>
{% endif %}
-<h2>NONE of this is working currently</h2>
-<h2>The code behind this page is under repair</h2>
-
+<div style="column-count: 2;">
+{%comment%}
<form name="reset" method="post" action="">
+{% csrf_token %}
<h3>Wipe:</h3>
<table>
<tr>
@@ -36,55 +35,68 @@
</tr>
</table>
</form>
-<h3>Import (non-destructive):</h3>
+{% endcomment %}
+
+<h3>Import (on top of existing data):</h3>
+<p>To get a fully-functioning system, all these imports must be done and in this order.
<form name="import" method="post" action="">
+{% csrf_token %}
<table>
<tr>
- <td>people from folk.csv using parsers\people.py</td>
+ <th colspan="2">Import from master files</th>
+ </tr>
+ <tr>
+ <td>all caves </td>
+ <td><input type="checkbox" name="import_caves"/></td>
+ </tr>
+ <tr>
+ <td>all people </td>
<td><input type="checkbox" name="import_people"/></td>
</tr>
<tr>
- <td>logbook entries using parsers\logbooks.py</td>
+ <td>all surveys scans </td>
+ <td><input type="checkbox" name="import_surveyscans"/></td>
+ </tr>
+ <tr>
+ <td>all logbooks</td>
<td><input type="checkbox" name="import_logbooks"/></td>
</tr>
<tr>
- <td>QMs using parsers\QMs.py</td>
+ <td>QMs (from old csv files)</td>
<td><input type="checkbox" name="import_QMs" /></td>
</tr>
<tr>
- <td>survey scans using parsers\surveys.py</td>
- <td><input type="checkbox" name="import_surveys" /></td>
+ <td>all drawings files</td>
+ <td><input type="checkbox" name="import_drawingsfiles" /></td>
</tr>
<tr>
- <td>survex data using parsers\survex.py</td>
+ <td>all survex data (10 minutes)</td>
<td><input type="checkbox" name="import_survex" /></td>
</tr>
</table>
<p>
<input type="submit" id="Import" value="Import">
- <input type="submit" name="check_all_import" id="check_all_import" value="Check all" disabled >
</p>
</form>
-<h3>Export to legacy formats:</h3>
+<p>
+
+
+<h3>Export to a different format:</h3>
+<p>This creates 'newlogbook.html' in the years/<year>/ folder
<table>
-
-<th>
- <td>Export onto server</td>
- <td>Export and Download</td>
-</th>
-
<tr>
- <td>
- surveys to Surveys.csv
- </td>
- <td>
- </td>
+
+ <th>Export and Download</th>
+
+</tr>
+<tr>
<td>
<form name="export" method="get" action={% url "downloadlogbook" %}>
+{% csrf_token %}
<p>Download a logbook file which is dynamically generated by Troggle.</p>
<p>
@@ -99,8 +111,8 @@
<p>
Output style:
<select name="extension">
- <option value="txt">.txt file with MediaWiki markup - 2008 style</option>
<option value="html">.html file - 2005 style</option>
+ <option value="txt">.txt file with MediaWiki markup</option>
</select>
</p>
<p>
@@ -112,5 +124,5 @@
</table>
</form>
-
+</div>
{% endblock %}
\ No newline at end of file