xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Imports in control panel work again

Browse Source
This commit is contained in:
Philip Sargent 2021-04-27 20:44:24 +01:00
parent 821aaa1f66
commit 5e478c7eb0
5 changed files with 106 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
core/views/other.py
View File

@ -9,7 +9,9 @@ from django.shortcuts import render
from django.template import Context, loader
from django.core.files.storage import FileSystemStorage, default_storage
import troggle.parsers.imports
from troggle.parsers.imports import import_caves, import_people, import_surveyscans
from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
from .login import login_required_if_public
@ -82,27 +84,60 @@ def frontpage(request):
return render(request,'frontpage.html', locals())
@login_required_if_public
def controlpanel(request):
'''This should be re-written to use ajax so that the user can see the progress
of the actions.
Also need to add reinit_db option
'''
jobs_completed=[]
if request.method=='POST':
if request.user.is_superuser: # expoadmin is both .is_staff and ._is_superuser
# NONE of this works now that databaseReset (now parsers.imports) has been so extensively rewritten
reinit_db()
def process_imports():
'''databaseReset.py
jq.enq("reinit",reinit_db)
jq.enq("caves",import_caves)
jq.enq("people",import_people)
jq.enq("scans",import_surveyscans)
jq.enq("logbooks",import_logbooks)
jq.enq("QMs",import_QMs)
jq.enq("drawings",import_drawingsfiles)
jq.enq("survex",import_survex)
'''
if request.POST.get("import_caves", False):
import_caves()
jobs_completed.append('Caves')
if request.POST.get("import_people", False):
import_people()
jobs_completed.append('People')
if request.POST.get("import_surveyscans", False):
import_surveyscans()
jobs_completed.append('Scans')
if request.POST.get("import_logbooks", False):
import_logbooks()
jobs_completed.append('Logbooks')
if request.POST.get("import_QMs", False):
import_QMs()
import_dwgfiles()
import_survexblks()
import_survexpos()
else:
if request.user.is_authenticated: #The user is logged in, but is not a superuser.
return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'error':'You must be a superuser to use that feature.'})
else:
return HttpResponseRedirect(reverse('auth_login'))
jobs_completed.append('QMs')
if request.POST.get("import_drawingsfiles", False):
import_drawingsfiles()
jobs_completed.append('Drawings')
if request.POST.get("import_survex", False):
import_survex()
jobs_completed.append('Survex')
print("", flush=True)
if not request.user.is_superuser: # expoadmin is both .is_staff and ._is_superuser
return render(request,'controlPanel.html', {'error': 'You are logged in, but not logged in as "expoadmin". \nLogout and login again to contnue.'})
else:
if request.method=='POST':
#reinit_db()
process_imports()
return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
else:
return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
def downloadlogbook(request,year=None,extension=None,queryset=None):

5
parsers/imports.py
View File

@ -15,7 +15,8 @@ import troggle.parsers.surveys
import troggle.parsers.logbooks
import troggle.parsers.QMs
'''Master data importUsed only by databaseReset.py currently
'''Master data import.
Used only by databaseReset.py and online controlpanel.
'''
def import_caves():
@ -46,9 +47,9 @@ def import_QMs():
def import_survex():
# when this import is moved to the top with the rest it all crashes horribly
print("-- Importing Survex and Entrance Positions")
with transaction.atomic():
import troggle.parsers.survex
print("-- Importing Survex and Entrance Positions")
print(" - Survex Blocks")
with transaction.atomic():
troggle.parsers.survex.LoadSurvexBlocks()

1
parsers/people.py
View File

@ -124,6 +124,7 @@ def load_people_expos():
lookupAttribs = {'person':person, 'expedition':expedition}
nonLookupAttribs = {'nickname':nickname, 'is_guest':(personline[header["Guest"]] == "1")}
save_carefully(PersonExpedition, lookupAttribs, nonLookupAttribs)
print("", flush=True)
# used in other referencing parser functions

21
parsers/surveys.py
View File

@ -14,8 +14,10 @@ from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
from troggle.core.models.troggle import DataIssue
from troggle.core.utils import save_carefully
'''Scans through all the :drawings: repository looking
'''Searches through all the :drawings: repository looking
for tunnel and therion files
Searches through all the survey scans directories in expofiles, looking for images to be referenced.
'''
@ -41,7 +43,7 @@ def listdir(*directories):
return [folder.rstrip(r"/") for folder in folders]
# handles url or file, so we can refer to a set of scans on another server
# handles url or file, so we can refer to a set of scans (not drawings) on another server
def GetListDir(sdir):
res = [ ]
if sdir[:7] == "http://":
@ -80,7 +82,7 @@ def LoadListScansFile(wallet):
# this iterates through the scans directories (either here or on the remote server)
# and builds up the models we can access later
def LoadListScans():
def load_all_scans():
print(' - Loading Survey Scans')
@ -116,17 +118,18 @@ def LoadListScans():
wallet = Wallet(fpath=ff, walletname=f)
wallet.save()
LoadListScansFile(wallet)
def find_tunnel_scan(dwgfile, path):
print("", flush=True)
def find_tunnel_file(dwgfile, path):
'''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file
which we have already seen when we imported all the files we could find in the surveyscans direstories
'''
wallet, scansfile = None, None
mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg))$", path)
mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg|gif))$", path)
if mscansdir:
scanswalletl = Wallet.objects.filter(walletname=mscansdir.group(1))
# This should properly detect if a list of folders is returned and do something sensible, not just pick the first.
# This should be chnaged to properly detect if a list of folders is returned and do something sensible, not just pick the first.
if len(scanswalletl):
wallet = scanswalletl[0]
if len(scanswalletl) > 1:
@ -148,7 +151,7 @@ def find_tunnel_scan(dwgfile, path):
if scansfile:
dwgfile.scans.add(scansfile)
elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif|pdf)$(?i)", path):
elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif)$(?i)", path):
name = os.path.split(path)[1]
rdwgfilel = DrawingFile.objects.filter(dwgname=name)
if len(rdwgfilel):
@ -248,7 +251,7 @@ def setdwgfileinfo(dwgfile):
# <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0">
for path, style in rx_pcpath.findall(ttext):
find_tunnel_scan(dwgfile, path.decode())
find_tunnel_file(dwgfile, path.decode())
# should also scan and look for survex blocks that might have been included, and image scans
# which would populate dwgfile.survexfile

70
templates/controlPanel.html
View File

@ -3,28 +3,27 @@
{% if jobs_completed %}
<div class="noticeBox">
Just finished running:
&nbsp;Just finished running:
<ul>
{% for job in jobs_completed %}
<li>{{ job }}</li>
{% endfor %}
</ul>
{% if settings.LOGFILE %}See the logfile at {{settings.LOGFILE}} for more information.{% endif %}
<a href="#" class="closeDiv">dismiss this message</a>
&nbsp;See <a href="/dataissues">Data Issues</a> report
<br>&nbsp;
</div>
{% endif %}
{% if error %}
<div class="noticeBox">
{{ error }}
<a href="#" class="closeDiv">dismiss this message</a>
</div>
{% endif %}
<h2>NONE of this is working currently</h2>
<h2>The code behind this page is under repair</h2>
<div style="column-count: 2;">
{%comment%}
<form name="reset" method="post" action="">
{% csrf_token %}
<h3>Wipe:</h3>
<table>
<tr>
@ -36,55 +35,68 @@
</tr>
</table>
</form>
<h3>Import (non-destructive):</h3>
{% endcomment %}
<h3>Import (on top of existing data):</h3>
<p>To get a fully-functioning system, all these imports must be done and in this order.
<form name="import" method="post" action="">
{% csrf_token %}
<table>
<tr>
<td>people from folk.csv using parsers\people.py</td>
<th colspan="2">Import from master files</th>
</tr>
<tr>
<td>all caves </td>
<td><input type="checkbox" name="import_caves"/></td>
</tr>
<tr>
<td>all people </td>
<td><input type="checkbox" name="import_people"/></td>
</tr>
<tr>
<td>logbook entries using parsers\logbooks.py</td>
<td>all surveys scans </td>
<td><input type="checkbox" name="import_surveyscans"/></td>
</tr>
<tr>
<td>all logbooks</td>
<td><input type="checkbox" name="import_logbooks"/></td>
</tr>
<tr>
<td>QMs using parsers\QMs.py</td>
<td>QMs (from old csv files)</td>
<td><input type="checkbox" name="import_QMs" /></td>
</tr>
<tr>
<td>survey scans using parsers\surveys.py</td>
<td><input type="checkbox" name="import_surveys" /></td>
<td>all drawings files</td>
<td><input type="checkbox" name="import_drawingsfiles" /></td>
</tr>
<tr>
<td>survex data using parsers\survex.py</td>
<td>all survex data (10 minutes)</td>
<td><input type="checkbox" name="import_survex" /></td>
</tr>
</table>
<p>
<input type="submit" id="Import" value="Import">
<input type="submit" name="check_all_import" id="check_all_import" value="Check all" disabled >
</p>
</form>
<h3>Export to legacy formats:</h3>
<p>&nbsp;
<h3>Export to a different format:</h3>
<p>This creates 'newlogbook.html' in the years/&lt;year&gt;/ folder
<table>
<th>
<td>Export onto server</td>
<td>Export and Download</td>
</th>
<tr>
<td>
surveys to Surveys.csv
</td>
<td>
</td>
<th>Export and Download</th>
</tr>
<tr>
<td>
<form name="export" method="get" action={% url "downloadlogbook" %}>
{% csrf_token %}
<p>Download a logbook file which is dynamically generated by Troggle.</p>
<p>
@ -99,8 +111,8 @@
<p>
Output style:
<select name="extension">
<option value="txt">.txt file with MediaWiki markup - 2008 style</option>
<option value="html">.html file - 2005 style</option>
<option value="txt">.txt file with MediaWiki markup</option>
</select>
</p>
<p>
@ -112,5 +124,5 @@
</table>
</form>
</div>
{% endblock %}