Imports in control panel work again

core/views/other.py

@@ -9,7 +9,9 @@ from django.shortcuts import render
 from django.template import Context, loader
 from django.core.files.storage import FileSystemStorage, default_storage
 
-import troggle.parsers.imports
+from troggle.parsers.imports import import_caves, import_people, import_surveyscans
+from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
+# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
 from troggle.core.models.troggle import Expedition, Person, PersonExpedition
 from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
 from .login import login_required_if_public
@@ -82,27 +84,60 @@ def frontpage(request):
     return render(request,'frontpage.html', locals())
 
 
+@login_required_if_public
 def controlpanel(request):
+    '''This should be re-written to use ajax so that the user can see the progress
+    of the actions.
+    
+    Also need to add reinit_db option
+    '''
     jobs_completed=[]
-    if request.method=='POST':
+    
-        if request.user.is_superuser: # expoadmin is both .is_staff and ._is_superuser
+    def process_imports():
-            # NONE of this works now that databaseReset (now parsers.imports) has been so extensively rewritten
+        '''databaseReset.py
-            reinit_db()
+            jq.enq("reinit",reinit_db)
+            jq.enq("caves",import_caves)
+            jq.enq("people",import_people)
+            jq.enq("scans",import_surveyscans)
+            jq.enq("logbooks",import_logbooks)
+            jq.enq("QMs",import_QMs)
+            jq.enq("drawings",import_drawingsfiles)
+            jq.enq("survex",import_survex)
+        '''
+        if request.POST.get("import_caves", False):
             import_caves()
+            jobs_completed.append('Caves')
+        if request.POST.get("import_people", False):
             import_people()
+            jobs_completed.append('People')
+        if request.POST.get("import_surveyscans", False):
             import_surveyscans()
+            jobs_completed.append('Scans')
+        if request.POST.get("import_logbooks", False):
             import_logbooks()
+            jobs_completed.append('Logbooks')
+        if request.POST.get("import_QMs", False):
             import_QMs()
-            import_dwgfiles()
+            jobs_completed.append('QMs')
-            import_survexblks()
+        if request.POST.get("import_drawingsfiles", False):
-            import_survexpos()
+            import_drawingsfiles()
-        else:
+            jobs_completed.append('Drawings')
-            if request.user.is_authenticated: #The user is logged in, but is not a superuser.
+        if request.POST.get("import_survex", False):
-                return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'error':'You must be a superuser to use that feature.'})
+            import_survex()
-            else:
+            jobs_completed.append('Survex')
-                return HttpResponseRedirect(reverse('auth_login'))
+            
+        print("", flush=True)
+
+    if not request.user.is_superuser: # expoadmin is both .is_staff and ._is_superuser
+        return render(request,'controlPanel.html', {'error': 'You are logged in, but not logged in as "expoadmin". \nLogout and login again to contnue.'})
+    else:
+        if request.method=='POST':
+            #reinit_db()
+            process_imports()
+            return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
+        else:
+            return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
 
-    return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
 
 
 def downloadlogbook(request,year=None,extension=None,queryset=None):

parsers/imports.py

@@ -15,7 +15,8 @@ import troggle.parsers.surveys
 import troggle.parsers.logbooks
 import troggle.parsers.QMs
 
-'''Master data importUsed only by databaseReset.py currently
+'''Master data import.
+Used only by databaseReset.py and online controlpanel.
 '''
 
 def import_caves():
@@ -46,9 +47,9 @@ def import_QMs():
 
 def import_survex():
     # when this import is moved to the top with the rest it all crashes horribly
+    print("-- Importing Survex and Entrance Positions")
     with transaction.atomic():
         import troggle.parsers.survex 
-    print("-- Importing Survex and Entrance Positions")
     print(" - Survex Blocks")
     with transaction.atomic():
         troggle.parsers.survex.LoadSurvexBlocks()

parsers/people.py

@@ -124,6 +124,7 @@ def load_people_expos():
                 lookupAttribs = {'person':person, 'expedition':expedition}
                 nonLookupAttribs = {'nickname':nickname, 'is_guest':(personline[header["Guest"]] == "1")}
                 save_carefully(PersonExpedition, lookupAttribs, nonLookupAttribs)
+    print("", flush=True)
 
 
 # used in other referencing parser functions

parsers/surveys.py

@@ -14,8 +14,10 @@ from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
 from troggle.core.models.troggle import DataIssue
 from troggle.core.utils import save_carefully
 
-'''Scans through all the :drawings: repository looking
+'''Searches through all the :drawings: repository looking
 for tunnel and therion files
+    
+Searches through all the survey scans directories in expofiles, looking for images to be referenced.
 '''
 
 
@@ -41,7 +43,7 @@ def listdir(*directories):
         return [folder.rstrip(r"/") for folder in folders]
 
 
-# handles url or file, so we can refer to a set of scans on another server
+# handles url or file, so we can refer to a set of scans (not drawings) on another server
 def GetListDir(sdir):
     res = [ ]
     if sdir[:7] == "http://":
@@ -80,7 +82,7 @@ def LoadListScansFile(wallet):
         
 # this iterates through the scans directories (either here or on the remote server)
 # and builds up the models we can access later
-def LoadListScans():
+def load_all_scans():
 
     print(' - Loading Survey Scans')
 
@@ -116,17 +118,18 @@ def LoadListScans():
             wallet = Wallet(fpath=ff, walletname=f)
             wallet.save()
             LoadListScansFile(wallet)
-            
 
-def find_tunnel_scan(dwgfile, path):
+    print("", flush=True)
+
+def find_tunnel_file(dwgfile, path):
     '''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file
     which we have already seen when we imported all the files we could find in the surveyscans direstories
     '''
     wallet, scansfile = None, None
-    mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg))$", path)
+    mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg|gif))$", path)
     if mscansdir:
         scanswalletl = Wallet.objects.filter(walletname=mscansdir.group(1))
-        # This should properly detect if a list of folders is returned and do something sensible, not just pick the first.
+        # This should be chnaged to properly detect if a list of folders is returned and do something sensible, not just pick the first.
         if len(scanswalletl):
             wallet = scanswalletl[0]
             if len(scanswalletl) > 1:
@@ -148,7 +151,7 @@ def find_tunnel_scan(dwgfile, path):
         if scansfile:
             dwgfile.scans.add(scansfile)
     
-    elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif|pdf)$(?i)", path):
+    elif path and not re.search(r"\.(?:png|jpg|pdf|jpeg|gif)$(?i)", path):
         name = os.path.split(path)[1]
         rdwgfilel = DrawingFile.objects.filter(dwgname=name)
         if len(rdwgfilel):
@@ -248,7 +251,7 @@ def setdwgfileinfo(dwgfile):
     # <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0">
     
     for path, style in rx_pcpath.findall(ttext):
-        find_tunnel_scan(dwgfile, path.decode())
+        find_tunnel_file(dwgfile, path.decode())
     
     # should also scan and look for survex blocks that might have been included, and image scans
     # which would populate dwgfile.survexfile

templates/controlPanel.html

@@ -3,28 +3,27 @@
 
 {% if jobs_completed %}
     <div class="noticeBox">
-            Just finished running:
+            &nbsp;Just finished running:
                 <ul>
                 {% for job in jobs_completed %}
                     <li>{{ job }}</li>
                 {% endfor %}
                 </ul>
-            {% if settings.LOGFILE %}See the logfile at {{settings.LOGFILE}} for more information.{% endif %}
+                &nbsp;See <a href="/dataissues">Data Issues</a> report
-            <a href="#" class="closeDiv">dismiss this message</a>
+                <br>&nbsp;
     </div>
 {% endif %}
 
 {% if error %}
     <div class="noticeBox">
             {{ error }}
-            <a href="#" class="closeDiv">dismiss this message</a>
     </div>
 {% endif %}
 
-<h2>NONE of this is working currently</h2>
+<div style="column-count: 2;">
-<h2>The code behind this page is under repair</h2>
+{%comment%}
-
 <form name="reset" method="post" action="">
+{% csrf_token %}              
     <h3>Wipe:</h3>
     <table>
         <tr>
@@ -36,55 +35,68 @@
         </tr>
     </table>
 </form>
-<h3>Import (non-destructive):</h3>
+{% endcomment %}
+
+<h3>Import (on top of existing data):</h3>
+<p>To get a fully-functioning system, all these imports must be done and in this order.
 <form name="import" method="post" action="">
+{% csrf_token %}              
     <table>
         <tr>
-            <td>people from folk.csv using parsers\people.py</td>
+            <th colspan="2">Import from master files</th>
+        </tr>    
+        <tr>
+            <td>all caves </td>
+            <td><input type="checkbox" name="import_caves"/></td>
+        </tr>
+        <tr>
+            <td>all people </td>
             <td><input type="checkbox" name="import_people"/></td>
         </tr>
         <tr>
-            <td>logbook entries using parsers\logbooks.py</td>
+            <td>all surveys scans </td>
+            <td><input type="checkbox" name="import_surveyscans"/></td>
+        </tr>
+        <tr>
+            <td>all logbooks</td>
             <td><input type="checkbox" name="import_logbooks"/></td>
         </tr>
         <tr>
-            <td>QMs using parsers\QMs.py</td>
+            <td>QMs (from old csv files)</td>
             <td><input type="checkbox" name="import_QMs" /></td>
         </tr>
         <tr>
-            <td>survey scans using parsers\surveys.py</td>
+            <td>all drawings files</td>
-            <td><input type="checkbox" name="import_surveys" /></td>
+            <td><input type="checkbox" name="import_drawingsfiles" /></td>
         </tr>
         <tr>
-            <td>survex data using parsers\survex.py</td>
+            <td>all survex data (10 minutes)</td>
             <td><input type="checkbox" name="import_survex" /></td>
         </tr>
     </table>
     <p>
       <input type="submit" id="Import" value="Import">
       
-      <input type="submit" name="check_all_import" id="check_all_import" value="Check all" disabled >
     </p>
 </form>
 
 
-<h3>Export to legacy formats:</h3>
+<p>&nbsp;
+
+
+<h3>Export to a different format:</h3>
+<p>This creates 'newlogbook.html' in the years/&lt;year&gt;/ folder
 <table>
-
-<th>
-    <td>Export onto server</td>
-    <td>Export and Download</td>
-</th>
-
 <tr>
-      <td>
-        surveys to Surveys.csv
-    </td>
-      <td>
 
-    </td>
+
+    <th>Export and Download</th>
+
+</tr>
+<tr>
     <td>
         <form name="export" method="get" action={% url "downloadlogbook" %}>
+{% csrf_token %}              
             <p>Download a logbook file which is dynamically generated by Troggle.</p>
             
             <p>
@@ -99,8 +111,8 @@
             <p>
             Output style: 
             <select name="extension">
-              <option value="txt">.txt file with MediaWiki markup - 2008 style</option>
               <option value="html">.html file - 2005 style</option>
+              <option value="txt">.txt file with MediaWiki markup</option>
             </select>
             </p>
             <p>
@@ -112,5 +124,5 @@
 
 </table>
 </form>
-
+</div>
 {% endblock %}