Progress dots on importing data

parsers/survex.py

@@ -1,7 +1,10 @@
-import troggle.settings as settings
-import troggle.core.models as models
-import troggle.settings as settings
-
+from __future__ import (absolute_import, division,
+                        print_function)
+import sys
+import os
+import re
+import time
+from datetime import datetime, timedelta
 from subprocess import call, Popen, PIPE
 
 from troggle.parsers.people import GetPersonExpeditionNameLookup
@@ -129,7 +132,7 @@ regex_team_member        = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)"
 regex_qm      = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')
 
 insp = ""
-
+callcount = 0
 def RecursiveLoad(survexblock, survexfile, fin, textlines):
     """Follows the *include links in all the survex files from the root file 1623.svx
     and reads in the survex blocks, other data and the wallet references (survexscansfolder) as it
@@ -141,11 +144,18 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
     stardata = stardatadefault
     teammembers = [ ]
     global insp
+    global callcount
 
     # uncomment to print out all files during parsing 
     print(insp+"  - Reading file: " + survexblock.survexfile.path + " <> " + survexfile.path)
     stamp = datetime.now()
     lineno = 0
+    
+    sys.stderr.flush();
+    callcount +=1
+    if callcount >=10:
+        callcount=0
+        print(".", file=sys.stderr,end='')
 
     # Try to find the cave in the DB if not use the string as before
     path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", survexblock.survexfile.path)
@@ -445,7 +455,7 @@ def LoadAllSurvexBlocks():
     models.DataIssue.objects.filter(parser='survex').delete()
     print(' - Loading All Survex Blocks...')
     
-    print('  - redirecting stdout to loadsurvexblks.log ...')
+    print('  - redirecting stdout to loadsurvexblks.log...')
     stdout_orig = sys.stdout
     # Redirect sys.stdout to the file
     sys.stdout = open('loadsurvexblks.log', 'w')
@@ -467,6 +477,10 @@ def LoadAllSurvexBlocks():
     
     # Close the file
     sys.stdout.close()
+    print("+", file=sys.stderr)
+    sys.stderr.flush();
+    
+    
     # Restore sys.stdout to our old saved file handler
     sys.stdout = stdout_orig
     print(' - Loaded All Survex Blocks.')

parsers/surveys.py

@@ -145,7 +145,7 @@ def LoadListScansFile(survexscansfolder):
             survexscansingle.save()
             c+=1
             if c>=10:
-                print(".", end=' ')
+                print(".", end='')
                 c = 0