From a0c5a34b3f1fc3e10a77d5e00b86bd98cfac1fe0 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 1 Jun 2020 00:42:48 +0100 Subject: [PATCH] Progress dots on importing data --- databaseReset.py | 29 +++++++++++++---------------- parsers/survex.py | 26 ++++++++++++++++++++------ parsers/surveys.py | 2 +- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/databaseReset.py b/databaseReset.py index 0b7a0d4..8f00a7f 100755 --- a/databaseReset.py +++ b/databaseReset.py @@ -124,6 +124,8 @@ def import_tunnelfiles(): #def dumplogbooks(): #def writeCaves(): + # Writes out all cave and entrance HTML files to + # folder specified in settings.CAVEDESCRIPTIONS # for cave in Cave.objects.all(): # cave.writeDataFile() # for entrance in Entrance.objects.all(): @@ -140,8 +142,8 @@ class JobQueue(): self.results = {} self.results_order=[ "date","runlabel","reinit", "caves", "people", - "logbooks", "QMs", "survexblks", "survexpos", - "tunnel", "scans", "surveyimgs", "test", "dirsredirect", "syncuser" ] + "logbooks", "QMs", "scans", "survexblks", "survexpos", + "tunnel", "surveyimgs", "test", "dirsredirect", "syncuser" ] for k in self.results_order: self.results[k]=[] self.tfile = "import_profile.json" @@ -319,6 +321,8 @@ class JobQueue(): for k in self.results_order: if k =="dirsredirect": break + if k =="surveyimgs": + break elif k =="syncuser": break elif k =="test": @@ -370,17 +374,15 @@ def usage(): profile - print the profile from previous runs. Import nothing. reset - normal usage: clear database and reread everything from files - time-consuming - caves - read in the caves + caves - read in the caves (must run first after reset) + people - read in the people from folk.csv (must run before logbooks) logbooks - read in the logbooks - people - read in the people from folk.csv QMs - read in the QM csv files (older caves only) - scans - the survey scans in all the wallets + scans - the survey scans in all the wallets (must run before survex) survex - read in the survex files - all the survex blocks but not the x/y/z positions survexpos - just the x/y/z Pos out of the survex files - survexall - both survex and survexpos tunnel - read in the Tunnel files - which scans the survey scans too - drawings - Tunnel, QMs, scans reinit - clear database (delete everything) and make empty tables. Import nothing. syncuser - needed after reloading database from SQL backup @@ -394,6 +396,8 @@ def usage(): caves and logbooks must be run on an empty db before the others as they set up db tables used by the others. + + the in-memory phase is on an empty db, so always runs reinit, caves & people for this phase """) if __name__ == "__main__": @@ -423,8 +427,6 @@ if __name__ == "__main__": jq.enq("people",import_people) elif "QMs" in sys.argv: jq.enq("QMs",import_QMs) - elif "reinit" in sys.argv: - jq.enq("reinit",reinit_db) elif "reset" in sys.argv: jq.enq("reinit",reinit_db) jq.enq("dirsredirect",dirsredirect) @@ -444,19 +446,14 @@ if __name__ == "__main__": jq.enq("survexpos",import_survexpos) elif "tunnel" in sys.argv: jq.enq("tunnel",import_tunnelfiles) - elif "survexall" in sys.argv: - jq.enq("survexblks",import_survexblks) - jq.enq("survexpos",import_survexpos) - elif "drawings" in sys.argv: - jq.enq("QMs",import_QMs) - jq.enq("scans",import_surveyscans) - jq.enq("tunnel",import_tunnelfiles) elif "surveyimgs" in sys.argv: jq.enq("surveyimgs",import_surveyimgs) # imports into tables which are never read elif "autologbooks" in sys.argv: # untested in 2020 import_auto_logbooks() elif "dumplogbooks" in sys.argv: # untested in 2020 dumplogbooks() +# elif "writecaves" in sys.argv: # untested in 2020 - will overwrite input files!! +# writeCaves() elif "profile" in sys.argv: jq.loadprofiles() jq.showprofile() diff --git a/parsers/survex.py b/parsers/survex.py index 38cae62..98eed39 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1,7 +1,10 @@ -import troggle.settings as settings -import troggle.core.models as models -import troggle.settings as settings - +from __future__ import (absolute_import, division, + print_function) +import sys +import os +import re +import time +from datetime import datetime, timedelta from subprocess import call, Popen, PIPE from troggle.parsers.people import GetPersonExpeditionNameLookup @@ -129,7 +132,7 @@ regex_team_member = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)" regex_qm = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$') insp = "" - +callcount = 0 def RecursiveLoad(survexblock, survexfile, fin, textlines): """Follows the *include links in all the survex files from the root file 1623.svx and reads in the survex blocks, other data and the wallet references (survexscansfolder) as it @@ -141,11 +144,18 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): stardata = stardatadefault teammembers = [ ] global insp + global callcount # uncomment to print out all files during parsing print(insp+" - Reading file: " + survexblock.survexfile.path + " <> " + survexfile.path) stamp = datetime.now() lineno = 0 + + sys.stderr.flush(); + callcount +=1 + if callcount >=10: + callcount=0 + print(".", file=sys.stderr,end='') # Try to find the cave in the DB if not use the string as before path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", survexblock.survexfile.path) @@ -445,7 +455,7 @@ def LoadAllSurvexBlocks(): models.DataIssue.objects.filter(parser='survex').delete() print(' - Loading All Survex Blocks...') - print(' - redirecting stdout to loadsurvexblks.log ...') + print(' - redirecting stdout to loadsurvexblks.log...') stdout_orig = sys.stdout # Redirect sys.stdout to the file sys.stdout = open('loadsurvexblks.log', 'w') @@ -467,6 +477,10 @@ def LoadAllSurvexBlocks(): # Close the file sys.stdout.close() + print("+", file=sys.stderr) + sys.stderr.flush(); + + # Restore sys.stdout to our old saved file handler sys.stdout = stdout_orig print(' - Loaded All Survex Blocks.') diff --git a/parsers/surveys.py b/parsers/surveys.py index 8a45bc3..5087690 100644 --- a/parsers/surveys.py +++ b/parsers/surveys.py @@ -145,7 +145,7 @@ def LoadListScansFile(survexscansfolder): survexscansingle.save() c+=1 if c>=10: - print(".", end=' ') + print(".", end='') c = 0