Made a jobqueue to time how long importing takes

databaseReset.py

@@ -1,5 +1,6 @@
 import os
 import time
+import timeit
 import settings
 os.environ['PYTHONPATH'] = settings.PYTHON_PATH
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
@@ -79,7 +80,7 @@ def pageredirects():
         f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
         f.save()
 
-def reset():
+def reset(): # unused now that we have a jobqueue
     """ Wipe the troggle database and import everything from legacy data
     """
     reload_db()
@@ -148,6 +149,93 @@ def dumplogbooks():
             f.write(unicode(output).encode( "utf-8" ))
             f.close()
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
+class JobQueue():
+    """ A list of import operations to run. Always reports times
+    in the same order """
+
+
+    #Constructor creates a list
+    def __init__(self,run):
+        self.runlabel = run
+        self.queue = [] # tuples of (jobname, jobfunction)
+        self.results = {}
+        self.results_order=[
+            "date","runlabel","reload", "caves", "people",
+            "logbooks", "surveyscans", "QMs", "survex"
+            "tunnel", "surveys", "test", "makedirs", "redirect" ]
+        for k in self.results_order:
+            self.results[k]=[]
+
+    #Adding elements to queue
+    def enq(self,label,func):
+        self.queue.append((label,func))
+        return True
+
+    #Removing the last element from the queue
+    def deq(self):
+        if len(self.queue)>0:
+            return self.queue.pop()
+        return ("Queue Empty!")
+
+    def size(self):
+        return len(self.queue)
+
+    def run(self):
+        import json
+        tfile = "import_times.json"
+        if os.path.isfile(tfile):
+            try:
+                f = open(tfile, "r")
+                data = json.load(f)
+                for j in data:
+                    self.results[j] = data[j]
+            except:
+                print "FAILURE parsing JSON file %s" % (tfile)
+                # Python bug: https://github.com/ShinNoNoir/twitterwebsearch/issues/12
+            f.close()
+
+        print "** Running job ", self.runlabel
+        for i in self.queue:
+           #print "*- Running \"", i[0], "\""
+           start = time.time()
+           i[1]()
+           duration = time.time()-start
+           #print "\n*- Ended \"",  i[0], "\""
+           self.results[i[0]].append(duration)
+        self.results["date"].append(start)
+        self.results["runlabel"].append(self.runlabel)
+        print "** Ended all jobs."
+        #print self.results
+
+        with open(tfile, 'w') as f:
+            json.dump(self.results, f)
+        
+        for i in self.results_order:
+            percen=0
+            if i == "runlabel":     
+                pass
+            if i =="date":
+                # Calculate dates as days before present to one decimal place
+                pass
+            elif len(self.results[i])>0:        
+                lst = self.results[i]
+                e = len(lst)-1
+                percen = 100* (lst[e] - lst[e-1])/lst[e-1]
+            if abs(percen) >0.1:
+                print '%15s %8.1f%%' % (i,  percen)
+            else:
+                print '%15s ' % (i)
+
+        return True
+
+def importtest():
+    from  random import randrange
+    k = 0
+    for i  in range(5+randrange(15)):
+        for j in range(i):
+            k += i
+        #print k,
+    return True
 
 def usage():
     print("""Usage is 'python databaseReset.py <command>'
@@ -160,14 +248,16 @@ def usage():
              reload_db - clear database i.e. delete everything
              scans     - NOT the scanned surveynotes ?!
              survex    - read in the survex files - all the survex blocks
-             survexpos - just the Pos out of the survex files
              surveys   - read in the scanned surveynotes
              tunnel    - read in the Tunnel files - which scans the surveyscans too
 
+             survexpos - just the Pos out of the survex files (not part of reset)
+
              resetend     - (archaic?)
              writecaves   - *disabled* (archaic?)
              autologbooks - read in autologbooks (what are these?)
              dumplogbooks - write out autologbooks (not working?)
+             test         - testing...
              """)
 
 if __name__ == "__main__":
@@ -176,38 +266,56 @@ if __name__ == "__main__":
     import django
     django.setup()
 
-    if "caves" in sys.argv:
+    runlabel = sys.argv[len(sys.argv)-1]
-        import_caves()
+    jq = JobQueue(runlabel)
+
+
+    if "test" in sys.argv:
+        jq.enq("test",importtest)
+        jq.enq("caves",importtest)
+        jq.enq("people",importtest)
+    elif "caves" in sys.argv:
+        jq.enq("caves",import_caves)
     elif "logbooks" in sys.argv:
         # management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
-        import_logbooks()
+        jq.enq("logbooks",import_logbooks)
     elif "people" in sys.argv:
-        import_people()
+        jq.enq("logbooks",import_people)
     elif "QMs" in sys.argv:
-        import_QMs()
+        jq.enq("QMs",import_QMs)
     elif "reload_db" in sys.argv:
-        reload_db()
+        jq.enq("reload_db",reload_db)
     elif "reset" in sys.argv:
-        reset()
+        jq.enq("reload",reload_db)
+        jq.enq("makedirs",make_dirs)
+        jq.enq("redirect",pageredirects)
+        jq.enq("caves",import_caves)
+        jq.enq("logbooks",import_people)
+        jq.enq("scans",import_surveyscans)
+        jq.enq("logbooks",import_logbooks)
+        jq.enq("QMs",import_QMs)
+        jq.enq("survex",import_survex)
+        jq.enq("tunnel",import_tunnelfiles)
+        jq.enq("surveys",import_surveys)
     elif "scans" in sys.argv:
-        import_surveyscans()
+        jq.enq("scans",import_surveyscans)
     elif "survex" in sys.argv:
         # management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
-        import_survex()
+        jq.enq("survex",import_survex)
     elif "survexpos" in sys.argv:
         # management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
         import parsers.survex
-        parsers.survex.LoadPos()    
+        jq.enq("survexpos",parsers.survex.LoadPos)
     elif "surveys" in sys.argv:
-        import_surveys()
+        jq.enq("surveys",import_surveys)
     elif "tunnel" in sys.argv:
-        import_tunnelfiles()
+        jq.enq("tunnel",import_tunnelfiles)
     elif "help" in sys.argv:
         usage()
     elif "resetend" in sys.argv:
-        import_QMs()
+        jq.enq("QMs",import_QMs)
-        import_tunnelfiles()
+        jq.enq("tunnel",import_tunnelfiles)
-        import_surveys()
+        jq.enq("surveys",import_surveys)
         #import_descriptions() # no longer present
         #parse_descriptions() # no longer present
     # elif "writeCaves" in sys.argv:
@@ -219,3 +327,5 @@ if __name__ == "__main__":
     else:
         print("%s not recognised" % sys.argv)
         usage()
+
+    jq.run()