import fixes & statistics table

databaseReset.py

@@ -18,7 +18,6 @@ print(" 1 settings on loading databaseReset.py")
 
 from troggle.core.models_caves import Cave, Entrance
 import troggle.parsers.caves
-#import troggle.settings
 import troggle.flatpages.models
 import troggle.logbooksdump
 import troggle.parsers.people
@@ -54,6 +53,8 @@ def reinit_db():
     in memory (django python models, not the database), so there is already a full load
     of stuff known. Deleting the db file does not clear memory.
     """
+    print("Reinitialising db ",end="")
+    print(django.db.connections.databases['default']['NAME'])
     currentdbname = settings.DATABASES['default']['NAME']
     if currentdbname == ':memory:':
         # closing connections should wipe the in-memory database
@@ -62,11 +63,15 @@ def reinit_db():
             print(" ! Closing another connection to db...")
             conn.close()
     elif django.db.connections.databases['default']['ENGINE'] == 'django.db.backends.sqlite3':
-        try:
-            os.remove(currentdbname)
-        except OSError:
-            print(" ! OSError on removing: " + currentdbname + " (Is the file open in another app?\n")
-            raise
+        if os.path.isfile(currentdbname):
+            try:
+                print(" - deleting " + currentdbname)
+                os.remove(currentdbname)
+            except OSError:
+                print(" ! OSError on removing: " + currentdbname + " (Is the file open in another app?\n")
+                raise
+        else:
+            print(" - No database file found: " + currentdbname + " ..continuing, will create it.\n")
     else:
         cursor = django.db.connection.cursor()
         cursor.execute("DROP DATABASE %s" % currentdbname)
@@ -74,15 +79,14 @@ def reinit_db():
         cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % currentdbname)
         cursor.execute("USE %s" % currentdbname)
 
-    #Sync user - needed after reload
-    print(" - Migrating: " + settings.DATABASES['default']['NAME'])
-    print(django.db.connections.databases['default']['NAME'])
+     #Sync user - needed after reload
+    print(" - Migrating: " + django.db.connections.databases['default']['NAME'])
 
     management.call_command('migrate', interactive=False)
     print(" - done migration on: " + settings.DATABASES['default']['NAME'])
+    print("users in db already: ",len(User.objects.all()))
     try:
-        print(" - Setting up admin user on: " + settings.DATABASES['default']['NAME'])
-        print(django.db.connections.databases['default']['NAME'])
+        print(" - Setting up admin user on: " + django.db.connections.databases['default']['NAME'])
         print(" - user: {} ({:.5}...) <{}> ".format(expouser, expouserpass, expouseremail))
         user = User.objects.create_user(expouser, expouseremail, expouserpass)
         user.is_staff = True
@@ -93,6 +97,8 @@ def reinit_db():
         print(django.db.connections.databases['default']['NAME'])
         print(" ! You probably have not got a clean db when you thought you had.\n")
         print(" ! Also you are probably NOT running an in-memory db now.\n")
+        print("users  in db: ",len(User.objects.all()))
+        print("tables in db: ",len(connection.introspection.table_names()))
         memdumpsql(fn='integrityfail.sql')
         django.db.connections.databases['default']['NAME'] = ':memory:'
         #raise
@@ -123,6 +129,9 @@ def restore_dbsettings():
 
 def set_in_memory_dbsettings():
     django.db.close_old_connections() # needed if MySQL running?
+    for conn in django.db.connections.all():
+        print(" ! Closing another connection to db in set_in_memory_dbsettings")
+        conn.close()
     settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3', 
         'AUTOCOMMIT': True, 
         'ATOMIC_REQUESTS': False, 
@@ -250,7 +259,6 @@ class JobQueue():
         with open(self.tfile, 'w') as f:
             json.dump(self.results, f)     
         return True
-    
 
     def runqonce(self):
         """Run all the jobs in the queue provided - once
@@ -309,10 +317,8 @@ class JobQueue():
         self.loadprofiles()
         store_dbsettings()
 
-        print("-- start   ",  settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
-        print(django.db.connections.databases['default']['NAME'])
+        print("-- start   ",  django.db.connections.databases['default']['ENGINE'], django.db.connections.databases['default']['NAME'])
 
-        
         if dbname ==":memory:":
             # just run, and save the sql file
             self.runqonce()
@@ -426,7 +432,7 @@ def usage():
              QMs       - read in the QM csv files (older caves only)
              scans     - the survey scans in all the wallets (must run before survex)
              survex    - read in the survex files - all the survex blocks but not the x/y/z positions
-             survexpos - just the x/y/z Pos out of the survex files (not needed)
+             survexpos - just the x/y/z Pos out of the survex files (not needed) -- Never used.
 
              tunnel    - read in the Tunnel files - which scans the survey scans too
 
@@ -459,15 +465,7 @@ if __name__ == "__main__":
     else: 
         runlabel=None
 
-    store_dbsettings()
-    set_in_memory_dbsettings()
-    print(" - django.setup - next")
-    try:
-        django.setup()
-    except:
-        print(" ! COMPLICATED FAILURE. Does not occur with a valid 'troggle.sqlite' database in place.")
-        raise
-    print(" - django.setup - done")
+
 
     jq = JobQueue(runlabel)
     
@@ -493,9 +491,9 @@ if __name__ == "__main__":
         jq.enq("scans",import_surveyscans)
         jq.enq("logbooks",import_logbooks)
         jq.enq("QMs",import_QMs)
-        jq.enq("survexblks",import_survexblks)
-        jq.enq("survexpos",import_survexpos)
         jq.enq("tunnel",import_tunnelfiles)
+        jq.enq("survexblks",import_survexblks)
+        #jq.enq("survexpos",import_survexpos)
     elif "scans" in sys.argv:
         jq.enq("scans",import_surveyscans)
     elif "survex" in sys.argv:
@@ -524,7 +522,18 @@ if __name__ == "__main__":
         print("%s not recognised as a command." % sys.argv[1])
         exit()
 
-    #jq.run_now_django_tests(1) 
+    store_dbsettings()
+    #set_in_memory_dbsettings()
+    print(" - django.setup - next")
+    try:
+        django.setup()
+    except:
+        print(" ! Cyclic reference failure. Can occur when the initial db is empty. Fixed now (in UploadFileForm) but easy to reintroduce..")
+        raise
+    print(" - django.setup - done")
+    
+    #set_in_memory_dbsettings() # seems to be ignored. Appears to be set but in reality.
+    #jq.run_now_django_tests(1) # actually does set db to :memory: - but invisibly !
 
     jq.run()
     jq.showprofile()