remove in-memory phase of data import

core/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2020-06-20 16:03
+# Generated by Django 1.11.29 on 2020-06-20 17:43
 from __future__ import unicode_literals
 
 from django.conf import settings

databaseReset.py

@@ -35,24 +35,15 @@ from troggle.parsers.imports import import_caves, import_people, import_surveysc
         import_logbooks, import_QMs, import_survex, import_tunnelfiles
 import troggle.logbooksdump
 
-
-# NOTE databaseReset.py is *imported* by views_other.py as it is used in the control panel
-# presented there.
-
 if os.geteuid() == 0:
     # This protects the server from having the wrong file permissions written on logs and caches
     print("This script should be run as expo not root - quitting")
     exit()
 
-dbengine = ""
-dbname = ""
-dbdefault =""
-
 expouser=settings.EXPOUSER
 expouserpass=settings.EXPOUSERPASS
 expouseremail=settings.EXPOUSER_EMAIL
 
-
 def reinit_db():
     """Rebuild database from scratch. Deletes the file first if sqlite is used,
     otherwise it drops the database and creates it.
@@ -75,7 +66,7 @@ def reinit_db():
                 print(" - deleting " + currentdbname)
                 os.remove(currentdbname)
             except OSError:
-                print(" ! OSError on removing: " + currentdbname + " (Is the file open in another app? Is the server running?\n")
+                print(" ! OSError on removing: " + currentdbname + "\n ! Is the file open in another app? Is the server running?\n")
                 raise
         else:
             print(" - No database file found: " + currentdbname + " ..continuing, will create it.\n")
@@ -86,7 +77,6 @@ def reinit_db():
         cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % currentdbname)
         cursor.execute("USE %s" % currentdbname)
 
-     #Sync user - needed after reload
     print(" - Migrating: " + django.db.connections.databases['default']['NAME'])
 
     management.call_command('makemigrations','core', interactive=False)
@@ -123,44 +113,6 @@ def memdumpsql(fn):
             f.write('%s\n' % line.encode("utf8"))
     return True
 
-def store_dbsettings():
-    global dbengine
-    global dbname
-    global dbdefault
-    dbengine = settings.DATABASES['default']['ENGINE']
-    dbname = settings.DATABASES['default']['NAME']
-    dbdefault = settings.DATABASES['default']
-
-def restore_dbsettings():
-    settings.DATABASES['default'] = dbdefault
-    settings.DATABASES['default']['ENGINE'] = dbengine
-    settings.DATABASES['default']['NAME'] = dbname
-    django.db.connections.databases['default'] = dbdefault
-    django.db.connections.databases['default']['ENGINE'] = dbengine
-    django.db.connections.databases['default']['NAME'] = dbname
-
-def set_in_memory_dbsettings():
-    django.db.close_old_connections() # needed if MySQL running?
-    for conn in django.db.connections.all():
-        print(" ! Closing another connection to db in set_in_memory_dbsettings")
-        conn.close()
-    settings.DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3', 
-        'AUTOCOMMIT': True, 
-        'ATOMIC_REQUESTS': False, 
-        'NAME': ':memory:', 
-        'CONN_MAX_AGE': 0, 
-        'TIME_ZONE': 'UTC', 
-        'OPTIONS': {}, 
-        'HOST': '', 
-        'USER': '', 
-        'TEST': {'COLLATION': None, 'CHARSET': None, 'NAME': None, 'MIRROR': None}, 
-        'PASSWORD': '', 
-        'PORT': ''}
-    settings.DATABASES['default']['ENGINE'] = 'django.db.backends.sqlite3'
-    settings.DATABASES['default']['NAME'] = ':memory:'
-    django.db.connections.databases['default']['ENGINE'] = 'django.db.backends.sqlite3'
-    django.db.connections.databases['default']['NAME'] = ':memory:'
-
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
 # These functions moved to a different file - not used currently.
 #import logbooksdump
@@ -259,83 +211,16 @@ class JobQueue():
             self.results[j].append(None) # append a placeholder
 
     def run_now_django_tests(self,n):
-        store_dbsettings()
-        # this leaves the db set to :memory: whatever it was initially
         management.call_command('test', verbosity=n)
         django.db.close_old_connections() 
-        restore_dbsettings()
-        # and whatever I do, it stays that way !
-
-    def skip_memory_phase(self):
-        if not self.runlabel:
-            return True
-        else:
-            if self.runlabel == "" or self.runlabel[0:2] == "F-":
-                return True
-        return False
 
     def run(self):
-        """First runs all the jobs in the queue against a scratch in-memory db
-        then re-runs the import against the db specified in settings.py
-        Default behaviour is to skip the in-memory phase.
-        When MySQL is the db the in-memory phase crashes as MySQL does not properly
-        relinquish some kind of db connection (not fixed yet)
-        """
         self.loadprofiles()
-        store_dbsettings()
-
         print("-- start   ",  django.db.connections.databases['default']['ENGINE'], django.db.connections.databases['default']['NAME'])
-
-        if dbname ==":memory:":
-            # just run, and save the sql file
         self.runqonce()
-            memdumpsql('memdump.sql') # saved contents of scratch db, could be imported later..
+        if settings.DATABASES['default']['NAME'] ==":memory:":
+            memdumpsql('memdump.sql') # saved contents of in-memory db, could be imported later..
         self.saveprofiles()
-        elif self.skip_memory_phase():
-            self.runqonce() 
-            self.saveprofiles()
-        else:
-            # run all the imports through :memory: first
-            set_in_memory_dbsettings()
-
-            print("-- phase 1 ",  settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
-
-            # the jobqueue may not start from scratch so we need to initialise the db properly first
-            # because we are using an empty :memory: database
-            # But initiating twice crashes it; so be sure to do it once only.
-
-            # Damn. migrate() is still calling MySQL somehow **conn_params not sqlite3. So crashes on expo server.
-            if ("reinit",reinit_db) not in self.queue:
-                reinit_db()
-            if ("caves",import_caves) not in self.queue:
-                import_caves() # sometime extract the initialising code from this and put in reinit...
-            if ("people",import_people) not in self.queue:
-                import_people() # sometime extract the initialising code from this and put in reinit...
-                
-            django.db.close_old_connections() # maybe not needed here
-            
-            self.runqonce() 
-            memdumpsql('memdump2.sql')
-            self.showprofile() 
-            
-            # restore the original db and import again
-            # if we wanted to, we could re-import the SQL generated in the first pass to be
-            # blazing fast. But for the present just re-import the lot.
-            restore_dbsettings()
-            print("-- phase 2 ",  settings.DATABASES['default']['ENGINE'], settings.DATABASES['default']['NAME'])
-            print(django.db.connections.databases['default']['NAME'])
-
-            django.db.close_old_connections() # maybe not needed here
-            for j in self.results_order:
-                self.results[j].pop() # throw away results from :memory: run
-            self.append_placeholders()
-
-            django.db.close_old_connections() 
-            #django.setup()  # should this be needed?
-            
-            self.runqonce() 
-            self.saveprofiles()
-    
         return True
 
     def showprofile(self):
@@ -407,15 +292,8 @@ def usage():
              and [runlabel] is an optional string identifying this run of the script
              in the stored profiling data 'import-profile.json'
              
-             if [runlabel] is absent or begins with "F-" then it will skip the :memory: pass
-             
              caves and logbooks must be run on an empty db before the others as they
              set up db tables used by the others.
-             
-             the commands are first run on an in-memory empty database before being run on
-             the actual persistent database. This is very fast and checks for import errors.
-             
-             the initial in-memory phase is on an empty db, so always runs caves & people for this phase
              """)
 
 if __name__ == "__main__":
@@ -479,11 +357,5 @@ if __name__ == "__main__":
         print("%s not recognised as a command." % sys.argv[1])
         exit()
 
-    store_dbsettings()
-    #set_in_memory_dbsettings()
-    
-    #set_in_memory_dbsettings() # seems to be ignored. Appears to be set but in reality.
-    #jq.run_now_django_tests(1) # actually does set db to :memory: - but invisibly !
-
     jq.run()
     jq.showprofile()

flatpages/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.29 on 2020-06-20 16:03
+# Generated by Django 1.11.29 on 2020-06-20 17:43
 from __future__ import unicode_literals
 
 from django.db import migrations, models

lines-of-python.txt

@@ -4,9 +4,11 @@
 6 ./core/views.py
 6 ./manage.py
 8 ./core/templatetags/csrffaker.py
+10 ./flatpages/models.py
+10 ./helper/__init__.py
+10 ./wsgi.py
 26 ./core/management/commands/reset_db.py
 26 ./export/toqms.py
-26 ./flatpages/migrations/0001_initial.py
 33 ./localsettingswindows.py
 37 ./localsettingsubuntu.py
 38 ./profiles/urls.py
@@ -14,6 +16,7 @@
 39 ./localsettings-expo-live.py
 39 ./localsettingsdocker.py
 39 ./localsettingsserver.py
+40 ./parsers/imports.py
 41 ./localsettingspotatohut.py
 41 ./middleware.py
 44 ./dump.py
@@ -22,6 +25,7 @@
 49 ./parsers/subcaves.py
 63 ./logbooksdump.py
 69 ./core/TESTS/tests.py
+70 ./urls.py
 73 ./localsettings.py
 73 ./localsettingsWSL.py
 81 ./settings.py
@@ -33,7 +37,9 @@
 103 ./core/view_surveys.py
 124 ./core/templatetags/wiki_markup.py
 135 ./utils.py
+150 ./parsers/surveys.py
 156 ./flatpages/views.py
+160 ./core/models_survex.py
 164 ./modelviz.py
 167 ./core/models.py
 175 ./core/views_other.py
@@ -41,10 +47,9 @@
 217 ./core/views_logbooks.py
 256 ./core/views_survex.py
 276 ./profiles/views.py
-359 ./databaseReset.py
+280 ./databaseReset.py
-382 ./troggle-inspectdb.py
 387 ./core/views_caves.py
 431 ./parsers/survex.py
-462 ./core/migrations/0001_initial.py
+450 ./core/models_caves.py
 515 ./parsers/logbooks.py
-5999
+5950

pre-run.sh

@@ -7,9 +7,9 @@ python manage.py inspectdb > troggle-inspectdb.py
 echo ""
 # count non-blank lines of python and template HTML code
 # includes all variants of settings.py files
-find . -name \*.html -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v "0$" | awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-templates.txt
+find . -name \*.html -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v ":0$" | awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-templates.txt
 
-find . -name \*.py -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v "0$" | awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-python.txt
+find . -name \*.py -print0 | xargs -0 egrep  -vc "#|^\s*$"  | grep -v ":0$" | grep -v "/migrations/" |grep -v "troggle-inspectdb.py"| awk -F ":" '{ sum +=$2; print $2, $1; } END {print sum}'| sort -n > lines-of-python.txt
 echo `tail -1 lines-of-python.txt`   non-comment lines of python.
 
 # This deletes the database so must run after generating troggle-inspectdb.py