From 1b693da5ed7430d1d49327a05bbd376970e34b61 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Tue, 16 Jun 2020 16:07:36 +0100 Subject: [PATCH] break recursive import cycle --- core/TESTS/tests.py | 10 +++++-- core/views_other.py | 42 +++++++------------------- databaseReset.py | 72 +++++++-------------------------------------- parsers/imports.py | 62 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 94 deletions(-) create mode 100644 parsers/imports.py diff --git a/core/TESTS/tests.py b/core/TESTS/tests.py index c3ab815..c3388e6 100644 --- a/core/TESTS/tests.py +++ b/core/TESTS/tests.py @@ -52,15 +52,21 @@ class SimpleTest(SimpleTestCase): from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm from troggle.helper import login_required_if_public def test_import_parses_mix(self): - import troggle.parsers.survex - import troggle.parsers.caves import troggle.settings import troggle.flatpages.models import troggle.logbooksdump + import troggle.parsers.caves import troggle.parsers.people import troggle.parsers.surveys import troggle.parsers.logbooks import troggle.parsers.QMs + import troggle.parsers.survex + def test_import_imports(self): + from django.core import management + from django.db import connection, close_old_connections, connections + from django.contrib.auth.models import User + from django.http import HttpResponse + from django.core.urlresolvers import reverse __test__ = {"doctest": """ Another way to test that 1 + 1 is equal to 2. diff --git a/core/views_other.py b/core/views_other.py index 9fa34c0..ae88305 100644 --- a/core/views_other.py +++ b/core/views_other.py @@ -8,7 +8,7 @@ from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render from django.template import Context, loader -import databaseReset +import troggle.parsers.imports from troggle.core.models import Expedition, Person, PersonExpedition from troggle.core.models_caves import LogbookEntry, QM, Cave, PersonTrip from troggle.core.models_survex import SurvexLeg @@ -35,18 +35,16 @@ def controlPanel(request): jobs_completed=[] if request.method=='POST': if request.user.is_superuser: - # NONE of this works now that databaseReset has been so extensively rewritten - - #importlist is mostly here so that things happen in the correct order. - #http post data seems to come in an unpredictable order, so we do it this way. - importlist=['reinit_db', 'import_people', 'import_caves', 'import_logbooks', - 'import_survexblks', 'import_QMs', 'import_surveyscans', 'import_tunnelfiles'] - databaseReset.dirsredirect() - for item in importlist: - if item in request.POST: - print(("running"+ " databaseReset."+item+"()")) - exec("databaseReset."+item+"()") - jobs_completed.append(item) + # NONE of this works now that databaseReset (now parsers.imports) has been so extensively rewritten + reinit_db() + import_caves() + import_people() + import_surveyscans() + import_logbooks() + import_QMs() + import_tunnelfiles() + import_survexblks() + import_survexpos() else: if request.user.is_authenticated(): #The user is logged in, but is not a superuser. return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'error':'You must be a superuser to use that feature.'}) @@ -212,24 +210,6 @@ def newFile(request, pslug = None): else: if pslug: pass - # if previouslbe.cave: - # tripForm = TripForm(initial={"date": previousdate, - # "title": previouslbe.title, - # "cave": previouslbe.cave.reference(), - # "location": None, - # "caveOrLocation": "cave", - # "html": previouslbe.text}) - # else: - # tripForm = TripForm(initial={"date": previousdate, - # "title": previouslbe.title, - # "cave": None, - # "location": previouslbe.place, - # "caveOrLocation": "location", - # "html": previouslbe.text}) - # personTripFormSet = PersonTripFormSet(initial=[{"name": get_name(py.personexpedition), - # "TU": py.time_underground, - # "author": py.is_logbook_entry_author} - # for py in previouslbe.persontrip_set.all()]) else: fileform = UploadFileForm() # An unbound form diff --git a/databaseReset.py b/databaseReset.py index 16e4235..a41c010 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -8,26 +8,25 @@ import settings os.environ['PYTHONPATH'] = settings.PYTHON_PATH os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') +import django from django.core import management from django.db import connection, close_old_connections, connections from django.contrib.auth.models import User from django.http import HttpResponse from django.core.urlresolvers import reverse -print(" 1 settings on loading databaseReset.py") +print(" - settings on loading databaseReset.py") from troggle.core.models_caves import Cave, Entrance -import troggle.parsers.caves import troggle.flatpages.models import troggle.logbooksdump -import troggle.parsers.people -import troggle.parsers.surveys -import troggle.parsers.logbooks -import troggle.parsers.QMs +#import troggle.parsers.imports + +from troggle.parsers.imports import import_caves, import_people, import_surveyscans, \ + import_logbooks, import_QMs, import_survex, import_survexpos, import_tunnelfiles import troggle.core.models import troggle.core.models_survex -import django # NOTE databaseReset.py is *imported* by views_other.py as it is used in the control panel # presented there. @@ -149,52 +148,6 @@ def set_in_memory_dbsettings(): django.db.connections.databases['default']['ENGINE'] = 'django.db.backends.sqlite3' django.db.connections.databases['default']['NAME'] = ':memory:' - -def import_caves(): - print("Importing Caves to ",end="") - print(django.db.connections.databases['default']['NAME']) - troggle.parsers.caves.readcaves() - -def import_people(): - print("Importing People (folk.csv) to ",end="") - print(django.db.connections.databases['default']['NAME']) - troggle.parsers.people.LoadPersonsExpos() - -def import_surveyscans(): - print("Importing Survey Scans") - troggle.parsers.surveys.LoadListScans() - -def import_logbooks(): - print("Importing Logbooks") - troggle.parsers.logbooks.LoadLogbooks() - -def import_QMs(): - print("Importing QMs (old caves)") - troggle.parsers.QMs.Load_QMs() - -def import_survexblks(): - # when this import is moved to the top with the rest it all crashes horribly - import troggle.parsers.survex - print("Importing Survex Blocks") - troggle.parsers.survex.LoadAllSurvexBlocks() - -def import_survexpos(): - import troggle.parsers.survex - print("Importing Survex x/y/z Positions") - troggle.parsers.survex.LoadPos() - -def import_surveyimgs(): - """This appears to store data in unused objects. The code is kept - for future re-working to manage progress against notes, plans and elevs. - """ - #import troggle.parsers.surveys - print("NOT Importing survey images") - #troggle.parsers.surveys.parseSurveys(logfile=settings.LOGFILE) - -def import_tunnelfiles(): - print("Importing Tunnel files") - troggle.parsers.surveys.LoadTunnelFiles() - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # These functions moved to a different file - not used currently. #import logbooksdump @@ -221,8 +174,8 @@ class JobQueue(): self.results = {} self.results_order=[ "date","runlabel","reinit", "caves", "people", - "logbooks", "QMs", "scans", "survexblks", "survexpos", - "tunnel", "surveyimgs", "test" ] + "logbooks", "QMs", "scans", "survex", + "tunnel", "test" ] for k in self.results_order: self.results[k]=[] self.tfile = "import_profile.json" @@ -432,9 +385,7 @@ def usage(): logbooks - read in the logbooks QMs - read in the QM csv files (older caves only) scans - the survey scans in all the wallets (must run before survex) - survex - read in the survex files - all the survex blocks but not the x/y/z positions - survexpos - set the x/y/z positions for entrances and fixed points - + survex - read in the survex files - all the survex blocks and entrances x/y/z tunnel - read in the Tunnel files - which scans the survey scans too autologbooks - Not used. read in autologbooks (what are these?) @@ -493,13 +444,12 @@ if __name__ == "__main__": jq.enq("logbooks",import_logbooks) jq.enq("QMs",import_QMs) jq.enq("tunnel",import_tunnelfiles) - jq.enq("survexblks",import_survexblks) + jq.enq("survex",import_survex) jq.enq("survexpos",import_survexpos) elif "scans" in sys.argv: jq.enq("scans",import_surveyscans) elif "survex" in sys.argv: - jq.enq("survexblks",import_survexblks) - elif "survexpos" in sys.argv: + jq.enq("survex",import_survex) jq.enq("survexpos",import_survexpos) elif "tunnel" in sys.argv: jq.enq("tunnel",import_tunnelfiles) diff --git a/parsers/imports.py b/parsers/imports.py new file mode 100644 index 0000000..65228f8 --- /dev/null +++ b/parsers/imports.py @@ -0,0 +1,62 @@ +import sys +import os + +import django +from django.core import management +from django.db import connection, close_old_connections, connections +from django.contrib.auth.models import User +from django.http import HttpResponse +from django.core.urlresolvers import reverse + +import troggle.settings +import troggle.parsers.caves +import troggle.parsers.people +import troggle.parsers.surveys +import troggle.parsers.logbooks +import troggle.parsers.QMs + +def import_caves(): + print("Importing Caves to ",end="") + print(django.db.connections.databases['default']['NAME']) + troggle.parsers.caves.readcaves() + +def import_people(): + print("Importing People (folk.csv) to ",end="") + print(django.db.connections.databases['default']['NAME']) + troggle.parsers.people.LoadPersonsExpos() + +def import_surveyscans(): + print("Importing Survey Scans") + troggle.parsers.surveys.LoadListScans() + +def import_logbooks(): + print("Importing Logbooks") + troggle.parsers.logbooks.LoadLogbooks() + +def import_QMs(): + print("Importing QMs (old caves)") + troggle.parsers.QMs.Load_QMs() + +def import_survex(): + # when this import is moved to the top with the rest it all crashes horribly + import troggle.parsers.survex + print("Importing Survex Blocks") + troggle.parsers.survex.LoadAllSurvexBlocks() + +def import_survexpos(): + import troggle.parsers.survex + print("Importing Survex x/y/z Positions") + troggle.parsers.survex.LoadPos() + +def import_surveyimgs(): + """This appears to store data in unused objects. The code is kept + for future re-working to manage progress against notes, plans and elevs. + """ + #import troggle.parsers.surveys + print("NOT Importing survey images") + #troggle.parsers.surveys.parseSurveys(logfile=settings.LOGFILE) + +def import_tunnelfiles(): + print("Importing Tunnel files") + troggle.parsers.surveys.LoadTunnelFiles() +