diff --git a/parsers/QMs.py b/parsers/QMs.py index 12028eb..efeeb19 100644 --- a/parsers/QMs.py +++ b/parsers/QMs.py @@ -5,8 +5,8 @@ from datetime import date from django.conf import settings -from troggle.core.models.troggle import DataIssue from troggle.core.models.caves import QM, Cave, LogbookEntry +from troggle.core.models.troggle import DataIssue from troggle.core.utils import save_carefully '''Reads the CSV files containg QMs for a select few caves diff --git a/parsers/caves.py b/parsers/caves.py index 71e19a5..8e1993a 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -6,9 +6,11 @@ from pathlib import Path from django.conf import settings from django.db import transaction -from troggle.settings import SURVEX_DATA, EXPOWEB, CAVEDESCRIPTIONS, ENTRANCEDESCRIPTIONS +from troggle.core.models.caves import (Area, Cave, CaveAndEntrance, CaveSlug, + Entrance, EntranceSlug, GetCaveLookup) from troggle.core.models.troggle import DataIssue -from troggle.core.models.caves import Area, Cave, Entrance, CaveSlug, EntranceSlug, CaveAndEntrance, GetCaveLookup +from troggle.settings import (CAVEDESCRIPTIONS, ENTRANCEDESCRIPTIONS, EXPOWEB, + SURVEX_DATA) '''Reads all the cave description data by parsing the xml files (stored as e.g. :EXPOWEB:/cave_data/1623-161.html ) and creating the various Cave, Entrance and necessary Area objects. diff --git a/parsers/drawings.py b/parsers/drawings.py index a688771..e3630b0 100644 --- a/parsers/drawings.py +++ b/parsers/drawings.py @@ -1,16 +1,15 @@ -import sys -import os -import types -import stat import csv -import re import datetime +import os +import re +import stat +import sys +import types +from functools import reduce from pathlib import Path -from functools import reduce - import settings -from troggle.core.models.survex import SingleScan, Wallet, DrawingFile +from troggle.core.models.survex import DrawingFile, SingleScan, Wallet from troggle.core.models.troggle import DataIssue from troggle.core.utils import save_carefully diff --git a/parsers/imports.py b/parsers/imports.py index 2ac4f1d..d8fa87d 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -1,20 +1,20 @@ -import sys import os +import sys import django -from django.core import management -from django.db import connection, close_old_connections, connections from django.contrib.auth.models import User +from django.core import management +from django.db import (close_old_connections, connection, connections, + transaction) from django.http import HttpResponse -from django.db import transaction -import troggle.settings import troggle.parsers.caves -import troggle.parsers.people import troggle.parsers.drawings import troggle.parsers.logbooks +import troggle.parsers.people import troggle.parsers.QMs -import troggle.parsers.scans +import troggle.parsers.scans +import troggle.settings '''Master data import. Used only by databaseReset.py and online controlpanel. diff --git a/parsers/logbooks.py b/parsers/logbooks.py index a6d94f4..df5b9d0 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -2,18 +2,19 @@ import csv import os import re import time -from random import randint -from datetime import datetime, date +from datetime import date, datetime from pathlib import Path +from random import randint from django.conf import settings from django.template.defaultfilters import slugify from django.utils.timezone import get_current_timezone, make_aware +from parsers.people import GetPersonExpeditionNameLookup +from troggle.core.models.caves import (Cave, GetCaveLookup, LogbookEntry, + PersonTrip) from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.utils import TROG, save_carefully -from troggle.core.models.caves import Cave, LogbookEntry, PersonTrip, GetCaveLookup -from parsers.people import GetPersonExpeditionNameLookup ''' Parses and imports logbooks in all their wonderful confusion diff --git a/parsers/people.py b/parsers/people.py index 0052f7e..bfacc2a 100644 --- a/parsers/people.py +++ b/parsers/people.py @@ -1,13 +1,17 @@ -import csv, re, datetime, os, shutil +import csv +import datetime +import os +import re +import shutil from html import unescape -from unidecode import unidecode from pathlib import Path from django.conf import settings +from unidecode import unidecode -from troggle.core.models.troggle import Expedition, Person, PersonExpedition -from troggle.core.models.troggle import DataIssue -from troggle.core.utils import save_carefully, TROG +from troggle.core.models.troggle import (DataIssue, Expedition, Person, + PersonExpedition) +from troggle.core.utils import TROG, save_carefully '''These functions do not match how the stand-alone folk script works. So the script produces an HTML file which has href links to pages in troggle which troggle does not think are right. diff --git a/parsers/scans.py b/parsers/scans.py index a3c4cdc..1bebe18 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -1,18 +1,18 @@ -import sys -import os -import subprocess -import types -import stat import csv -import re import datetime -import shutil, filecmp - +import filecmp +import os +import re +import shutil +import stat +import subprocess +import sys +import types from functools import reduce from pathlib import Path import settings -from troggle.core.models.survex import SingleScan, Wallet, DrawingFile +from troggle.core.models.survex import DrawingFile, SingleScan, Wallet from troggle.core.models.troggle import DataIssue from troggle.core.utils import save_carefully from troggle.core.views.scans import datewallet diff --git a/parsers/survex.py b/parsers/survex.py index b35292d..c1c6c72 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1,23 +1,24 @@ -import sys +import copy import os import re -import time -import copy import subprocess - +import sys +import time +from datetime import date, datetime, timedelta, timezone from pathlib import Path -from datetime import datetime, timedelta, date, timezone -from django.utils.timezone import get_current_timezone -from django.utils.timezone import make_aware +from django.utils.timezone import get_current_timezone, make_aware import troggle.settings as settings -from troggle.core.models.caves import Cave, Entrance, QM, LogbookEntry -from troggle.core.utils import get_process_memory, chaosmonkey -from troggle.parsers.people import GetPersonExpeditionNameLookup, known_foreigner -from troggle.parsers.logbooks import GetCaveLookup +from troggle.core.models.caves import QM, Cave, Entrance, LogbookEntry +from troggle.core.models.survex import (SurvexBlock, SurvexDirectory, + SurvexFile, SurvexPersonRole, + SurvexStation, Wallet) from troggle.core.models.troggle import DataIssue, Expedition -from troggle.core.models.survex import SurvexPersonRole, Wallet, SurvexDirectory, SurvexFile, SurvexBlock, SurvexStation +from troggle.core.utils import chaosmonkey, get_process_memory +from troggle.parsers.logbooks import GetCaveLookup +from troggle.parsers.people import (GetPersonExpeditionNameLookup, + known_foreigner) '''Imports the tree of survex files following from a defined root .svx file It also scans the Loser repo for all the svx files, which it loads individually afterwards. @@ -1602,7 +1603,8 @@ def FindAndLoadSurvex(survexblockroot): fcollate.write(f";*include {survexfileroot.path}\n") flinear.write(f"{svx_scan.depthinclude:2} {indent} *include {survexfileroot.path}\n") - import cProfile, pstats + import cProfile + import pstats from pstats import SortKey pr = cProfile.Profile() pr.enable()