From 6565b3f9c4d93c72f2fd0e7d49b24adaf9f94b9c Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Fri, 27 Jan 2023 17:41:10 +0000 Subject: [PATCH] refactored, faster deletion prior to loading --- core/views/logbooks.py | 2 +- parsers/logbooks.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/views/logbooks.py b/core/views/logbooks.py index 21cc849..415c7f8 100644 --- a/core/views/logbooks.py +++ b/core/views/logbooks.py @@ -18,7 +18,7 @@ from troggle.core.models.caves import LogbookEntry, PersonTrip from troggle.core.models.survex import SurvexBlock, Wallet from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.utils import TROG -from troggle.parsers.logbooks import LoadLogbookForExpedition +from troggle.parsers.logbooks import parse_logbook_for_expedition from troggle.parsers.people import GetPersonExpeditionNameLookup from .auth import login_required_if_public diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 838c253..86016af 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -480,6 +480,10 @@ def parser_blog(year, expedition, txt, sq=""): entrytuple = (tripdate, location, tripname, tripcontent, trippeople, expedition, tu, tid) logentries.append(entrytuple) +def clean_all_logbooks(): + DataIssue.objects.filter(parser="logbooks").delete() + LogbookEntry.objects.all().delete() + def clean_logbook_for_expedition(expedition): def cleanerrors(year): dataissues = DataIssue.objects.filter(parser="logbooks") @@ -576,12 +580,12 @@ def parse_logbook_for_expedition(expedition): def LoadLogbook(year): """One off logbook for testing purposes""" global LOGBOOK_PARSER_SETTINGS - nlbe = {} expo = Expedition.objects.get(year=year) year = expo.year # some type funny clean_logbook_for_expedition(expo) + nlbe[expo] = parse_logbook_for_expedition(expo) # this actually loads the logbook for one expo if year in BLOG_PARSER_SETTINGS: print("BLOG parsing") @@ -600,7 +604,7 @@ def LoadLogbooks(): """ global entries - DataIssue.objects.filter(parser="logbooks").delete() + clean_all_logbooks() expos = Expedition.objects.all() if len(expos) <= 1: message = " ! - No expeditions found. Load 'people' first" @@ -648,7 +652,7 @@ def LoadLogbooks(): orig = (DEFAULT_LOGBOOK_FILE, DEFAULT_LOGBOOK_PARSER) LOGBOOK_PARSER_SETTINGS[str(b)] = BLOG_PARSER_SETTINGS[str(b)] print(f" - BLOG: {b}") - nlbe[b] = parse_logbook_for_expedition(b, clean=False) # this loads the blog logbook for one expo + nlbe[b] = parse_logbook_for_expedition(b) # no clean. loads the blog logbook for one expo LOGBOOK_PARSER_SETTINGS[str(b)] = orig # tried to use map with concurrent threads - but sqlite database is not concurrent, so failed with database lock