xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues4 Pull Requests Projects1 Wiki Activity

faster db creation, safer file reading with 'with'

Browse Source
This commit is contained in:
Philip Sargent 2023-01-28 14:04:32 +00:00
parent d9a4069662
commit 2704fc42d4
1 changed files with 90 additions and 97 deletions

29
parsers/QMs.py

@ -1,6 +1,7 @@
import csv import csv
import os import os
import re import re
from pathlib import Path
from django.conf import settings from django.conf import settings
@ -65,9 +66,9 @@ def parseCaveQMs(cave, inputFile, ticked=False):
return nqms return nqms
# qmPath = settings.EXPOWEB+inputFile # qmPath = settings.EXPOWEB+inputFile
qmPath = os.path.join(settings.EXPOWEB, inputFile) # why not use the pathlib stuff ? qmPath = Path(settings.EXPOWEB, inputFile)
qmCSVContents = open(qmPath, "r") with open(qmPath, "r") as qmCSVContents:
dialect = csv.Sniffer().sniff(qmCSVContents.read()) dialect = csv.Sniffer().sniff(qmCSVContents.read())
qmCSVContents.seek(0, 0) qmCSVContents.seek(0, 0)
qmReader = csv.reader(qmCSVContents, dialect=dialect) qmReader = csv.reader(qmCSVContents, dialect=dialect)
@ -134,8 +135,8 @@ def parseCaveQMs(cave, inputFile, ticked=False):
def parse_KH_QMs(kh, inputFile, ticked): def parse_KH_QMs(kh, inputFile, ticked):
"""import QMs from the 1623-161 (Kaninchenhohle) html pages, different format""" """import QMs from the 1623-161 (Kaninchenhohle) html pages, different format"""
khQMs = open(os.path.join(settings.EXPOWEB, inputFile), "r") with open(os.path.join(settings.EXPOWEB, inputFile), "r") as khQMfile:
khQMs = khQMs.readlines() khQMs = khQMfile.readlines()
nqms = 0 nqms = 0
for line in khQMs: for line in khQMs:
res = re.search( res = re.search(
@ -145,14 +146,8 @@ def parse_KH_QMs(kh, inputFile, ticked):
if res: if res:
res = res.groupdict() res = res.groupdict()
year = int(res["year"]) year = int(res["year"])
# logbook placeholder code was previously here. No longer needed.
# check if placeholder exists for given year, create it if not lookupAttribs = {
# message = " ! - "+ str(year) + " logbook: placeholder entry for '161 KH' created. DUMMY EXPEDITION ID. Should be re-attached to the actual trip."
# placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="161", title="placeholder for QMs in 161", text=message, entry_type="DUMMY", expedition_id=1, defaults={"date": date((year), 1, 1),"cave_slug":str(kh)})
# # if hadToCreate:
# print(message)
# DataIssue.objects.create(parser='QMs', message=message)
lookupArgs = {
#'found_by':placeholder, #'found_by':placeholder,
"blockname": "", "blockname": "",
"expoyear": year, "expoyear": year,
@ -160,16 +155,14 @@ def parse_KH_QMs(kh, inputFile, ticked):
"cave": kh, "cave": kh,
"grade": res["grade"], "grade": res["grade"],
} }
nonLookupArgs = { nonLookupAttribs = {
"ticked": ticked, "ticked": ticked,
"nearest_station_name": res["nearest_station"], "nearest_station_name": res["nearest_station"],
"location_description": res["description"], "location_description": res["description"],
} }
instance, created = save_carefully(QM, lookupArgs, nonLookupArgs) # Create new. We know it doesn't exist as we deleted evrything when we started.
# if created: instance = QM.objects.create(**nonLookupAttribs, **lookupAttribs)
# message = f" ! - {instance.code()} QM entry for '161 KH' created. ticked: {ticked}"
# print(message)
# DataIssue.objects.create(parser='QMs', message=message)
nqms += 1 nqms += 1
return nqms return nqms