expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-25 16:51:54 +00:00
Code Issues 19 Projects 3 Activity

faster db creation, safer file reading with 'with'

Browse Source
This commit is contained in:
Philip Sargent 2023-01-28 14:04:32 +00:00
parent d9a4069662
commit 2704fc42d4
1 changed files with 90 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
parsers/QMs.py
View File

@ -1,6 +1,7 @@
import csv
import os
import re
from pathlib import Path
from django.conf import settings
@ -65,9 +66,9 @@ def parseCaveQMs(cave, inputFile, ticked=False):
return nqms
# qmPath = settings.EXPOWEB+inputFile
qmPath = os.path.join(settings.EXPOWEB, inputFile) # why not use the pathlib stuff ?
qmPath = Path(settings.EXPOWEB, inputFile)
qmCSVContents = open(qmPath, "r")
with open(qmPath, "r") as qmCSVContents:
dialect = csv.Sniffer().sniff(qmCSVContents.read())
qmCSVContents.seek(0, 0)
qmReader = csv.reader(qmCSVContents, dialect=dialect)
@ -134,8 +135,8 @@ def parseCaveQMs(cave, inputFile, ticked=False):
def parse_KH_QMs(kh, inputFile, ticked):
"""import QMs from the 1623-161 (Kaninchenhohle) html pages, different format"""
khQMs = open(os.path.join(settings.EXPOWEB, inputFile), "r")
khQMs = khQMs.readlines()
with open(os.path.join(settings.EXPOWEB, inputFile), "r") as khQMfile:
khQMs = khQMfile.readlines()
nqms = 0
for line in khQMs:
res = re.search(
@ -145,14 +146,8 @@ def parse_KH_QMs(kh, inputFile, ticked):
if res:
res = res.groupdict()
year = int(res["year"])
# logbook placeholder code was previously here. No longer needed.
# check if placeholder exists for given year, create it if not
# message = " ! - "+ str(year) + " logbook: placeholder entry for '161 KH' created. DUMMY EXPEDITION ID. Should be re-attached to the actual trip."
# placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="161", title="placeholder for QMs in 161", text=message, entry_type="DUMMY", expedition_id=1, defaults={"date": date((year), 1, 1),"cave_slug":str(kh)})
# # if hadToCreate:
# print(message)
# DataIssue.objects.create(parser='QMs', message=message)
lookupArgs = {
lookupAttribs = {
#'found_by':placeholder,
"blockname": "",
"expoyear": year,
@ -160,16 +155,14 @@ def parse_KH_QMs(kh, inputFile, ticked):
"cave": kh,
"grade": res["grade"],
}
nonLookupArgs = {
nonLookupAttribs = {
"ticked": ticked,
"nearest_station_name": res["nearest_station"],
"location_description": res["description"],
}
instance, created = save_carefully(QM, lookupArgs, nonLookupArgs)
# if created:
# message = f" ! - {instance.code()} QM entry for '161 KH' created. ticked: {ticked}"
# print(message)
# DataIssue.objects.create(parser='QMs', message=message)
# Create new. We know it doesn't exist as we deleted evrything when we started.
instance = QM.objects.create(**nonLookupAttribs, **lookupAttribs)
nqms += 1
return nqms