xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Much QM re-engineering

Browse Source
This commit is contained in:
Philip Sargent
2023-03-17 20:01:52 +00:00
parent de54576d11
commit 7345e3a328
10 changed files with 252 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
parsers/QMs.py
View File

@@ -65,7 +65,6 @@ def parseCaveQMs(cave, inputFile, ticked=False):
nqms = parse_KH_QMs(kh, inputFile=inputFile, ticked=ticked)
return nqms
# qmPath = settings.EXPOWEB+inputFile
qmPath = Path(settings.EXPOWEB, inputFile)
with open(qmPath, "r") as qmCSVContents:
@@ -76,15 +75,17 @@ def parseCaveQMs(cave, inputFile, ticked=False):
n = 0
nqms = 0
for line in qmReader:
#"Number","Grade","Area","Description","Page reference","Nearest survey station","Completion description","Comment"
try:
n += 1
year = int(line[0][1:5])
f"PH_{int(year)}_{int(n):02d}"
QMnum = re.match(r".*?-\d*?-X?(?P<numb>\d*)", line[0]).group("numb")
newQM = QM()
newQM = QM() # creates python object, does not touch db yet
# newQM.found_by=placeholder
newQM.number = QMnum
newQM.cave = caveid
newQM.expoyear = year
newQM.blockname = ""
if line[1] == "Dig":
newQM.grade = "D"
@@ -92,32 +93,28 @@ def parseCaveQMs(cave, inputFile, ticked=False):
newQM.grade = line[1]
newQM.area = line[2]
newQM.location_description = line[3]
newQM.page_ref = line[4]
# In the table, completion is indicated by the presence of a completion discription.
newQM.completion_description = line[4]
newQM.nearest_station_description = line[5]
newQM.nearest_station_name = line[5]
newQM.completion_description = line[6]
if newQM.completion_description:
newQM.ticked = True
else:
newQM.ticked = False
newQM.comment = line[6]
newQM.comment = line[7]
try:
# year and number are unique for a cave in CSV imports
preexistingQM = QM.objects.get(
number=QMnum, found_by__date__year=year
) # if we don't have this one in the DB, save it
if (
preexistingQM.new_since_parsing is False
): # if the pre-existing QM has not been modified, overwrite it - VERY OLD THING
number=QMnum, expoyear=year, cave=caveid,
)
if preexistingQM:
message = f" ! - {qmPath} PRE-EXISTING QM - should not exist ! {str(line)} "
print(message)
DataIssue.objects.create(parser="QMs", message=message)
preexistingQM.delete()
newQM.expoyear = year
newQM.save()
else: # otherwise, print that it was ignored
print((" - preserving " + str(preexistingQM) + ", which was edited in admin \r"))
newQM.save()
except QM.DoesNotExist: # if there is no pre-existing QM, save the new one
newQM.expoyear = year
newQM.save()
nqms += 1
except KeyError: # check on this one
@@ -130,6 +127,12 @@ def parseCaveQMs(cave, inputFile, ticked=False):
print(message)
DataIssue.objects.create(parser="QMs", message=message)
continue
except:
message = f" ! - {qmPath} UNKNOWN error {str(line)} "
print(message)
DataIssue.objects.create(parser="QMs", message=message)
raise
continue
return nqms
@@ -139,8 +142,9 @@ def parse_KH_QMs(kh, inputFile, ticked):
khQMs = khQMfile.readlines()
nqms = 0
for line in khQMs:
# <dt><a href="sibria.htm#qC1997-161-27" name="C1997-161-27">C1997-161-27</a> A<dd>Sib: pitch at end of Fuzzy Logic [Paradox Rift - continues] [sep.fuzzy.13]
res = re.search(
r"name=\"[CB](?P<year>\d*)-(?P<cave>\d*)-(?P<number>\d*).*</a> (?P<grade>[ABDCV])<dd>(?P<description>.*)\[(?P<nearest_station>.*)\]",
r"name=\"[CB](?P<year>\d*)-(?P<cave>\d*)-(?P<number>\d*).*</a> (?P<grade>[ABCDX])<dd>(?P<location_description>.*)\[(?P<nearest_station_name>.*)\]",
line,
)
if res:
@@ -157,8 +161,10 @@ def parse_KH_QMs(kh, inputFile, ticked):
}
nonLookupAttribs = {
"ticked": ticked,
"nearest_station_name": res["nearest_station"],
"location_description": res["description"],
"page_ref": "",
"completion_description": "",
"nearest_station_name": res["nearest_station_name"],
"location_description": res["location_description"],
}
# Create new. We know it doesn't exist as we deleted evrything when we started.
instance = QM.objects.create(**nonLookupAttribs, **lookupAttribs)
@@ -169,8 +175,11 @@ def parse_KH_QMs(kh, inputFile, ticked):
def Load_QMs():
deleteQMs()
n204 = parseCaveQMs(cave="204-steinBH", inputFile=r"1623/204/qm.csv")
n234 = parseCaveQMs(cave="234-Hauch", inputFile=r"1623/234/qm.csv")
#Number Grade Area Description Page reference Nearest station Completion description Comment
n204 = parseCaveQMs(cave="204-steinBH", inputFile=r"1623/204/qm-204.csv") # TAB separated values
#"Number","Grade","Area","Description","Page reference","Nearest survey station","Completion description","Comment"
n234 = parseCaveQMs(cave="234-Hauch", inputFile=r"1623/234/qm-234.csv") # COMMA separated values, with quotes.
n161 = parseCaveQMs(cave="161-KH", inputFile="1623/161/qmtodo.htm", ticked=False)
t161 = parseCaveQMs(cave="161-KH", inputFile="1623/161/qmdone.htm", ticked=True)
# parseCaveQMs(cave='balkonhoehle',inputFile=r"1623/264/qm.csv")