xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

More compact parsing report to terminal

Browse Source
This commit is contained in:
Philip Sargent 2023-07-24 13:14:42 +03:00
parent 9473b22bd9
commit 9f2b77bf1d
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
parsers/logbooks.py
View File

@ -57,7 +57,7 @@ LOGBOOK_PARSER_SETTINGS = {
LOGBOOKS_DIR = "years" # subfolder of settings.EXPOWEB LOGBOOKS_DIR = "years" # subfolder of settings.EXPOWEB
ENTRIES = { ENTRIES = {
"2023": 27, "2023": 32,
"2022": 90, "2022": 90,
"2019": 55, "2019": 55,
"2018": 95, "2018": 95,
@ -305,8 +305,8 @@ def parser_html(year, expedition, txt, seq=""):
endmatch = re.match(r"(?i)(?s).*<hr\s*/>([\s\S]*?)(?=</body)", txt) endmatch = re.match(r"(?i)(?s).*<hr\s*/>([\s\S]*?)(?=</body)", txt)
endpara = endmatch.groups()[0].strip() endpara = endmatch.groups()[0].strip()
#print(f" - endpara:\n'{endpara}'")
if len(endpara) > 0: if len(endpara) > 0:
print(f"\n - {year} endpara:\n'{endpara}'")
endpath = Path(settings.EXPOWEB, LOGBOOKS_DIR, year, "endmatter.html") endpath = Path(settings.EXPOWEB, LOGBOOKS_DIR, year, "endmatter.html")
with open(endpath, "w") as end: with open(endpath, "w") as end:
end.write(endpara + "\n") end.write(endpara + "\n")
@ -560,7 +560,8 @@ def parse_logbook_for_expedition(expedition, blog=False):
if logbook_parseable: if logbook_parseable:
# -------------------- # --------------------
parser = globals()[parsefunc] parser = globals()[parsefunc]
print(f" - {year} parsing with {parsefunc} - {lb}") # print(f" - {year} parsing with {parsefunc} - {lb}")
print(" .", end="")
logentries = parser(year, expedition, txt, sq) # this launches the right parser logentries = parser(year, expedition, txt, sq) # this launches the right parser
# -------------------- # --------------------
@ -665,7 +666,7 @@ def LoadLogbooks():
logentries = parse_logbook_for_expedition(b, blog=True) # loads the blog logbook for one expo logentries = parse_logbook_for_expedition(b, blog=True) # loads the blog logbook for one expo
allentries += logentries allentries += logentries
print(f"total {len(allentries):,} log entries parsed in all expeditions") print(f"\ntotal {len(allentries):,} log entries parsed in all expeditions")
mem = get_process_memory() mem = get_process_memory()
print(f" - MEM: {mem:7.2f} MB in use, {mem-mem1:7.2f} MB more", file=sys.stderr) print(f" - MEM: {mem:7.2f} MB in use, {mem-mem1:7.2f} MB more", file=sys.stderr)
duration = time.time() - start duration = time.time() - start
@ -682,6 +683,7 @@ def LoadLogbooks():
for expo in expos: for expo in expos:
expo.save() # to save logbook name property expo.save() # to save logbook name property
mem = get_process_memory() mem = get_process_memory()
print(f"total {len(allentries):,} log entries saved into database")
print(f" - MEM: {mem:7.2f} MB in use, {mem-mem1:7.2f} MB more", file=sys.stderr) print(f" - MEM: {mem:7.2f} MB in use, {mem-mem1:7.2f} MB more", file=sys.stderr)
duration = time.time() - start duration = time.time() - start
print(f" - TIME: {duration:7.2f} s", file=sys.stderr) print(f" - TIME: {duration:7.2f} s", file=sys.stderr)