xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Merge branch 'master' of ssh://expo.survex.com/home/expo/troggle

Browse Source
This commit is contained in:
Martin Green 2023-07-10 17:00:56 +02:00
commit 67361fa66c
7 changed files with 64 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/admin.py
View File

@ -3,7 +3,7 @@ from django.core import serializers
from django.http import HttpResponse
from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance
from troggle.core.models.logbooks import QM, LogbookEntry, PersonLogEntry
from troggle.core.models.logbooks import QM, LogbookEntry, PersonLogEntry, CaveSlug
from troggle.core.models.survex import (
DrawingFile,
SingleScan,
@ -139,6 +139,7 @@ admin.site.register(Cave, CaveAdmin)
admin.site.register(Area)
admin.site.register(CaveAndEntrance)
admin.site.register(Entrance, EntranceAdmin)
admin.site.register(CaveSlug)
admin.site.register(SurvexBlock, SurvexBlockAdmin)
admin.site.register(DrawingFile, DrawingFileAdmin)
admin.site.register(Expedition)

14
core/models/logbooks.py
View File

@ -11,8 +11,15 @@ from troggle.core.models.troggle import Expedition, TroggleModel
"""
todo = """
- Can we rewrite things to eliminate the CaveSlug and objects? Surely
foreign keys work fine ?!
- Can we rewrite things to eliminate the CaveSlug and objects? No
Surely foreign keys work fine ?! No
Foreign keys do not allow for there being multiple ways to refer to a cave, eg 1623-1999-03 aka 1623-204
Having slugs allows for much more loose coupling to caves, which removes alot of the need to reset the database, which interupts work flow.
It also means we do not have to be creating temporary cave objects in the database, where we do not have the underlying file in cave_data.
To Do move Cave Slug back to troggle.core.models
"""
@ -25,6 +32,9 @@ class CaveSlug(models.Model):
slug = models.SlugField(max_length=50, unique=True)
primary = models.BooleanField(default=False)
def __str__(self):
return f"{self.slug}: {self.cave}"
class LogbookEntry(TroggleModel):
"""Single parsed entry from Logbook

2
core/views/editor_helpers.py
View File

@ -95,7 +95,7 @@ def new_image_form(request, path):
# Create directories if required
for full_path in image_path, thumb_path, desc_path:
print(full_path, full_path.parent)
full_path.parent.mkdir(parents=False, exist_ok=True)
full_path.parent.mkdir(parents=True, exist_ok=True)
try:
change_message = form.cleaned_data["change_message"]
write_and_commit(

10
parsers/logbooks.py
View File

@ -57,7 +57,7 @@ LOGBOOK_PARSER_SETTINGS = {
LOGBOOKS_DIR = "years" # subfolder of settings.EXPOWEB
ENTRIES = {
"2023": 1,
"2023": 11,
"2022": 90,
"2019": 55,
"2018": 95,
@ -295,7 +295,7 @@ def parser_html(year, expedition, txt, seq=""):
headmatch = re.match(r"(?i)(?s).*<body[^>]*>(.*?)<hr.*", txt)
headpara = headmatch.groups()[0].strip()
# print(f" - headpara:\n'{headpara}'")
#print(f" - headpara:\n'{headpara}'")
if len(headpara) > 0:
frontpath = Path(settings.EXPOWEB, LOGBOOKS_DIR, year, "frontmatter.html")
with open(frontpath, "w") as front:
@ -305,7 +305,7 @@ def parser_html(year, expedition, txt, seq=""):
endmatch = re.match(r"(?i)(?s).*<hr\s*/>([\s\S]*?)(?=</body)", txt)
endpara = endmatch.groups()[0].strip()
# print(f" - endpara:\n'{endpara}'")
#print(f" - endpara:\n'{endpara}'")
if len(endpara) > 0:
endpath = Path(settings.EXPOWEB, LOGBOOKS_DIR, year, "endmatter.html")
with open(endpath, "w") as end:
@ -333,7 +333,7 @@ def parser_html(year, expedition, txt, seq=""):
if s:
tripid, tripid1, tripdate, trippeople, triptitle, triptext, tu = s.groups()
else: # allow title and people to be swapped in order
msg = f" !- {year} Can't parse:{logbook_entry_count} '{trippara[:50]}'..."
msg = f" !- {year} Can't parse:{logbook_entry_count} '{trippara[:55]}'...'{trippara}'"
print(msg)
DataIssue.objects.create(parser="logbooks", message=msg)
@ -353,7 +353,7 @@ def parser_html(year, expedition, txt, seq=""):
tripid, tripid1, tripdate, triptitle, trippeople, triptext, tu = s2.groups()
else:
# if not re.search(r"Rigging Guide", trippara):
msg = f" !- Logbook. Can't parse entry on 2nd pass:{logbook_entry_count} '{trippara[:50]}'..."
msg = f" !- Logbook. Can't parse entry on 2nd pass:{logbook_entry_count} '{trippara[:55]}'...'{trippara}'"
print(msg)
DataIssue.objects.create(parser="logbooks", message=msg)
continue

46
parsers/survex.py
View File

@ -695,12 +695,15 @@ class LoadingSurvex:
)
oline = line
perps = get_people_on_trip(survexblock) # What, you don't know Judge Dredd slang ?
if len(line) > 10:
message = "! DATE Warning LONG DATE '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
print(self.insp+message)
stash_data_issue(parser='svxdate', message=message, url=None, sb=(survexblock.survexfile.path))
if line[10] == "-": # ie a range, just look at first date
line = line[0:10]
if len(line) == 10:
year = line[:4]
# TO DO set to correct Austrian timezone Europe/Vienna ?
@ -708,7 +711,6 @@ class LoadingSurvex:
survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d")
elif len(line) == 7:
year = line[:4]
perps = get_people_on_trip(survexblock) # What, you don't know Judge Dredd slang ?
message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
print(self.insp + message)
stash_data_issue(
@ -717,13 +719,36 @@ class LoadingSurvex:
survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m") # sets to first of month
elif len(line) == 4:
year = line[:4]
perps = get_people_on_trip(survexblock)
message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
)
survexblock.date = datetime.strptime(line, "%Y") # sets to January 1st
elif len(line) == 9 or len(line) == 8:
year = line[:4]
message = f"! DATE format WARNING, single digit day or month number,'{oline}' [{line[-5]}][{line[-2]}] ({survexblock}) {survexblock.survexfile.path}"
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
)
if line[-2] == "-" or line[-2] == ".":
line = line[:-1] + '0' + line[-1]
survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d")
print(f"! DATE -2 '{line}' '{survexblock.date}'")
elif line[-5] == "-" or line[-5] == ".":
line = line[:-4] + '0' + line[-4:]
survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d")
print(f"! DATE -5 '{line}' '{survexblock.date}'")
else:
year = line[:4]
message = (
f"! DATE Error SHORT LINE '{line}' '{oline}-{survexblock}' ({type(survexblock)}) {survexblock.survexfile.path}"
)
print(self.insp + message)
stash_data_issue(
parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
)
else:
# these errors are reporting the wrong survexblock, which is actually a SurvexFile (!)
# see To Do notes on how to trigger this. Still needs investigating..
@ -737,9 +762,16 @@ class LoadingSurvex:
print(f" {type(survexblock)=}") # survexblock.parent fails as a SurvexFile has no .parent ...ugh.
print(f" {survexblock.survexpath=}")
print(f" {survexblock.survexfile=}")
# Not setting 'year' crashes entire import on databaseReset.
year = line[:4]
perps = get_people_on_trip(survexblock)
# raise
setdate_on_survexblock(year)
try:
setdate_on_survexblock(year)
except NameError:
print(f">> why is year not set ?! {survexblock.survexfile.path}")
setdate_on_survexblock("1976")
if survexblock.date:
# do not actually need a distict variable 'currentdate' but it makes the code clearer
self.currentdate = survexblock.date
@ -1377,7 +1409,13 @@ class LoadingSurvex:
survexblock.survexfile.cave.slug()
self.fix_undated(survexblock) # null-op if already set
expoyear = str(survexblock.date.year)
try:
expoyear = str(survexblock.date.year)
except:
print(f">> why is survexblock not set ?! in LoadSurvexQM()/n {survexblock.survexfile.path}")
expoyear = "1970"
try:
qm = QM.objects.create(

2
templates/expedition.html
View File

@ -37,7 +37,7 @@ an "<b>S</b>" for a survey trip. The colours of the "<b>T</b>" and "<b>S</b>" a
<th>Caver</th>
{% for d in dates %}
<th>
{{d.day}}
{{d.day}}/{{d.month}}
</th>
{% endfor %}
</tr>

2
templates/menu.html
View File

@ -38,7 +38,7 @@
<ul><li><a href="/years/{{ year }}">{{ year }}</a></li></ul>
{% endif %}
<li><a href="/guidebook/areas.htm">Areas</a></li>
<li><a href="/expedition/2022">Troggle</a></li>
<li><a href="/expedition/2023">Troggle</a></li>
<li><form name=P method=get action="/search" target="_top">
<input id="omega-autofocus" type=search name=P size=8 autofocus>
<input type=submit value="Search"></form></li>