expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-25 08:41:51 +00:00
Code Issues 19 Projects 3 Activity

delete old forms, templates. fix logdataissues

Browse Source
This commit is contained in:
Philip Sargent 2021-04-23 11:43:25 +01:00
parent dbd186e299
commit 343d6cf350
8 changed files with 70 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
core/forms.py
View File

@ -15,8 +15,8 @@ from troggle.core.models.caves import Cave, LogbookEntry, QM, Entrance, CaveAndE
Some are not used and need renovating or destroying.
'''
todo = '''Fix UploadFileForm
delete TripForm once working
todo = '''Fix UploadFileForm - long list of actions
'''
class CaveForm(ModelForm):
@ -106,39 +106,6 @@ class EntranceLetterForm(ModelForm):
model = CaveAndEntrance
exclude = ('cave', 'entrance')
def getTripForm(expedition):
class TripForm(forms.Form):
date = forms.DateField()
title = forms.CharField(max_length=200)
caves = sorted([cave.reference() for cave in Cave.objects.all()])
caves = ["-----"] + caves
cave = forms.ChoiceField([(c, c) for c in caves], required=False)
location = forms.CharField(max_length=200, required=False)
caveOrLocation = forms.ChoiceField([("cave", "Cave"), ("location", "Location")], widget = forms.widgets.RadioSelect())
# html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30}))
html = forms.CharField(widget=forms.Textarea(attrs={"cols":80, "rows":20}))
def clean(self):
print((dir(self)))
if self.cleaned_data.get("caveOrLocation") == "cave" and not self.cleaned_data.get("cave"):
self._errors["cave"] = self.error_class(["This field is required"])
if self.cleaned_data.get("caveOrLocation") == "location" and not self.cleaned_data.get("location"):
self._errors["location"] = self.error_class(["This field is required"])
return self.cleaned_data
class PersonTripForm(forms.Form):
names = sorted([get_name(pe) for pe in PersonExpedition.objects.filter(expedition = expedition)])
names = ["-----"] + names
name = forms.ChoiceField([(n, n) for n in names])
TU = forms.FloatField(required=False)
author = forms.BooleanField(required=False, default=False)
PersonTripFormSet = formset_factory(PersonTripForm, extra=1)
return PersonTripFormSet, TripForm
def get_name(pe):
if pe.nickname:
return pe.nickname

4
core/views/logbooks.py
View File

@ -12,7 +12,6 @@ from django.template.defaultfilters import slugify
from django.utils import timezone
from django.views.generic.list import ListView
from troggle.core.forms import getTripForm # , get_name
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.utils import TROG
from troggle.core.models.caves import LogbookEntry, PersonTrip
@ -60,7 +59,8 @@ def expedition(request, expeditionname):
if request.user.is_authenticated:
if "reload" in request.GET:
this_expedition = Expedition.objects.get(year=int(expeditionname))
# Need to delete the exisitng entries or we get duplicaiton
# Need to delete the exisitng entries or we get duplication
# Need to delete both in the Django ORM and in our own object-store.
entries = this_expedition.logbookentry_set.all()
print(f'! - expo {expeditionname} {len(entries)} entries')
for entry in entries:

58
core/views/other.py
View File

@ -22,17 +22,19 @@ Also has code to download a logbook in a choice of formats (why?!) and to
download all QMs (not working)
'''
todo = '''Delete the newfile & TripForm code once we have a proper file-upload system working.
meanwhile keep it as an example to consider.
todo = '''
- Check that the logbookdownloader works by testing with a round trip.
Check that the logbookdownloader works by testing with a round trip.
Use it to convert all older logbooks into the 2005-variant of HTML then we can
- Use it to convert all older logbooks into the 2005-variant of HTML then we can
get rid of the parsers for older formats. There are no images stored in the database,
so this is only a tool for a first pass, to be followed by extensive hand-editing!
When we have done all the old logbooks, delete this function and the two templates.
But how does this interact with troggle/logbooksdump.py ?S
- But how does this interact with troggle/logbooksdump.py ?
- deleted nefile() - check on deleted UploadFileForm using the editfile.html template which is about re-submitting
a LBE aka TripReport
'''
def todos(request, module):
@ -174,50 +176,6 @@ def ajax_QM_number(request):
return HttpResponse(res)
@login_required_if_public
def newfile(request, pslug = None):
'''
If not POST, it goes straight to UploadFileForm using the editfile.html template which is about re-submitting
a LBE aka TripReport
If it is POST, then it is a new LBE so an HTML formatted version of it is produced, using template.render()
and a format 'dataformat/logbookentry.html' which is then put in expoweb/years<year>/autologbook/<LBEnamedfile>
'''
if pslug:
previousfile = LogbookEntry.objects.get(slug = pslug, date = previousdate, expedition = expedition)
if request.method == 'POST': # If the form has been submitted...
tripForm = TripForm(request.POST) # A form bound to the POST data
personTripFormSet = PersonTripFormSet(request.POST)
if tripForm.is_valid() and personTripFormSet.is_valid(): # All validation rules pass
dateStr = tripForm.cleaned_data["date"].strftime("%Y-%m-%d")
directory = os.path.join(settings.EXPOWEB,
"years",
expedition.year,
"autologbook")
filename = os.path.join(directory,
dateStr + "." + slugify(tripForm.cleaned_data["title"])[:50] + ".html")
if not os.path.isdir(directory):
os.mkdir(directory)
if pslug and pdate:
delLogbookEntry(previouslbe)
f = open(filename, "w")
template = loader.get_template('dataformat/logbookentry.html')
context = {'trip': tripForm.cleaned_data,
'persons': personTripFormSet.cleaned_data,
'date': dateStr,
'expeditionyear': expeditionyear}
f.write(template.render(context))
f.close()
print(logbookparsers.parseAutoLogBookEntry(filename))
return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST
else:
if pslug:
pass
else:
fileform = UploadFileForm() # An unbound form
return render(request, 'editfile.html', {'fileForm': fileform, })
@login_required_if_public
def scanupload(request, year='2050'):
print(f'! - FORM scanupload - start')

100
parsers/logbooks.py
View File

@ -58,10 +58,12 @@ trips ={}
#
# the logbook loading section
#
def GetTripPersons(trippeople, expedition, logtime_underground):
def GetTripPersons(trippeople, expedition, logtime_underground, tid="!"):
res = [ ]
author = None
round_bracket_regex = re.compile(r"[\(\[].*?[\)\]]")
if tid =="!":
tid = expedition.year + "." + tripperson
for tripperson in re.split(r",|\+|&amp;|&(?!\w+;)| and ", trippeople):
tripperson = tripperson.strip()
mul = re.match(r"<u>(.*?)</u>$(?i)", tripperson)
@ -71,10 +73,10 @@ def GetTripPersons(trippeople, expedition, logtime_underground):
tripperson = re.sub(round_bracket_regex, "", tripperson).strip()
personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower())
if not personyear:
message = "No name match for: ||'%s'|| in year '%s'" % (tripperson, expedition.year)
message = f" ! - {expedition.year} No name match for: '{tripperson}' "
print(message)
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[expedition.year + "~" + tripperson]=message
logdataissues[tid]=message
res.append((personyear, logtime_underground))
if mul:
author = personyear
@ -84,37 +86,18 @@ def GetTripPersons(trippeople, expedition, logtime_underground):
author = res[-1][0]
return res, author
def GetTripCave(place):
try:
katastNumRes=[]
katastNumRes=list(Cave.objects.filter(kataster_number=int(place)))
except ValueError:
message = " ! - ValueError on finding place " + str(place) + " entered. " + tripdate + " - " + year
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues["author"]=message
officialNameRes=list(Cave.objects.filter(official_name=place))
tripCaveRes=officialNameRes+katastNumRes
if len(tripCaveRes)==1:
return tripCaveRes[0]
elif len(tripCaveRes)>1:
message = " ! - Ambiguous place " + str(place) + " entered. " + tripdate + " - " + year + " " + str(tripCaveRes)
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues["author"]=message
return tripCaveRes[0]
else:
print((" " , place))
message = " ! - No cave found for place:" + str(place) + tripdate + " - " + year
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues["author"]=message
return None
def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki"):
def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki", tid="!"):
""" saves a logbook entry and related persontrips
Does NOT save the expeditionday_id - all NULLs. why?
"""
trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground)
try:
trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground, tid=tid)
except:
message = " ! - Skipping logentry: %s - GetTripPersons FAIL in year '%s'" % (title, expedition.year)
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues["title"]=message
return
if not author:
print(" ! - Skipping logentry: " + title + " - no author for entry")
message = " ! - Skipping logentry: %s - no author for entry in year '%s'" % (title, expedition.year)
@ -264,6 +247,7 @@ def Parseloghtmltxt(year, expedition, txt):
logbook_entry_count = 0
for trippara in tripparas:
logbook_entry_count += 1
tid= "n{}-s{:02d}".format(year,logbook_entry_count)
s = re.match(r'''(?x)(?:\s*<div\sclass="tripdate"\sid=".*?">.*?</div>\s*<p>)? # second date
\s*(?:<a\s+id="(.*?)"\s*/>\s*</a>)?
@ -298,7 +282,7 @@ def Parseloghtmltxt(year, expedition, txt):
EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext,
trippeople=trippeople, expedition=expedition, logtime_underground=0,
entry_type="html")
entry_type="html", tid=tid)
EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
"html", tripid1, logbook_entry_count)
@ -314,13 +298,13 @@ def Parseloghtml01(year, expedition, txt):
logbook_entry_count = 0
for trippara in tripparas:
logbook_entry_count += 1
tid= f"{year}.s{logbook_entry_count:02d}"
try:
tripentry = year + "." + str(logbook_entry_count)
s = re.match(r"(?s)\s*(?:<p>)?(.*?)</?p>(.*)$(?i)", trippara)
if not s:
message = " ! - Skipping logentry on failure to parse header: " + tripentry + trippara[:300] + "..."
message = " ! - Skipping logentry on failure to parse header: " + tid + trippara[:300] + "..."
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tripentry]=message
logdataissues[tid]=message
print(message)
break
tripheader, triptext = s.group(1), s.group(2)
@ -356,23 +340,35 @@ def Parseloghtml01(year, expedition, txt):
entrytuple = (ldate, tripcave, triptitle, ltriptext,
trippeople, expedition, tu, "html01", tripid)
logentries.append(entrytuple)
try:
EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext,
trippeople=trippeople, expedition=expedition, logtime_underground=0,
entry_type="html")
except:
message = " ! - Enter log entry into database FAIL exception in: " + tid
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
try:
EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu,
"html01", tripid, logbook_entry_count)
except:
message = " ! - Skipping logentry due to exception in: " + tripentry
message = " ! - Enter log entry into ObjectStore FAIL exception in: " + tid
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tripentry]=message
logdataissues[tid]=message
print(message)
except:
message = " ! - Skipping logentry due to exception in: " + tid
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tid]=message
print(message)
errorcount += 1
if errorcount >5 :
message = " !!- TOO MANY ERRORS - aborting logbook: " + year
message = f" !!- TOO MANY ERRORS - aborting at '{tid}' logbook: {year}"
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tripentry]=message
logdataissues[tid]=message
print(message)
return
@ -385,12 +381,13 @@ def Parseloghtml03(year, expedition, txt):
logbook_entry_count = 0
for trippara in tripparas:
logbook_entry_count += 1
tid= f"{year}.s{logbook_entry_count:02d}"
s = re.match(r"(?s)\s*<p>(.*?)</p>(.*)$", trippara)
if not ( s ) :
message = " ! - Skipping logentry on failure to parse Parseloghtml03: {} {} {}...".format(tripentry,s,trippara[:300])
message = " ! - Skipping logentry on failure to parse Parseloghtml03: {} {} {}...".format(tid,s,trippara[:300])
DataIssue.objects.create(parser='logbooks', message=message)
logdataissues[tripentry]=message
logdataissues[tid]=message
print(message)
break
@ -415,7 +412,6 @@ def Parseloghtml03(year, expedition, txt):
ltriptext = re.sub(r"<p>", "\n\n", ltriptext).strip()
ltriptext = re.sub(r"[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext)
tid= "n{}-s{:02d}".format(str(ldate),logbook_entry_count)
entrytuple = (ldate, tripcave, triptitle, ltriptext,
trippeople, expedition, tu, "html03", tid)
@ -477,21 +473,23 @@ def LoadLogbookForExpedition(expedition, expect):
def cleanerrors(year):
global logdataissues
print(f' - CLEAN {year} {len(logdataissues)} data issues in total')
dataissues = DataIssue.objects.filter(parser='logbooks')
for di in dataissues:
ph = "t" + year + "-"
ph = year
if re.search(ph, di.message) is not None:
print(f' - CLEANING dataissue {di.message}')
di.delete()
for te, content in logdataissues:
print(f' - CLEAN {year} {len(logdataissues)} {type(logdataissues)} data issues for this year')
dellist = []
for key, value in logdataissues.items():
# tripentry = year + "." + str(logbook_entry_count)
print(f' - CLEAN {te}')
if te.startswith(year + "."):
print(f' - CLEANING logdataissue {te}')
logdataissues.pop(te)
print(f' - CLEAN [{key}]')
if key.startswith(year + "."):
print(f' - CLEANING logdataissues [{key:12}]: value ')
dellist.append(key)
for i in dellist:
del logdataissues[i]
cleanerrors(expedition.year)

26
templates/dataformat/logbookentry.html
View File

@ -1,26 +0,0 @@
{% autoescape off %}
<!DOCTYPE html>
<html>
<head>
<style type="text/css">.author {text-decoration:underline}</style>
</head>
<body>
<H1>{{trip.title}}</H1>
<span class="date">{{date}}</span> - <span class="expeditionyear">{{expeditionyear}}</span>
{% if trip.caveOrLocation == "cave" %}
<span class="cave">{{trip.cave}}</span>
{% else %}
<span class="location">{{trip.location}}</span>
{% endif %}
{% for person in persons %}
<div class="person">
<span class="name{% if person.author %} author{% endif %}">{{person.name}}</span>
TU<span class="TU">{% if person.TU %}{{person.TU}}{% else %}0{% endif %}</span>hours
</div>
{% endfor %}
<div class="report">{{trip.html}}</div>
</body>
</html>
{% endautoescape %}

19
templates/editcave.html
View File

@ -1,19 +0,0 @@
{% extends "base.html" %}
{% load csrffaker %}
{% block title %}Logbook {{logbookentry.id}}{% endblock %}
{% block head %}
<link rel="stylesheet" href="{{ settings.JSLIB_URL }}jquery-ui/css/lightness/jquery-ui.css" type="text/css" media="all" />
<script src="{{ settings.JSLIB_URL }}jquery-ui/jquery-ui.min.js" type="text/javascript"></script>
<script src="{{ settings.MEDIA_URL }}js/jquery.formset.min.js" type="text/javascript"></script>
<!--<script src="{{ settings.TINY_MCE_MEDIA_URL }}tiny_mce.js" type="text/javascript"></script>-->
{{ tripForm.media }}
{% endblock %}
{% block content %}
<form action="" method="post">{% csrf_token %}
{{ form }}
<p><input type="submit" value="Submit" /></p>
</form>
{% endblock %}

84
templates/logbookentrynew-pending-deletion.html
View File

@ -1,84 +0,0 @@
{% extends "base.html" %}
<!-- logbookentry.html - this text visible because this template has been included -->
{% load csrffaker %}
{% block title %}Logbook {{logbookentry.id}}{% endblock %}
{% block head %}
<script>
$(function() {
$("#id_date").datepicker({dateFormat: "yy-mm-dd"});
$('.persontrips tbody tr').formset();
$(":radio[name*='caveOrLocation']").change(setLocationType);
$(setLocationType());
function setLocationType () {
$("#cave").hide();
$("#location").hide();
$("#" + $(":radio[name*='caveOrLocation']:checked")[0].value).show();
};
});
</script>
<link rel="stylesheet" href="{{ settings.JSLIB_URL }}jquery-ui/css/lightness/jquery-ui.css" type="text/css" media="all" />
<script src="{{ settings.JSLIB_URL }}jquery/jquery-ui.min.js" type="text/javascript"></script>
<script src="{{ settings.MEDIA_URL }}js/jquery.formset.min.js" type="text/javascript"></script>
<!--<script src="{{ settings.TINY_MCE_MEDIA_URL }}tiny_mce.js" type="text/javascript"></script>-->
{{ tripForm.media }}
{% endblock %}
{% block content %}
<form action="" method="post">{% csrf_token %}
{{ tripForm.non_field_errors }}
<div class="fieldWrapper">
{{ tripForm.title.errors }}
<label for="id_title">Title:</label>
{{ tripForm.title }}
</div>
<div class="fieldWrapper">
{{ tripForm.date.errors }}
<label for="id_date">Date:</label>
{{ tripForm.date }}
</div>
<div class="fieldWrapper">
{{ tripForm.caveOrLocation.errors }}
<label for="id_caveOrLocation">Location Type:</label>
{{ tripForm.caveOrLocation }}
</div>
<div class="fieldWrapper" id="cave">
{{ tripForm.cave.errors }}
<label for="id_cave">Cave:</label>
{{ tripForm.cave }}
</div>
<div class="fieldWrapper" id="location">
{{ tripForm.location.errors }}
<label for="id_location">Location:</label>
{{ tripForm.location }}
</div>
<table class="persontrips" border="0" cellpadding="0" cellspacing="0">
<tbody>
<tr>
<th>Person</th>
<th>TU /hours</th>
<th>Author</th>
<th></th>
</tr>
{% for form in personTripFormSet.forms %}
<tr>
<td>{{ form.name.errors }}{{ form.name }}</td>
<td>{{ form.TU.errors }}{{ form.TU }}</td>
<td>{{ form.author.errors }}{{ form.author }}</td>
<td></td>
{{ form.non_field_errors }}
</tr>
{% endfor %}
</tbody>
</table>
{{ personTripFormSet.management_form }}
<div class="fieldWrapper">
{{ tripForm.html.errors }}
<label for="id_date">Content:</label>
{{ tripForm.html }}
</div>
<p><input type="submit" value="Sumbit Trip Report" /></p>
</form>
{% endblock %}

3
urls.py
View File

@ -10,7 +10,7 @@ from django.urls import reverse, resolve
from troggle.core.views import caves, statistics, survex
from troggle.core.views.surveys import surveyscansingle, surveyscansfolder, surveyscansfolders, dwgdata, dwgfilesingle, dwgfileupload
from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, newfile, scanupload
from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, scanupload
from troggle.core.views.other import downloadlogbook, ajax_QM_number, downloadQMs
from troggle.core.views.caves import ent, cavepage
from troggle.core.views.logbooks import get_logbook_entries, logbookentry, logbookSearch
@ -94,7 +94,6 @@ trogglepatterns = [
# Logbook entries
re_path(r'^logbookentry/(?P<date>.*)/(?P<slug>.*)/?$', logbookentry,name="logbookentry"),
re_path(r'^newfile', newfile, name="newFile"), # oddly broken, needs investigating more
re_path(r'^logbooksearch/(.*)/?$', logbookSearch), # name 'search' not defined in views/logbooks.py
re_path(r'^logbook(?P<year>\d\d\d\d)\.(?P<extension>.*)/?$', downloadlogbook), # e.g. /logbook2019.html # working but old CSS in template
re_path(r'^logbook/?$', downloadlogbook, name="downloadlogbook"),