From 343d6cf35060c580a9e7cd91605bf481c4f6444b Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Fri, 23 Apr 2021 11:43:25 +0100 Subject: [PATCH] delete old forms, templates. fix logdataissues --- core/forms.py | 37 +----- core/views/logbooks.py | 4 +- core/views/other.py | 62 ++-------- parsers/logbooks.py | 112 +++++++++--------- templates/dataformat/logbookentry.html | 26 ---- templates/editcave.html | 19 --- .../logbookentrynew-pending-deletion.html | 84 ------------- urls.py | 3 +- 8 files changed, 70 insertions(+), 277 deletions(-) delete mode 100644 templates/dataformat/logbookentry.html delete mode 100644 templates/editcave.html delete mode 100644 templates/logbookentrynew-pending-deletion.html diff --git a/core/forms.py b/core/forms.py index 1f0d3e0..ff75872 100644 --- a/core/forms.py +++ b/core/forms.py @@ -15,8 +15,8 @@ from troggle.core.models.caves import Cave, LogbookEntry, QM, Entrance, CaveAndE Some are not used and need renovating or destroying. ''' -todo = '''Fix UploadFileForm -delete TripForm once working +todo = '''Fix UploadFileForm - long list of actions + ''' class CaveForm(ModelForm): @@ -106,39 +106,6 @@ class EntranceLetterForm(ModelForm): model = CaveAndEntrance exclude = ('cave', 'entrance') - -def getTripForm(expedition): - - class TripForm(forms.Form): - date = forms.DateField() - title = forms.CharField(max_length=200) - caves = sorted([cave.reference() for cave in Cave.objects.all()]) - caves = ["-----"] + caves - cave = forms.ChoiceField([(c, c) for c in caves], required=False) - location = forms.CharField(max_length=200, required=False) - caveOrLocation = forms.ChoiceField([("cave", "Cave"), ("location", "Location")], widget = forms.widgets.RadioSelect()) -# html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30})) - html = forms.CharField(widget=forms.Textarea(attrs={"cols":80, "rows":20})) - - def clean(self): - print((dir(self))) - if self.cleaned_data.get("caveOrLocation") == "cave" and not self.cleaned_data.get("cave"): - self._errors["cave"] = self.error_class(["This field is required"]) - if self.cleaned_data.get("caveOrLocation") == "location" and not self.cleaned_data.get("location"): - self._errors["location"] = self.error_class(["This field is required"]) - return self.cleaned_data - - class PersonTripForm(forms.Form): - names = sorted([get_name(pe) for pe in PersonExpedition.objects.filter(expedition = expedition)]) - names = ["-----"] + names - name = forms.ChoiceField([(n, n) for n in names]) - TU = forms.FloatField(required=False) - author = forms.BooleanField(required=False, default=False) - - PersonTripFormSet = formset_factory(PersonTripForm, extra=1) - - return PersonTripFormSet, TripForm - def get_name(pe): if pe.nickname: return pe.nickname diff --git a/core/views/logbooks.py b/core/views/logbooks.py index 454483c..cb32bab 100644 --- a/core/views/logbooks.py +++ b/core/views/logbooks.py @@ -12,7 +12,6 @@ from django.template.defaultfilters import slugify from django.utils import timezone from django.views.generic.list import ListView -from troggle.core.forms import getTripForm # , get_name from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.utils import TROG from troggle.core.models.caves import LogbookEntry, PersonTrip @@ -60,7 +59,8 @@ def expedition(request, expeditionname): if request.user.is_authenticated: if "reload" in request.GET: this_expedition = Expedition.objects.get(year=int(expeditionname)) - # Need to delete the exisitng entries or we get duplicaiton + # Need to delete the exisitng entries or we get duplication + # Need to delete both in the Django ORM and in our own object-store. entries = this_expedition.logbookentry_set.all() print(f'! - expo {expeditionname} {len(entries)} entries') for entry in entries: diff --git a/core/views/other.py b/core/views/other.py index 9a12b1d..45d781c 100644 --- a/core/views/other.py +++ b/core/views/other.py @@ -22,17 +22,19 @@ Also has code to download a logbook in a choice of formats (why?!) and to download all QMs (not working) ''' -todo = '''Delete the newfile & TripForm code once we have a proper file-upload system working. -meanwhile keep it as an example to consider. +todo = ''' +- Check that the logbookdownloader works by testing with a round trip. -Check that the logbookdownloader works by testing with a round trip. +- Use it to convert all older logbooks into the 2005-variant of HTML then we can + get rid of the parsers for older formats. There are no images stored in the database, + so this is only a tool for a first pass, to be followed by extensive hand-editing! + When we have done all the old logbooks, delete this function and the two templates. -Use it to convert all older logbooks into the 2005-variant of HTML then we can -get rid of the parsers for older formats. There are no images stored in the database, -so this is only a tool for a first pass, to be followed by extensive hand-editing! -When we have done all the old logbooks, delete this function and the two templates. +- But how does this interact with troggle/logbooksdump.py ? + +- deleted nefile() - check on deleted UploadFileForm using the editfile.html template which is about re-submitting + a LBE aka TripReport -But how does this interact with troggle/logbooksdump.py ?S ''' def todos(request, module): @@ -174,50 +176,6 @@ def ajax_QM_number(request): return HttpResponse(res) -@login_required_if_public -def newfile(request, pslug = None): - ''' - If not POST, it goes straight to UploadFileForm using the editfile.html template which is about re-submitting - a LBE aka TripReport - - If it is POST, then it is a new LBE so an HTML formatted version of it is produced, using template.render() - and a format 'dataformat/logbookentry.html' which is then put in expoweb/years/autologbook/ - ''' - if pslug: - previousfile = LogbookEntry.objects.get(slug = pslug, date = previousdate, expedition = expedition) - if request.method == 'POST': # If the form has been submitted... - tripForm = TripForm(request.POST) # A form bound to the POST data - personTripFormSet = PersonTripFormSet(request.POST) - if tripForm.is_valid() and personTripFormSet.is_valid(): # All validation rules pass - dateStr = tripForm.cleaned_data["date"].strftime("%Y-%m-%d") - directory = os.path.join(settings.EXPOWEB, - "years", - expedition.year, - "autologbook") - filename = os.path.join(directory, - dateStr + "." + slugify(tripForm.cleaned_data["title"])[:50] + ".html") - if not os.path.isdir(directory): - os.mkdir(directory) - if pslug and pdate: - delLogbookEntry(previouslbe) - f = open(filename, "w") - template = loader.get_template('dataformat/logbookentry.html') - context = {'trip': tripForm.cleaned_data, - 'persons': personTripFormSet.cleaned_data, - 'date': dateStr, - 'expeditionyear': expeditionyear} - f.write(template.render(context)) - f.close() - print(logbookparsers.parseAutoLogBookEntry(filename)) - return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST - else: - if pslug: - pass - else: - fileform = UploadFileForm() # An unbound form - - return render(request, 'editfile.html', {'fileForm': fileform, }) - @login_required_if_public def scanupload(request, year='2050'): print(f'! - FORM scanupload - start') diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 46aba96..d615930 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -58,10 +58,12 @@ trips ={} # # the logbook loading section # -def GetTripPersons(trippeople, expedition, logtime_underground): +def GetTripPersons(trippeople, expedition, logtime_underground, tid="!"): res = [ ] author = None round_bracket_regex = re.compile(r"[\(\[].*?[\)\]]") + if tid =="!": + tid = expedition.year + "." + tripperson for tripperson in re.split(r",|\+|&|&(?!\w+;)| and ", trippeople): tripperson = tripperson.strip() mul = re.match(r"(.*?)$(?i)", tripperson) @@ -71,10 +73,10 @@ def GetTripPersons(trippeople, expedition, logtime_underground): tripperson = re.sub(round_bracket_regex, "", tripperson).strip() personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower()) if not personyear: - message = "No name match for: ||'%s'|| in year '%s'" % (tripperson, expedition.year) + message = f" ! - {expedition.year} No name match for: '{tripperson}' " print(message) DataIssue.objects.create(parser='logbooks', message=message) - logdataissues[expedition.year + "~" + tripperson]=message + logdataissues[tid]=message res.append((personyear, logtime_underground)) if mul: author = personyear @@ -84,37 +86,18 @@ def GetTripPersons(trippeople, expedition, logtime_underground): author = res[-1][0] return res, author -def GetTripCave(place): - try: - katastNumRes=[] - katastNumRes=list(Cave.objects.filter(kataster_number=int(place))) - except ValueError: - message = " ! - ValueError on finding place " + str(place) + " entered. " + tripdate + " - " + year - DataIssue.objects.create(parser='logbooks', message=message) - logdataissues["author"]=message - officialNameRes=list(Cave.objects.filter(official_name=place)) - tripCaveRes=officialNameRes+katastNumRes - - if len(tripCaveRes)==1: - return tripCaveRes[0] - elif len(tripCaveRes)>1: - message = " ! - Ambiguous place " + str(place) + " entered. " + tripdate + " - " + year + " " + str(tripCaveRes) - DataIssue.objects.create(parser='logbooks', message=message) - logdataissues["author"]=message - return tripCaveRes[0] - else: - print((" " , place)) - message = " ! - No cave found for place:" + str(place) + tripdate + " - " + year - DataIssue.objects.create(parser='logbooks', message=message) - logdataissues["author"]=message - return None - - -def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki"): +def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki", tid="!"): """ saves a logbook entry and related persontrips Does NOT save the expeditionday_id - all NULLs. why? """ - trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground) + try: + trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground, tid=tid) + except: + message = " ! - Skipping logentry: %s - GetTripPersons FAIL in year '%s'" % (title, expedition.year) + DataIssue.objects.create(parser='logbooks', message=message) + logdataissues["title"]=message + return + if not author: print(" ! - Skipping logentry: " + title + " - no author for entry") message = " ! - Skipping logentry: %s - no author for entry in year '%s'" % (title, expedition.year) @@ -264,7 +247,8 @@ def Parseloghtmltxt(year, expedition, txt): logbook_entry_count = 0 for trippara in tripparas: logbook_entry_count += 1 - + tid= "n{}-s{:02d}".format(year,logbook_entry_count) + s = re.match(r'''(?x)(?:\s*.*?\s*

)? # second date \s*(?:\s*)? \s*(.*?)(?:

)? @@ -298,7 +282,7 @@ def Parseloghtmltxt(year, expedition, txt): EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0, - entry_type="html") + entry_type="html", tid=tid) EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, "html", tripid1, logbook_entry_count) @@ -314,13 +298,13 @@ def Parseloghtml01(year, expedition, txt): logbook_entry_count = 0 for trippara in tripparas: logbook_entry_count += 1 + tid= f"{year}.s{logbook_entry_count:02d}" try: - tripentry = year + "." + str(logbook_entry_count) s = re.match(r"(?s)\s*(?:

)?(.*?)(.*)$(?i)", trippara) if not s: - message = " ! - Skipping logentry on failure to parse header: " + tripentry + trippara[:300] + "..." + message = " ! - Skipping logentry on failure to parse header: " + tid + trippara[:300] + "..." DataIssue.objects.create(parser='logbooks', message=message) - logdataissues[tripentry]=message + logdataissues[tid]=message print(message) break tripheader, triptext = s.group(1), s.group(2) @@ -356,23 +340,35 @@ def Parseloghtml01(year, expedition, txt): entrytuple = (ldate, tripcave, triptitle, ltriptext, trippeople, expedition, tu, "html01", tripid) logentries.append(entrytuple) + try: + EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, + trippeople=trippeople, expedition=expedition, logtime_underground=0, + entry_type="html") - EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, - trippeople=trippeople, expedition=expedition, logtime_underground=0, - entry_type="html") - - EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, - "html01", tripid, logbook_entry_count) + except: + message = " ! - Enter log entry into database FAIL exception in: " + tid + DataIssue.objects.create(parser='logbooks', message=message) + logdataissues[tid]=message + print(message) + try: + EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, + "html01", tripid, logbook_entry_count) + except: + message = " ! - Enter log entry into ObjectStore FAIL exception in: " + tid + DataIssue.objects.create(parser='logbooks', message=message) + logdataissues[tid]=message + print(message) + except: - message = " ! - Skipping logentry due to exception in: " + tripentry + message = " ! - Skipping logentry due to exception in: " + tid DataIssue.objects.create(parser='logbooks', message=message) - logdataissues[tripentry]=message + logdataissues[tid]=message print(message) errorcount += 1 if errorcount >5 : - message = " !!- TOO MANY ERRORS - aborting logbook: " + year + message = f" !!- TOO MANY ERRORS - aborting at '{tid}' logbook: {year}" DataIssue.objects.create(parser='logbooks', message=message) - logdataissues[tripentry]=message + logdataissues[tid]=message print(message) return @@ -385,12 +381,13 @@ def Parseloghtml03(year, expedition, txt): logbook_entry_count = 0 for trippara in tripparas: logbook_entry_count += 1 + tid= f"{year}.s{logbook_entry_count:02d}" s = re.match(r"(?s)\s*

(.*?)

(.*)$", trippara) if not ( s ) : - message = " ! - Skipping logentry on failure to parse Parseloghtml03: {} {} {}...".format(tripentry,s,trippara[:300]) + message = " ! - Skipping logentry on failure to parse Parseloghtml03: {} {} {}...".format(tid,s,trippara[:300]) DataIssue.objects.create(parser='logbooks', message=message) - logdataissues[tripentry]=message + logdataissues[tid]=message print(message) break @@ -415,7 +412,6 @@ def Parseloghtml03(year, expedition, txt): ltriptext = re.sub(r"

", "\n\n", ltriptext).strip() ltriptext = re.sub(r"[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext) - tid= "n{}-s{:02d}".format(str(ldate),logbook_entry_count) entrytuple = (ldate, tripcave, triptitle, ltriptext, trippeople, expedition, tu, "html03", tid) @@ -477,21 +473,23 @@ def LoadLogbookForExpedition(expedition, expect): def cleanerrors(year): global logdataissues - print(f' - CLEAN {year} {len(logdataissues)} data issues in total') dataissues = DataIssue.objects.filter(parser='logbooks') for di in dataissues: - ph = "t" + year + "-" + ph = year if re.search(ph, di.message) is not None: print(f' - CLEANING dataissue {di.message}') di.delete() - for te, content in logdataissues: + print(f' - CLEAN {year} {len(logdataissues)} {type(logdataissues)} data issues for this year') + dellist = [] + for key, value in logdataissues.items(): # tripentry = year + "." + str(logbook_entry_count) - print(f' - CLEAN {te}') - if te.startswith(year + "."): - print(f' - CLEANING logdataissue {te}') - logdataissues.pop(te) - + print(f' - CLEAN [{key}]') + if key.startswith(year + "."): + print(f' - CLEANING logdataissues [{key:12}]: value ') + dellist.append(key) + for i in dellist: + del logdataissues[i] cleanerrors(expedition.year) diff --git a/templates/dataformat/logbookentry.html b/templates/dataformat/logbookentry.html deleted file mode 100644 index 4d4046b..0000000 --- a/templates/dataformat/logbookentry.html +++ /dev/null @@ -1,26 +0,0 @@ -{% autoescape off %} - - - - - - -

{{trip.title}}

-{{date}} - {{expeditionyear}} - -{% if trip.caveOrLocation == "cave" %} -{{trip.cave}} -{% else %} -{{trip.location}} -{% endif %} - -{% for person in persons %} -
-{{person.name}} -TU{% if person.TU %}{{person.TU}}{% else %}0{% endif %}hours -
-{% endfor %} -
{{trip.html}}
- - -{% endautoescape %} diff --git a/templates/editcave.html b/templates/editcave.html deleted file mode 100644 index b8a3e95..0000000 --- a/templates/editcave.html +++ /dev/null @@ -1,19 +0,0 @@ -{% extends "base.html" %} -{% load csrffaker %} -{% block title %}Logbook {{logbookentry.id}}{% endblock %} -{% block head %} - - - - - -{{ tripForm.media }} -{% endblock %} -{% block content %} - -
{% csrf_token %} - {{ form }} -

-
- -{% endblock %} diff --git a/templates/logbookentrynew-pending-deletion.html b/templates/logbookentrynew-pending-deletion.html deleted file mode 100644 index bdb0919..0000000 --- a/templates/logbookentrynew-pending-deletion.html +++ /dev/null @@ -1,84 +0,0 @@ -{% extends "base.html" %} - -{% load csrffaker %} -{% block title %}Logbook {{logbookentry.id}}{% endblock %} -{% block head %} - - - - - -{{ tripForm.media }} -{% endblock %} -{% block content %} - -
{% csrf_token %} - {{ tripForm.non_field_errors }} -
- {{ tripForm.title.errors }} - - {{ tripForm.title }} -
-
- {{ tripForm.date.errors }} - - {{ tripForm.date }} -
-
- {{ tripForm.caveOrLocation.errors }} - - {{ tripForm.caveOrLocation }} -
-
- {{ tripForm.cave.errors }} - - {{ tripForm.cave }} -
-
- {{ tripForm.location.errors }} - - {{ tripForm.location }} -
- - - - - - - - - {% for form in personTripFormSet.forms %} - - - - - - {{ form.non_field_errors }} - - {% endfor %} - -
PersonTU /hoursAuthor
{{ form.name.errors }}{{ form.name }}{{ form.TU.errors }}{{ form.TU }}{{ form.author.errors }}{{ form.author }}
- {{ personTripFormSet.management_form }} -
- {{ tripForm.html.errors }} - - {{ tripForm.html }} -
-

-
- -{% endblock %} diff --git a/urls.py b/urls.py index 5d475a9..03c316c 100644 --- a/urls.py +++ b/urls.py @@ -10,7 +10,7 @@ from django.urls import reverse, resolve from troggle.core.views import caves, statistics, survex from troggle.core.views.surveys import surveyscansingle, surveyscansfolder, surveyscansfolders, dwgdata, dwgfilesingle, dwgfileupload -from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, newfile, scanupload +from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, scanupload from troggle.core.views.other import downloadlogbook, ajax_QM_number, downloadQMs from troggle.core.views.caves import ent, cavepage from troggle.core.views.logbooks import get_logbook_entries, logbookentry, logbookSearch @@ -94,7 +94,6 @@ trogglepatterns = [ # Logbook entries re_path(r'^logbookentry/(?P.*)/(?P.*)/?$', logbookentry,name="logbookentry"), - re_path(r'^newfile', newfile, name="newFile"), # oddly broken, needs investigating more re_path(r'^logbooksearch/(.*)/?$', logbookSearch), # name 'search' not defined in views/logbooks.py re_path(r'^logbook(?P\d\d\d\d)\.(?P.*)/?$', downloadlogbook), # e.g. /logbook2019.html # working but old CSS in template re_path(r'^logbook/?$', downloadlogbook, name="downloadlogbook"),