delete old forms, templates. fix logdataissues

core/forms.py

@@ -15,8 +15,8 @@ from troggle.core.models.caves import Cave, LogbookEntry, QM, Entrance, CaveAndE
 Some are not used and need renovating or destroying.
 '''
 
-todo = '''Fix UploadFileForm
+todo = '''Fix UploadFileForm - long list of actions
-delete TripForm once working
+
 '''
 
 class CaveForm(ModelForm):
@@ -106,39 +106,6 @@ class EntranceLetterForm(ModelForm):
         model = CaveAndEntrance
         exclude = ('cave', 'entrance')
 
-
-def getTripForm(expedition):
-
-    class TripForm(forms.Form):
-        date = forms.DateField()
-        title = forms.CharField(max_length=200)
-        caves = sorted([cave.reference() for cave in Cave.objects.all()])
-        caves = ["-----"] + caves
-        cave = forms.ChoiceField([(c, c) for c in caves], required=False)
-        location = forms.CharField(max_length=200, required=False) 
-        caveOrLocation = forms.ChoiceField([("cave", "Cave"), ("location", "Location")], widget = forms.widgets.RadioSelect())
-#        html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30})) 
-        html = forms.CharField(widget=forms.Textarea(attrs={"cols":80, "rows":20}))
-
-        def clean(self):
-            print((dir(self)))
-            if self.cleaned_data.get("caveOrLocation") == "cave" and not self.cleaned_data.get("cave"):
-                self._errors["cave"] = self.error_class(["This field is required"]) 
-            if self.cleaned_data.get("caveOrLocation") == "location" and not self.cleaned_data.get("location"):
-                self._errors["location"] = self.error_class(["This field is required"]) 
-            return self.cleaned_data
-  
-    class PersonTripForm(forms.Form):
-        names = sorted([get_name(pe) for pe in PersonExpedition.objects.filter(expedition = expedition)])
-        names = ["-----"] + names
-        name = forms.ChoiceField([(n, n) for n in names])
-        TU = forms.FloatField(required=False)
-        author = forms.BooleanField(required=False, default=False)
-
-    PersonTripFormSet = formset_factory(PersonTripForm, extra=1)
-
-    return PersonTripFormSet, TripForm
-    
 def get_name(pe):
             if pe.nickname:
                 return pe.nickname

core/views/logbooks.py

@@ -12,7 +12,6 @@ from django.template.defaultfilters import slugify
 from django.utils import timezone
 from django.views.generic.list import ListView
 
-from troggle.core.forms import getTripForm  # , get_name
 from troggle.core.models.troggle import Expedition, Person, PersonExpedition
 from troggle.core.utils import TROG
 from troggle.core.models.caves import LogbookEntry, PersonTrip
@@ -60,7 +59,8 @@ def expedition(request, expeditionname):
     if request.user.is_authenticated: 
         if "reload" in request.GET:
             this_expedition = Expedition.objects.get(year=int(expeditionname))
-            # Need to delete the exisitng entries or we get duplicaiton
+            # Need to delete the exisitng entries or we get duplication
+            # Need to delete both in the Django ORM and in our own object-store.
             entries = this_expedition.logbookentry_set.all()
             print(f'! - expo {expeditionname} {len(entries)}  entries')
             for entry in entries:

core/views/other.py

@@ -22,17 +22,19 @@ Also has code to download a logbook in a choice of formats (why?!) and to
 download all QMs (not working)
 '''
 
-todo = '''Delete the newfile & TripForm code once we have a proper file-upload system working.
+todo = '''
-meanwhile keep it as an example to consider.
+- Check that the logbookdownloader works by testing with a round trip.
 
-Check that the logbookdownloader works by testing with a round trip.
+- Use it to convert all older logbooks into the 2005-variant of HTML then we can 
+    get rid of the parsers for older formats.  There are no images stored in the database, 
+    so this is only a tool for a first pass, to be followed by extensive hand-editing!
+    When we have done all the old logbooks, delete this function and the two templates.
 
-Use it to convert all older logbooks into the 2005-variant of HTML then we can 
+- But how does this interact with troggle/logbooksdump.py ?
-get rid of the parsers for older formats.  There are no images stored in the database, 
+
-so this is only a tool for a first pass, to be followed by extensive hand-editing!
+- deleted nefile() - check on deleted UploadFileForm using the editfile.html template which is about re-submitting 
-When we have done all the old logbooks, delete this function and the two templates.
+    a LBE aka TripReport
 
-But how does this interact with troggle/logbooksdump.py ?S
 '''
 
 def todos(request, module): 
@@ -174,50 +176,6 @@ def ajax_QM_number(request):
 
     return HttpResponse(res)
 
-@login_required_if_public
-def newfile(request, pslug = None):
-    '''     
-    If not POST, it goes straight to UploadFileForm using the editfile.html template which is about re-submitting 
-    a LBE aka TripReport
-    
-    If it is POST, then it is a new LBE so an HTML formatted version of it is produced, using template.render()
-    and a format 'dataformat/logbookentry.html' which is then put in expoweb/years<year>/autologbook/<LBEnamedfile>
-    '''
-    if pslug:
-        previousfile = LogbookEntry.objects.get(slug = pslug, date = previousdate, expedition = expedition)
-    if request.method == 'POST': # If the form has been submitted...
-        tripForm = TripForm(request.POST) # A form bound to the POST data
-        personTripFormSet = PersonTripFormSet(request.POST)
-        if tripForm.is_valid() and personTripFormSet.is_valid(): # All validation rules pass
-            dateStr = tripForm.cleaned_data["date"].strftime("%Y-%m-%d")
-            directory = os.path.join(settings.EXPOWEB, 
-                                   "years", 
-                                   expedition.year, 
-                                   "autologbook")
-            filename = os.path.join(directory, 
-                                   dateStr + "." + slugify(tripForm.cleaned_data["title"])[:50] + ".html")
-            if not os.path.isdir(directory):
-                os.mkdir(directory)
-            if pslug and pdate:
-                delLogbookEntry(previouslbe)
-            f = open(filename, "w")
-            template = loader.get_template('dataformat/logbookentry.html')
-            context = {'trip': tripForm.cleaned_data, 
-                              'persons': personTripFormSet.cleaned_data,
-                              'date': dateStr,
-                              'expeditionyear': expeditionyear}
-            f.write(template.render(context))
-            f.close()
-            print(logbookparsers.parseAutoLogBookEntry(filename))
-            return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST
-    else:
-        if pslug:
-            pass
-        else:        
-            fileform = UploadFileForm() # An unbound form
-
-    return render(request, 'editfile.html', {'fileForm': fileform, })
-
 @login_required_if_public
 def scanupload(request, year='2050'):
     print(f'! - FORM scanupload - start')

parsers/logbooks.py

@@ -58,10 +58,12 @@ trips ={}
 #
 # the logbook loading section
 #
-def GetTripPersons(trippeople, expedition, logtime_underground):
+def GetTripPersons(trippeople, expedition, logtime_underground, tid="!"):
     res = [ ]
     author = None
     round_bracket_regex = re.compile(r"[\(\[].*?[\)\]]")
+    if tid =="!":
+        tid = expedition.year + "." + tripperson
     for tripperson in re.split(r",|\+|&|&(?!\w+;)| and ", trippeople):
         tripperson = tripperson.strip()
         mul = re.match(r"<u>(.*?)</u>$(?i)", tripperson)
@@ -71,10 +73,10 @@ def GetTripPersons(trippeople, expedition, logtime_underground):
             tripperson = re.sub(round_bracket_regex, "", tripperson).strip()
             personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower())
             if not personyear:
-                message = "No name match for: ||'%s'|| in year '%s'" % (tripperson, expedition.year)
+                message = f" ! - {expedition.year} No name match for: '{tripperson}' " 
                 print(message)
                 DataIssue.objects.create(parser='logbooks', message=message)
-                logdataissues[expedition.year + "~" + tripperson]=message
+                logdataissues[tid]=message
             res.append((personyear, logtime_underground))
             if mul:
                 author = personyear
@@ -84,37 +86,18 @@ def GetTripPersons(trippeople, expedition, logtime_underground):
         author = res[-1][0]
     return res, author
 
-def GetTripCave(place):  
+def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki", tid="!"):
-    try:
-        katastNumRes=[]
-        katastNumRes=list(Cave.objects.filter(kataster_number=int(place)))
-    except ValueError:
-        message = " ! - ValueError on finding place " + str(place) + " entered. " + tripdate + " - " + year 
-        DataIssue.objects.create(parser='logbooks', message=message)
-        logdataissues["author"]=message
-    officialNameRes=list(Cave.objects.filter(official_name=place))
-    tripCaveRes=officialNameRes+katastNumRes
-
-    if len(tripCaveRes)==1:
-        return tripCaveRes[0]
-    elif len(tripCaveRes)>1:
-        message = " ! - Ambiguous place " + str(place) + " entered. " + tripdate + " - " + year + " " + str(tripCaveRes)
-        DataIssue.objects.create(parser='logbooks', message=message)
-        logdataissues["author"]=message
-        return tripCaveRes[0]
-    else:
-        print((" " , place))
-        message = " ! - No cave found for place:" + str(place) + tripdate + " - " + year 
-        DataIssue.objects.create(parser='logbooks', message=message)
-        logdataissues["author"]=message
-        return None
-
-
-def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki"):
     """ saves a logbook entry and related persontrips 
     Does NOT save the expeditionday_id  - all NULLs. why?
     """
-    trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground)
+    try:
+        trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground, tid=tid)
+    except:
+        message = " ! - Skipping logentry: %s - GetTripPersons FAIL in year '%s'" % (title, expedition.year)
+        DataIssue.objects.create(parser='logbooks', message=message)
+        logdataissues["title"]=message
+        return
+        
     if not author:
         print(" ! - Skipping logentry: " + title + " - no author for entry")
         message = " ! - Skipping logentry: %s - no author for entry in year '%s'" % (title, expedition.year)
@@ -264,7 +247,8 @@ def Parseloghtmltxt(year, expedition, txt):
     logbook_entry_count = 0
     for trippara in tripparas:
         logbook_entry_count += 1
-        
+        tid= "n{}-s{:02d}".format(year,logbook_entry_count)
+       
         s = re.match(r'''(?x)(?:\s*<div\sclass="tripdate"\sid=".*?">.*?</div>\s*<p>)?  # second date
                             \s*(?:<a\s+id="(.*?)"\s*/>\s*</a>)?
                             \s*<div\s+class="tripdate"\s*(?:id="(.*?)")?>(.*?)</div>(?:<p>)?
@@ -298,7 +282,7 @@ def Parseloghtmltxt(year, expedition, txt):
 
         EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext,
                           trippeople=trippeople, expedition=expedition, logtime_underground=0,
-                          entry_type="html")
+                          entry_type="html", tid=tid)
 
         EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, 
                             "html", tripid1, logbook_entry_count)
@@ -314,13 +298,13 @@ def Parseloghtml01(year, expedition, txt):
     logbook_entry_count = 0
     for trippara in tripparas:
         logbook_entry_count += 1
+        tid= f"{year}.s{logbook_entry_count:02d}"
         try:
-            tripentry = year + "." + str(logbook_entry_count)
             s = re.match(r"(?s)\s*(?:<p>)?(.*?)</?p>(.*)$(?i)", trippara)
             if not s:
-                message = " ! - Skipping logentry on failure to parse header: " + tripentry + trippara[:300] + "..."
+                message = " ! - Skipping logentry on failure to parse header: " + tid + trippara[:300] + "..."
                 DataIssue.objects.create(parser='logbooks', message=message)
-                logdataissues[tripentry]=message
+                logdataissues[tid]=message
                 print(message)
                 break
             tripheader, triptext = s.group(1), s.group(2)
@@ -356,23 +340,35 @@ def Parseloghtml01(year, expedition, txt):
             entrytuple = (ldate, tripcave, triptitle, ltriptext, 
                     trippeople, expedition, tu, "html01", tripid)
             logentries.append(entrytuple)
+            try:
+                EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext,
+                                  trippeople=trippeople, expedition=expedition, logtime_underground=0,
+                                  entry_type="html")
 
-            EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext,
+            except:
-                              trippeople=trippeople, expedition=expedition, logtime_underground=0,
+                message = " ! - Enter log entry into database FAIL  exception in: " + tid 
-                              entry_type="html")
+                DataIssue.objects.create(parser='logbooks', message=message)
-
+                logdataissues[tid]=message
-            EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, 
+                print(message)
-                                "html01", tripid, logbook_entry_count)
+            try:
+                EnterLogIntoObjStore(year, ldate, tripcave, triptitle, ltriptext, trippeople, tu, 
+                                    "html01", tripid, logbook_entry_count)
+            except:
+                message = " ! - Enter log entry into ObjectStore FAIL  exception in: " + tid 
+                DataIssue.objects.create(parser='logbooks', message=message)
+                logdataissues[tid]=message
+                print(message)
+                
         except:
-            message = " ! - Skipping logentry due to exception in: " + tripentry 
+            message = " ! - Skipping logentry due to exception in: " + tid 
             DataIssue.objects.create(parser='logbooks', message=message)
-            logdataissues[tripentry]=message
+            logdataissues[tid]=message
             print(message)
             errorcount += 1
             if errorcount >5 :
-                message = " !!- TOO MANY ERRORS - aborting logbook: " + year
+                message = f" !!- TOO MANY ERRORS - aborting at '{tid}' logbook: {year}"
                 DataIssue.objects.create(parser='logbooks', message=message)
-                logdataissues[tripentry]=message
+                logdataissues[tid]=message
                 print(message)
                 return
 
@@ -385,12 +381,13 @@ def Parseloghtml03(year, expedition, txt):
     logbook_entry_count = 0
     for trippara in tripparas:
         logbook_entry_count += 1
+        tid= f"{year}.s{logbook_entry_count:02d}"
         
         s = re.match(r"(?s)\s*<p>(.*?)</p>(.*)$", trippara)
         if not ( s ) :
-            message = " ! - Skipping logentry on failure to parse Parseloghtml03: {} {} {}...".format(tripentry,s,trippara[:300])
+            message = " ! - Skipping logentry on failure to parse Parseloghtml03: {} {} {}...".format(tid,s,trippara[:300])
             DataIssue.objects.create(parser='logbooks', message=message)
-            logdataissues[tripentry]=message
+            logdataissues[tid]=message
             print(message)
             break
         
@@ -415,7 +412,6 @@ def Parseloghtml03(year, expedition, txt):
         ltriptext = re.sub(r"<p>", "\n\n", ltriptext).strip()
         ltriptext = re.sub(r"[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext)
 
-        tid= "n{}-s{:02d}".format(str(ldate),logbook_entry_count)
 
         entrytuple = (ldate, tripcave, triptitle, ltriptext, 
                 trippeople, expedition, tu, "html03", tid)
@@ -477,21 +473,23 @@ def LoadLogbookForExpedition(expedition, expect):
     
     def cleanerrors(year):
         global logdataissues
-        print(f'   - CLEAN {year} {len(logdataissues)} data issues in total')
         dataissues = DataIssue.objects.filter(parser='logbooks')
         for di in dataissues:
-            ph = "t" + year + "-"
+            ph = year
             if re.search(ph, di.message) is not None:
                 print(f'   - CLEANING dataissue {di.message}')
                 di.delete()
      
-        for te, content in logdataissues:
+        print(f'   - CLEAN {year} {len(logdataissues)} {type(logdataissues)} data issues for this year')
+        dellist = []
+        for key, value in logdataissues.items():
             #  tripentry = year + "." + str(logbook_entry_count)
-            print(f'   - CLEAN {te}')
+            print(f'   - CLEAN [{key}]')
-            if te.startswith(year + "."):
+            if key.startswith(year + "."):
-                print(f'   - CLEANING logdataissue {te}')
+                print(f'   - CLEANING logdataissues [{key:12}]: value ')
-                logdataissues.pop(te)
+                dellist.append(key)
-
+        for i in dellist:
+            del logdataissues[i]
 
     cleanerrors(expedition.year)
 

templates/dataformat/logbookentry.html

@@ -1,26 +0,0 @@
-{% autoescape off %}
-<!DOCTYPE html>
-<html>
-<head>
-<style type="text/css">.author {text-decoration:underline}</style>
-</head>
-<body>
-<H1>{{trip.title}}</H1>
-<span class="date">{{date}}</span> - <span class="expeditionyear">{{expeditionyear}}</span>
-
-{% if trip.caveOrLocation == "cave" %}
-<span class="cave">{{trip.cave}}</span>
-{% else %}
-<span class="location">{{trip.location}}</span>
-{% endif %}
-
-{% for person in persons %}
-<div class="person">
-<span class="name{% if person.author %} author{% endif %}">{{person.name}}</span>
-TU<span class="TU">{% if person.TU %}{{person.TU}}{% else %}0{% endif %}</span>hours
-</div>
-{% endfor %}
-<div class="report">{{trip.html}}</div>
-</body>
-</html>
-{% endautoescape %}

templates/editcave.html

@@ -1,19 +0,0 @@
-{% extends "base.html" %}
-{% load csrffaker %} 
-{% block title %}Logbook {{logbookentry.id}}{% endblock %}
-{% block head %}
-
-<link rel="stylesheet" href="{{ settings.JSLIB_URL }}jquery-ui/css/lightness/jquery-ui.css" type="text/css" media="all" />
-<script src="{{ settings.JSLIB_URL }}jquery-ui/jquery-ui.min.js" type="text/javascript"></script>
-<script src="{{ settings.MEDIA_URL }}js/jquery.formset.min.js" type="text/javascript"></script>
-<!--<script src="{{ settings.TINY_MCE_MEDIA_URL }}tiny_mce.js" type="text/javascript"></script>-->
-{{ tripForm.media }}
-{% endblock %}
-{% block content %}
-
-<form action="" method="post">{% csrf_token %}
-    {{ form }}
-    <p><input type="submit" value="Submit" /></p>
-</form>
-
-{% endblock %}

templates/logbookentrynew-pending-deletion.html

@@ -1,84 +0,0 @@
-{% extends "base.html" %}
-<!-- logbookentry.html - this text visible because this template has been included -->
-{% load csrffaker %} 
-{% block title %}Logbook {{logbookentry.id}}{% endblock %}
-{% block head %}
-<script>
-	$(function() {
-		$("#id_date").datepicker({dateFormat: "yy-mm-dd"});
-                $('.persontrips tbody tr').formset();
-                $(":radio[name*='caveOrLocation']").change(setLocationType);
-                $(setLocationType());
-                function setLocationType () {
-                    $("#cave").hide();
-                    $("#location").hide();
-                    $("#" + $(":radio[name*='caveOrLocation']:checked")[0].value).show();
-                    };
-           
-	});
-
-</script>
-<link rel="stylesheet" href="{{ settings.JSLIB_URL }}jquery-ui/css/lightness/jquery-ui.css" type="text/css" media="all" />
-<script src="{{ settings.JSLIB_URL }}jquery/jquery-ui.min.js" type="text/javascript"></script>
-<script src="{{ settings.MEDIA_URL }}js/jquery.formset.min.js" type="text/javascript"></script>
-<!--<script src="{{ settings.TINY_MCE_MEDIA_URL }}tiny_mce.js" type="text/javascript"></script>-->
-{{ tripForm.media }}
-{% endblock %}
-{% block content %}
-
-<form action="" method="post">{% csrf_token %}
-    {{ tripForm.non_field_errors }}
-    <div class="fieldWrapper">
-        {{ tripForm.title.errors }}
-        <label for="id_title">Title:</label>
-        {{ tripForm.title }}
-    </div>
-    <div class="fieldWrapper">
-        {{ tripForm.date.errors }}
-        <label for="id_date">Date:</label>
-        {{ tripForm.date }}
-    </div>
-    <div class="fieldWrapper">
-        {{ tripForm.caveOrLocation.errors }}
-        <label for="id_caveOrLocation">Location Type:</label>
-        {{ tripForm.caveOrLocation }}
-    </div>
-    <div class="fieldWrapper" id="cave">
-        {{ tripForm.cave.errors }}
-        <label for="id_cave">Cave:</label>
-        {{ tripForm.cave }}
-    </div>
-    <div class="fieldWrapper" id="location">
-        {{ tripForm.location.errors }}
-        <label for="id_location">Location:</label>
-        {{ tripForm.location }}
-    </div>
-    <table class="persontrips" border="0" cellpadding="0" cellspacing="0">
-         <tbody>
-             <tr>
-                <th>Person</th>
-                <th>TU /hours</th>
-                <th>Author</th>
-                <th></th>
-             </tr>
-             {% for form in personTripFormSet.forms %}
-             <tr>
-                <td>{{ form.name.errors }}{{ form.name }}</td>
-                <td>{{ form.TU.errors }}{{ form.TU }}</td>
-                <td>{{ form.author.errors }}{{ form.author }}</td>
-                <td></td>
-               {{ form.non_field_errors }}
-             </tr>
-             {% endfor %}
-         </tbody>
-    </table>
-    {{ personTripFormSet.management_form }}
-    <div class="fieldWrapper">
-        {{ tripForm.html.errors }}
-        <label for="id_date">Content:</label>
-        {{ tripForm.html }}
-    </div>
-    <p><input type="submit" value="Sumbit Trip Report" /></p>
-</form>
-
-{% endblock %}

urls.py

@@ -10,7 +10,7 @@ from django.urls import reverse, resolve
 
 from troggle.core.views import caves, statistics, survex
 from troggle.core.views.surveys import surveyscansingle, surveyscansfolder, surveyscansfolders, dwgdata, dwgfilesingle, dwgfileupload
-from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, newfile, scanupload
+from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, scanupload
 from troggle.core.views.other import downloadlogbook, ajax_QM_number, downloadQMs
 from troggle.core.views.caves import ent, cavepage
 from troggle.core.views.logbooks import get_logbook_entries, logbookentry, logbookSearch
@@ -94,7 +94,6 @@ trogglepatterns = [
 
 #   Logbook entries
     re_path(r'^logbookentry/(?P<date>.*)/(?P<slug>.*)/?$', logbookentry,name="logbookentry"),
-    re_path(r'^newfile', newfile, name="newFile"), # oddly broken, needs investigating more
     re_path(r'^logbooksearch/(.*)/?$', logbookSearch), # name 'search' not defined in views/logbooks.py
     re_path(r'^logbook(?P<year>\d\d\d\d)\.(?P<extension>.*)/?$', downloadlogbook), # e.g. /logbook2019.html # working but old CSS in template 
     re_path(r'^logbook/?$', downloadlogbook, name="downloadlogbook"),