Fully working dj 1.11.29

core/migrations/0002_auto_20200619_1511.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-06-19 14:11
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='logbookentry',
+            name='cave_slug',
+            field=models.SlugField(blank=True),
+        ),
+    ]

core/migrations/0003_auto_20200619_1513.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-06-19 14:13
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0002_auto_20200619_1511'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='logbookentry',
+            name='cave_slug',
+            field=models.SlugField(blank=True, null=True),
+        ),
+    ]

core/models_caves.py

@@ -125,7 +125,7 @@ class Cave(TroggleModel):
         return urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,}))
 
     def __str__(self, sep = ": "):
-        return str("slug:"+str(self.slug()))
+        return str(self.slug())
 
     def get_QMs(self):
         return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all())
@@ -418,7 +418,7 @@ class LogbookEntry(TroggleModel):
     expeditionday = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.CASCADE)#MJG wants to KILL THIS (redundant information)
     expedition  = models.ForeignKey(Expedition,blank=True, null=True,on_delete=models.CASCADE)  # yes this is double-
     title      = models.CharField(max_length=settings.MAX_LOGBOOK_ENTRY_TITLE_LENGTH)
-    cave_slug  = models.SlugField(max_length=50)
+    cave_slug  = models.SlugField(max_length=50, blank=True, null=True)
     place      = models.CharField(max_length=100,blank=True, null=True,help_text="Only use this if you haven't chosen a cave")
     text       = models.TextField()
     slug       = models.SlugField(max_length=50)

deprecations.txt

@@ -0,0 +1,19 @@
+/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/django/db/models/sql/query.py:11: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
+  from collections import Counter, Iterator, Mapping, OrderedDict
+/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/django/db/models/sql/query.py:11: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
+  from collections import Counter, Iterator, Mapping, OrderedDict
+/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/django/core/paginator.py:126: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working
+  class Page(collections.Sequence):
+/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/registration/auth_urls_classes.py:19: DeprecationWarning: 
+include('registration.auth_urls') is deprecated and will be
+removed in django-registration 3.0. Use
+include('django.contrib.auth.urls') instead.
+
+  DeprecationWarning
+/mnt/d/CUCC-Expo/t37/lib/python3.7/site-packages/registration/backends/default/urls.py:25: DeprecationWarning: 
+include('registration.backends.default.urls') is deprecated and
+will be removed in django-registration 3.0. Use
+include('registration.backends.model_activation.urls')
+instead.
+
+  DeprecationWarning

parsers/QMs.py

@@ -5,6 +5,7 @@ from datetime import *
 
 from django.conf import settings
 
+from troggle.core.models import DataIssue
 from troggle.core.models_caves import QM, Cave, LogbookEntry
 from utils import save_carefully
 
@@ -15,19 +16,19 @@ def deleteQMs():
 def parseCaveQMs(cave,inputFile):
     """Runs through the CSV file at inputFile (which is a relative path from expoweb) and saves each QM as a QM instance."""
 
-    if cave=='stein':
+    if cave=='204-steinBH':
         try:
             steinBr=Cave.objects.get(official_name="Steinbrückenhöhle")
         except Cave.DoesNotExist:
             print("Steinbruckenhoehle is not in the database. Please run parsers.")
             return
-    elif cave=='hauch':
+    elif cave=='234-Hauch':
         try:
             hauchHl=Cave.objects.get(official_name="Hauchhöhle")
         except Cave.DoesNotExist:
-            print("Hauchhoele is not in the database. Please run parsers.")
+            print("Hauchhoehle is not in the database. Please run parsers.")
             return
-    elif cave =='kh':
+    elif cave =='161-KH':
         try:
             kh=Cave.objects.get(official_name="Kaninchenhöhle")
         except Cave.DoesNotExist:
@@ -45,12 +46,14 @@ def parseCaveQMs(cave,inputFile):
         try:
             year=int(line[0][1:5])
             #check if placeholder exists for given year, create it if not
-            if cave=='stein':
-                placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, title="placeholder for QMs in 204", text="QMs temporarily attached to this should be re-attached to their actual trips", defaults={"date": date(year, 1, 1),"cave":steinBr})
-            elif cave=='hauch':
-                placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, title="placeholder for QMs in 234", text="QMs temporarily attached to this should be re-attached to their actual trips", defaults={"date": date(year, 1, 1),"cave":hauchHl})            
+            message = " ! - "+ str(year) + " logbook: placeholder entry for '" + cave + "' created. Should be re-attached to the actual trip."
+            if cave=='204-steinBH':
+                placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="204", title="placeholder for QMs in 204", text=message, defaults={"date": date(year, 1, 1),"cave_slug":str(steinBr)})
+            elif cave=='234-Hauch':
+                placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="234", title="placeholder for QMs in 234", text=message, defaults={"date": date(year, 1, 1),"cave_slug":str(hauchHl)})
             if hadToCreate:
-                print((" - placeholder logbook entry for " + cave + " " + str(year) + " added to database"))
+                print(message)
+                DataIssue.objects.create(parser='QMs', message=message)
             QMnum=re.match(r".*?-\d*?-X?(?P<numb>\d*)",line[0]).group("numb")
             newQM = QM()
             newQM.found_by=placeholder
@@ -98,7 +101,11 @@ def parse_KH_QMs(kh, inputFile):
             res=res.groupdict()
             year=int(res['year'])
             #check if placeholder exists for given year, create it if not
-            placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, title="placeholder for QMs in 161", text="QMs temporarily attached to this should be re-attached to their actual trips", defaults={"date": date((year), 1, 1),"cave":kh})
+            message = " ! - "+ str(year) + " logbook: placeholder entry for '161 KH' created. Should be re-attached to the actual trip."
+            placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, place="161", title="placeholder for QMs in 161", text=message, defaults={"date": date((year), 1, 1),"cave_slug":str(kh)})
+            if hadToCreate:
+                print(message)
+                DataIssue.objects.create(parser='QMs', message=message)
             lookupArgs={
                 'found_by':placeholder,
                 'number':res['number']
@@ -112,7 +119,7 @@ def parse_KH_QMs(kh, inputFile):
             save_carefully(QM,lookupArgs,nonLookupArgs)
         
 def Load_QMs():
-    parseCaveQMs(cave='stein',inputFile=r"1623/204/qm.csv")
-    parseCaveQMs(cave='hauch',inputFile=r"1623/234/qm.csv")
-    parseCaveQMs(cave='kh', inputFile="1623/161/qmtodo.htm")
+    parseCaveQMs(cave='204-steinBH',inputFile=r"1623/204/qm.csv")
+    parseCaveQMs(cave='234-Hauch',inputFile=r"1623/234/qm.csv")
+    parseCaveQMs(cave='161-KH', inputFile="1623/161/qmtodo.htm")
     #parseCaveQMs(cave='balkonhoehle',inputFile=r"1623/264/qm.csv")

parsers/logbooks.py

@@ -76,9 +76,10 @@ def GetTripCave(place):
         return tripCaveRes[correctIndex]
     else:
         print(("No cave found for place " , place))
-        return
+        return None
 
 # lookup function modelled on GetPersonExpeditionNameLookup
+# repeated assignment each call, needs refactoring
 Gcavelookup = None
 def GetCaveLookup():
     global Gcavelookup
@@ -91,19 +92,28 @@ def GetCaveLookup():
             Gcavelookup[cave.kataster_number] = cave
         if cave.unofficial_number:
             Gcavelookup[cave.unofficial_number] = cave
-
+    # These are exact matches! edit to check for prefix only!
     Gcavelookup["tunnocks"] = Gcavelookup["258"]
     Gcavelookup["hauchhole"] = Gcavelookup["234"]
+    Gcavelookup["KH"] = Gcavelookup["161"]
+    Gcavelookup["Balcony"] = Gcavelookup["264"]
+    Gcavelookup["Balkon"] = Gcavelookup["264"]
+    Gcavelookup["FGH"] = Gcavelookup["290"]
+    Gcavelookup["GSH"] = Gcavelookup["291"]
+    Gcavelookup["Homecoming"] = Gcavelookup["2018-dm-07"]
     return Gcavelookup
 
 
 logentries = [] # the entire logbook for one year is a single object: a list of entries
-noncaveplaces = [ "Journey", "Loser Plateau" ]
+noncaveplaces = [ "QMplaceholder", "Journey", "Loser Plateau", "UNKNOWN", 'plateau', 
+        'base camp', 'basecamp', 'top camp', 'topcamp' ]
 logdataissues = {}
 trips ={}
 
 def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki"):
-    """ saves a logbook entry and related persontrips """
+    """ saves a logbook entry and related persontrips 
+    Does NOT save the expeditionday_id  - all NULLs. why?
+    """
 
     trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground)
     if not author:
@@ -112,16 +122,23 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_
         DataIssue.objects.create(parser='logbooks', message=message)
         return
 
-    #tripCave = GetTripCave(place)
+    # This needs attention. The slug field is derived from 'title'
+    # both GetCaveLookup() and GetTripCave() need to work together better. None of this data is *used* though?
+    #tripCave = GetTripCave(place):
 
     lplace = place.lower()
+    cave=None
     if lplace not in noncaveplaces:
         cave = GetCaveLookup().get(lplace)
+        # message = " ! - '" + lplace + "' place not in noncaveplaces."
+        # print(message)
+        # DataIssue.objects.create(parser='logbooks', message=message)
 
     #Check for an existing copy of the current entry, and save
     expeditionday = expedition.get_expedition_day(date)
     lookupAttribs={'date':date, 'title':title}
-    nonLookupAttribs={'place':place, 'text':text, 'expedition':expedition, 'cave':cave, 'slug':slugify(title)[:50], 'entry_type':entry_type}
+    # 'cave' is converted to a string doing this, which renders as the cave slug.
+    nonLookupAttribs={'place':place, 'text':text, 'expedition':expedition, 'cave_slug':str(cave), 'slug':slugify(title)[:50], 'entry_type':entry_type}
     lbo, created=save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs)
 
     

parsers/people.py

@@ -93,8 +93,13 @@ def LoadPersonsExpos():
         if len(names) == 1:
             lastname = ""
 
+        if personline[header["VfHO member"]] =='':
+            vfho = False
+        else:
+            vfho = True
+
         lookupAttribs={'first_name':firstname, 'last_name':(lastname or "")}
-        nonLookupAttribs={'is_vfho':personline[header["VfHO member"]], 'fullname':fullname}
+        nonLookupAttribs={'is_vfho':vfho, 'fullname':fullname}
         person, created = save_carefully(models.Person, lookupAttribs, nonLookupAttribs)
 
         parseMugShotAndBlurb(personline=personline, header=header, person=person)

requirements.txt

@@ -1,7 +1,8 @@
-Django==1.7
-django-extensions==2.2.9
-django-registration==2.0
-django-tinymce==2.0.1
+confusable-homoglyphs==2.0.2
+Django==1.11.29
+django-registration==2.3
 Pillow==7.1.2
-six==1.14.0
+pytz==2020.1
+six==1.15.0
+sqlparse==0.3.1
 Unidecode==1.1.1

utils.py

@@ -45,14 +45,23 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
         defined in core.models.TroggleModel.
     
     """
-    print(" !! - SAVE CAREFULLY Django 1.11.29 ", objectType)
+    try:
         instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
-
+    except:
+        print(" !! - SAVE CAREFULLY   ===================", objectType)
+        print(" !! - -- objects.get_or_create()")
+        print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
+        raise
     if not created and not instance.new_since_parsing:
         for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title)
             setattr(instance, k, v)
+        try:
             instance.save()
-    
+        except:
+            print(" !! - SAVE CAREFULLY   ===================", objectType)
+            print(" !! - -- instance.save()")
+            print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
+            raise
     if created:
         logging.info(str(instance) + ' was just added to the database for the first time. \n')