Merge branch 'master' of ssh://expo.survex.com/home/expo/troggle

core/admin.py

@@ -3,7 +3,7 @@ from django.core import serializers
 from django.http import HttpResponse
 
 from troggle.core.models.caves import Area, Cave, CaveAndEntrance, Entrance
-from troggle.core.models.logbooks import QM, LogbookEntry, PersonLogEntry
+from troggle.core.models.logbooks import QM, LogbookEntry, PersonLogEntry, CaveSlug
 from troggle.core.models.survex import (
     DrawingFile,
     SingleScan,
@@ -139,6 +139,7 @@ admin.site.register(Cave, CaveAdmin)
 admin.site.register(Area)
 admin.site.register(CaveAndEntrance)
 admin.site.register(Entrance, EntranceAdmin)
+admin.site.register(CaveSlug)
 admin.site.register(SurvexBlock, SurvexBlockAdmin)
 admin.site.register(DrawingFile, DrawingFileAdmin)
 admin.site.register(Expedition)

core/models/logbooks.py

@@ -11,8 +11,15 @@ from troggle.core.models.troggle import Expedition, TroggleModel
 """
 
 todo = """
-- Can we rewrite things to eliminate the CaveSlug and objects? Surely
+- Can we rewrite things to eliminate the CaveSlug and objects? No
-  foreign keys work fine ?!
+  Surely foreign keys work fine ?! No
+  
+  Foreign keys do not allow for there being multiple ways to refer to a cave, eg 1623-1999-03 aka 1623-204
+  Having slugs allows for much more loose coupling to caves, which removes alot of the need to reset the database, which interupts work flow.
+  It also means we do not have to be creating temporary cave objects in the database, where we do not have the underlying file in cave_data.
+
+  To Do move Cave Slug back to troggle.core.models
+	
 """
 
 
@@ -25,6 +32,9 @@ class CaveSlug(models.Model):
     slug = models.SlugField(max_length=50, unique=True)
     primary = models.BooleanField(default=False)
 
+    def __str__(self):
+        return f"{self.slug}: {self.cave}"
+
 
 class LogbookEntry(TroggleModel):
     """Single parsed entry from Logbook

core/views/editor_helpers.py

@@ -95,7 +95,7 @@ def new_image_form(request, path):
             # Create directories if required
             for full_path in image_path, thumb_path, desc_path:
                 print(full_path, full_path.parent)
-                full_path.parent.mkdir(parents=False, exist_ok=True)
+                full_path.parent.mkdir(parents=True, exist_ok=True)
             try:
                 change_message = form.cleaned_data["change_message"]
                 write_and_commit(

parsers/logbooks.py

@@ -57,7 +57,7 @@ LOGBOOK_PARSER_SETTINGS = {
 LOGBOOKS_DIR = "years" # subfolder of settings.EXPOWEB
 
 ENTRIES = {
-    "2023": 1,
+    "2023": 11,
     "2022": 90,
     "2019": 55,
     "2018": 95,
@@ -295,7 +295,7 @@ def parser_html(year, expedition, txt, seq=""):
     headmatch = re.match(r"(?i)(?s).*<body[^>]*>(.*?)<hr.*", txt)
     headpara = headmatch.groups()[0].strip()
 
-    # print(f" - headpara:\n'{headpara}'")
+    #print(f" - headpara:\n'{headpara}'")
     if len(headpara) > 0:
         frontpath = Path(settings.EXPOWEB, LOGBOOKS_DIR, year, "frontmatter.html")
         with open(frontpath, "w") as front:
@@ -305,7 +305,7 @@ def parser_html(year, expedition, txt, seq=""):
     endmatch = re.match(r"(?i)(?s).*<hr\s*/>([\s\S]*?)(?=</body)", txt)
     endpara = endmatch.groups()[0].strip()
 
-    # print(f" - endpara:\n'{endpara}'")
+    #print(f" - endpara:\n'{endpara}'")
     if len(endpara) > 0:
         endpath = Path(settings.EXPOWEB, LOGBOOKS_DIR, year, "endmatter.html")
         with open(endpath, "w") as end:
@@ -333,7 +333,7 @@ def parser_html(year, expedition, txt, seq=""):
         if s:
             tripid, tripid1, tripdate, trippeople, triptitle, triptext, tu = s.groups()
         else:  # allow title and people to be swapped in order
-            msg = f" !- {year} Can't parse:{logbook_entry_count} '{trippara[:50]}'..."
+            msg = f" !- {year} Can't parse:{logbook_entry_count} '{trippara[:55]}'...'{trippara}'"
             print(msg)
             DataIssue.objects.create(parser="logbooks", message=msg)
 
@@ -353,7 +353,7 @@ def parser_html(year, expedition, txt, seq=""):
                 tripid, tripid1, tripdate, triptitle, trippeople, triptext, tu = s2.groups()
             else:
                 # if not re.search(r"Rigging Guide", trippara):
-                msg = f" !- Logbook. Can't parse entry on 2nd pass:{logbook_entry_count} '{trippara[:50]}'..."
+                msg = f" !- Logbook. Can't parse entry on 2nd pass:{logbook_entry_count} '{trippara[:55]}'...'{trippara}'"
                 print(msg)
                 DataIssue.objects.create(parser="logbooks", message=msg)
                 continue

parsers/survex.py

@@ -695,12 +695,15 @@ class LoadingSurvex:
                             )
 
         oline = line
+        perps = get_people_on_trip(survexblock)  # What, you don't know Judge Dredd slang ?
+
         if len(line) > 10:
             message = "! DATE Warning LONG DATE '{}' ({}) {}".format(oline, survexblock, survexblock.survexfile.path)
             print(self.insp+message)
             stash_data_issue(parser='svxdate', message=message,  url=None, sb=(survexblock.survexfile.path))
             if line[10] == "-":  # ie a range, just look at first date
                 line = line[0:10]
+                
         if len(line) == 10:
             year = line[:4]
             # TO DO set to correct Austrian timezone Europe/Vienna ?
@@ -708,7 +711,6 @@ class LoadingSurvex:
             survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d")
         elif len(line) == 7:
             year = line[:4]
-            perps = get_people_on_trip(survexblock)  # What, you don't know Judge Dredd slang ?
             message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
             print(self.insp + message)
             stash_data_issue(
@@ -717,13 +719,36 @@ class LoadingSurvex:
             survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m")  # sets to first of month
         elif len(line) == 4:
             year = line[:4]
-            perps = get_people_on_trip(survexblock)
             message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}"
             print(self.insp + message)
             stash_data_issue(
                 parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
             )
             survexblock.date = datetime.strptime(line, "%Y")  # sets to January 1st
+        elif len(line) == 9 or len(line) == 8:
+            year = line[:4]
+            message = f"! DATE format WARNING, single digit day or month number,'{oline}' [{line[-5]}][{line[-2]}] ({survexblock}) {survexblock.survexfile.path}"
+            print(self.insp + message)
+            stash_data_issue(
+                parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
+            )
+            if line[-2] == "-" or line[-2] == ".":
+                line = line[:-1] + '0' + line[-1]
+                survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d")
+                print(f"! DATE -2 '{line}' '{survexblock.date}'")
+            elif line[-5] == "-" or line[-5] == ".":
+                line = line[:-4] + '0' + line[-4:]
+                survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m-%d") 
+                print(f"! DATE -5 '{line}' '{survexblock.date}'")
+            else:  
+                year = line[:4]
+                message = (
+                    f"! DATE Error SHORT LINE '{line}' '{oline}-{survexblock}' ({type(survexblock)}) {survexblock.survexfile.path}"
+                )
+                print(self.insp + message)
+                stash_data_issue(
+                    parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path)
+                )       
         else:
             # these errors are reporting the wrong survexblock, which is actually a SurvexFile (!)
             # see To Do notes on how to trigger this. Still needs investigating..
@@ -737,9 +762,16 @@ class LoadingSurvex:
             print(f"  {type(survexblock)=}")  # survexblock.parent fails as a SurvexFile has no .parent ...ugh.
             print(f"  {survexblock.survexpath=}")
             print(f"  {survexblock.survexfile=}")
+            # Not setting 'year' crashes entire import on databaseReset.
+            year = line[:4]
+            perps = get_people_on_trip(survexblock)
             # raise
             
-        setdate_on_survexblock(year)
+        try: 
+            setdate_on_survexblock(year)
+        except NameError:
+            print(f">> why is year not set ?! {survexblock.survexfile.path}")
+            setdate_on_survexblock("1976")
         if survexblock.date:
             # do not actually need a distict variable 'currentdate' but it makes the code clearer
             self.currentdate = survexblock.date
@@ -1377,7 +1409,13 @@ class LoadingSurvex:
             survexblock.survexfile.cave.slug()
             
         self.fix_undated(survexblock) # null-op if already set
-        expoyear = str(survexblock.date.year)
+        try: 
+            expoyear = str(survexblock.date.year)
+        except:
+            print(f">> why is survexblock not set ?! in LoadSurvexQM()/n {survexblock.survexfile.path}")
+            expoyear = "1970"
+
+        
 
         try:
             qm = QM.objects.create(

templates/expedition.html

@@ -37,7 +37,7 @@ an "<b>S</b>" for a survey trip.  The colours of the "<b>T</b>" and "<b>S</b>" a
 <th>Caver</th>
 {% for d in dates %}
 <th>
-  {{d.day}}
+  {{d.day}}/{{d.month}}
 </th>
 {% endfor %}
 </tr>

templates/menu.html

@@ -38,7 +38,7 @@
 <ul><li><a href="/years/{{ year }}">{{ year }}</a></li></ul>
 {% endif %}
 <li><a href="/guidebook/areas.htm">Areas</a></li>
-<li><a href="/expedition/2022">Troggle</a></li>
+<li><a href="/expedition/2023">Troggle</a></li>
 <li><form name=P method=get action="/search" target="_top">
     <input id="omega-autofocus" type=search name=P size=8 autofocus>
     <input type=submit value="Search"></form></li>