checking wallets earlier int he process

core/models/wallets.py

@@ -10,6 +10,7 @@ from django.conf import settings
 from django.db import models
 from django.urls import reverse
 
+from troggle.core.models.troggle import DataIssue
 from troggle.core.views.caves import get_cave_leniently
 
 # from troggle.core.models.survex import SurvexBlock
@@ -143,8 +144,26 @@ class Wallet(models.Model):
                             message = f"! {str(self.walletname)} Date format not ISO {waldata['date']}. Failed to load from {jsonfile} JSON file"
                             from troggle.core.models.troggle import DataIssue
                             DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
+                            
+ 
         return waldata
-        
+
+    def check_survexlist(self):
+        wurl = f"/walletedit/{self.walletname}".replace('#', ':')
+        if not (waldata := self.get_json()): # WALRUS
+            return None
+        if waldata["survex file"]:
+            if not type(waldata["survex file"]) == list:  # a string also is a sequence type, so do it this way
+                waldata["survex file"] = [waldata["survex file"]]
+            for sx in waldata["survex file"]:
+                # this logic appears in several places, inc get_ticks(). and wallets_edit.py Refactor.
+                if sx != "":
+                    if Path(sx).suffix.lower() != ".svx":
+                        sx = sx + ".svx"
+                    if not (Path(settings.SURVEX_DATA) / sx).is_file():
+                        message=f"{self} Survex file {sx} was not found in LOSER repo"
+                        DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
+    
     def allcaves(self):
         """Reads all the JSON data just to get the JSON date."""
         if not (jsondata := self.get_json()): # WALRUS
@@ -209,23 +228,19 @@ class Wallet(models.Model):
             self.save()
             return self.walletdate
 
-    # for gods sake redo this, it parse JSON twice every time..
     def people(self):
-        if not self.get_json():
+        if not (jsondata := self.get_json()): # WALRUS
             return None
-        jsondata = self.get_json()
         return jsondata["people"]
 
     def cave(self):
-        if not self.get_json():
+        if not (jsondata := self.get_json()): # WALRUS
             return None
-        jsondata = self.get_json()
         return jsondata["cave"]
 
     def name(self):
-        if not self.get_json():
+        if not (jsondata := self.get_json()): # WALRUS
             return None
-        jsondata = self.get_json()
         return jsondata["name"]
 
     def get_fnames(self):

core/views/caves.py

@@ -405,6 +405,7 @@ def edit_cave(request, path="", slug=None):
         if form.is_valid(): # and ceFormSet.is_valid():
             # print(f'! POST is valid. {cave}')
             cave = form.save(commit=False)
+            print(cave)
             if not cave.filename:
                 cave.filename = cave.areacode + "-" + cave.number() + ".html"
             if not cave.url:

core/views/wallets_edit.py

@@ -156,7 +156,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
                     message = f"! {file_complaint}"
                     print(message)
                     DataIssue.objects.update_or_create(
-                        parser="scans", message=message, url=wurl
+                        parser="wallets", message=message, url=wurl
                     )  # set URL to this wallet folder
                 else:
                     try:
@@ -165,6 +165,10 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
                     except MultipleObjectsReturned:
                         # can happen if connecting a wallet to a survex file.. i think..
                         QSsvxfiles = SurvexFile.objects.filter(path=sxpath)
+                        message = f"! {wallet} Urk, multiple SurvexFile objects {sxpath}"
+                        DataIssue.objects.update_or_create(
+                            parser="wallets", message=message, url=wurl
+                        )  # set URL to this wallet folder
                         for s in QSsvxfiles:
                             print(s.path, s.cave, s.primary)
                         # QSsvxfiles[0]  # dont' know how this happened, fix later..
@@ -176,7 +180,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
                         message = f"! {file_complaint}"
                         print(message)
                         DataIssue.objects.update_or_create(
-                            parser="scans", message=message, url=wurl
+                            parser="wallets", message=message, url=wurl
                         )  # set URL to this wallet folder
 
     if waldata["survex not required"] and waldata["survex file"] != [""]:

parsers/scans.py

@@ -17,7 +17,7 @@ git = settings.GIT
 
 
 def set_walletyear(wallet):
-    _ = wallet.year()  # don't need return value. Just calling this saves it as w.walletyear
+    _ = wallet.year()  # don't need return value. Just calling this saves it as w.walletyear Syntactic.
 
 def set_JSONwalletdate(wallet):
     """At this point in the import process, the survex files have not been imported so
@@ -30,12 +30,9 @@ def set_caves(wallet):
 def load_all_scans():
     """This iterates through the scans directories (either here or on the remote server)
     and builds up the models we can access later.
-
-    It does NOT read or validate anything in the JSON data attached to each wallet. Those checks
-    are done at runtime, when a wallet is accessed, not at import time.
     
-    Loads people as a simple string of fullnames. We should replace this with a list of Person slugs.
+    Loads people as a simple string of fullnames. We should replace this with a list of Person slugs,
-
+    and change the wallet editor to save People as slugs.
     """
     print(" - Loading Survey Scans")
 
@@ -115,11 +112,11 @@ def load_all_scans():
                     print("", flush=True, end="")
                     # Create the wallet object. But we don't have a date for it yet.
                     wallet = Wallet(fpath=fpath, walletname=walletname)
+                    wallets[walletname] = wallet
                     set_walletyear(wallet)
                     wallet.save()
                     set_caves(wallet)
-                    wallets[walletname] = wallet
+ 
-
                 singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet)
                 singlescan.save()
 
@@ -160,7 +157,6 @@ def load_all_scans():
                         wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
                         wallets[walletname] = wallet
                         # Now also load the json 
-                        # BUT can't check linked survex blocks as they haven't been imported yet
                         set_JSONwalletdate(wallet)
                         set_walletyear(wallet)
                         set_caves(wallet)
@@ -170,8 +166,17 @@ def load_all_scans():
                             )
                             wallet.save()
     print(f"\n  - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets")
-    
+
     # Only the 1999 wallets have filenames which mean that the walletyear will be unset:
     wallets = Wallet.objects.filter(walletyear=None)
     for w in wallets:
-        w.walletyear = datetime.date(1999, 1, 1)
+        w.walletyear = datetime.date(1999, 1, 1)    
+       
+    # Although the survex files haven't been processed yet, we can at least check if the wallets refer to a real file or not
+    for wallet in Wallet.objects.all():
+        # this reads JSON
+        wallet.check_survexlist()
+
+        
+
+   

templates/manywallets.html

@@ -27,28 +27,37 @@ otherwise they come from *ref statements in survex files as of the most recent d
 <!-- This should all be restructured to use .prefetch_related() and .select_related()
 see https://docs.djangoproject.com/en/dev/ref/models/querysets/#prefetch-related
 -->
+
 <table width=95%>
-<tr><th>Scans folder</th><th>Files</th><th>Survex blocks</th><th>Cave</th></tr>
+<tr><th>Scans folder</th><th>Files</th><th>Survex blocks</th><th>Cave (new)</th><th>Cave (old)</th></tr>
-{% for scanswallet in manywallets %}
+{% for wallet in manywallets %}
   <tr>
-    <td style="padding:2px"><a href="{{scanswallet.get_absolute_url}}">{{scanswallet.walletname}}</a></td>
+    <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
-    <td align="right" style="padding:2px">{{scanswallet.singlescan_set.all|length}}</td>
+    <td align="right" style="padding:2px">{{wallet.singlescan_set.all|length}}</td>
     <td style="padding:2px">
-    {% for survexblock in scanswallet.survexblock_set.all %}
+    {% for survexblock in wallet.survexblock_set.all %}
       <a href="{% url "svx" survexblock.survexfile.path %}">{{survexblock}}</a>
     {% endfor %}
     </td>
+
+    <td style="padding:2px; font-family: monospace; font-size: 90%;">
+    {% for c  in wallet.caves.all %}
+        <a href="/cave/scans/{{c.slug}}">{{c}}</a>
+    {% endfor %}
+    </td>
+
     <td style="padding:2px">
-    {% for survexblock in scanswallet.survexblock_set.all %}
+    {% for survexblock in wallet.survexblock_set.all %}
         {% ifchanged survexblock.survexfile.cave %}
         <a href="/cave/scans/{{survexblock.survexfile.cave.slug}}">{{survexblock.survexfile.cave.slug}}</a>
         {% endifchanged %}
     {% empty %}
-    {% if scanswallet.cave %}
+    {% if wallet.cave %}
-        <em><a href="/cave/scans/{{scanswallet.cave}}">{{scanswallet.cave}}</a></em>
+        <em><a href="/cave/scans/{{wallet.cave}}">{{wallet.cave}}</a></em>
     {% endif %}
     {% endfor %}
     </td>
+
   </tr>
 {% endfor %}
 </table>