wallets & caves now many:many

core/models/caves.py

@@ -88,7 +88,7 @@ class Cave(TroggleModel):
     url = models.CharField(max_length=300, blank=True, null=True, unique = True)
 
     class Meta:
-        # we do not enforce uniqueness at the db level as that causes confusing errors for users.
+        # we do not enforce uniqueness at the db level as that causes confusing errors for newbie maintainers
         #    unique_together = (("area", "kataster_number"), ("area", "unofficial_number")) 
         ordering = ("kataster_code", "unofficial_number")
 

core/models/wallets.py

@@ -10,6 +10,8 @@ from django.conf import settings
 from django.db import models
 from django.urls import reverse
 
+from troggle.core.views.caves import get_cave_leniently
+
 # from troggle.core.models.survex import SurvexBlock
 # from troggle.core.models.troggle import DataIssue # circular import. Hmm
 
@@ -74,12 +76,15 @@ archaic_wallets = [
 class Wallet(models.Model):
     """We do not keep the JSON values in the database, we query them afresh each time,
     but we may change this if we need to do a Django query on e.g. personame
+    
+    ManyToMany field uses modern Django: a hidden Class, unlike CaveAndEntrances which is explict and visible.
     """
 
     fpath = models.CharField(max_length=200)
     walletname = models.CharField(max_length=200)
     walletdate = models.DateField(blank=True, null=True)
     walletyear = models.DateField(blank=True, null=True)
+    caves =  models.ManyToManyField("Cave", related_name="wallets")
 
     class Meta:
         ordering = ("walletname",)
@@ -139,7 +144,31 @@ class Wallet(models.Model):
                             from troggle.core.models.troggle import DataIssue
                             DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
         return waldata
-
+        
+    def allcaves(self):
+        """Reads all the JSON data just to get the JSON date."""
+        if not (jsondata := self.get_json()): # WALRUS
+            print(f"Failed to read JSON file {self}")
+            return None
+        cavelist = jsondata["cave"]
+        if type(cavelist) is list:
+                for i in cavelist:
+                    if i != "":
+                        i = i.replace("/", "-")
+                        caveobject = get_cave_leniently(i)  
+                        self.caves.add(caveobject) # new many-to-many field
+        else:
+            # either single cave or the square barckets have been removed and it s a singoe string
+            ids = cavelist.split(",")
+            for i in ids:
+                j = i.replace("'","").replace("/", "-").strip('[] "')
+                if i != "":
+                    try:
+                        caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data
+                        self.caves.add(caveobject)
+                    except: 
+                        print(f"FAIL adding cave to wallet.caves  '{j}'")
+                        pass
     def year(self):
         """This gets the year syntactically without opening and reading the JSON"""
         if len(self.walletname) < 5:
@@ -180,6 +209,7 @@ class Wallet(models.Model):
             self.save()
             return self.walletdate
 
+    # for gods sake redo this, it parse JSON twice every time..
     def people(self):
         if not self.get_json():
             return None

core/views/caves.py

@@ -40,6 +40,20 @@ todo = """
   https://zerotobyte.com/how-to-use-django-select-related-and-prefetch-related/
 """
 
+def get_cave_leniently(caveid):
+    try:
+        c = getCave(caveid)
+        if c:
+            return c
+    except:
+        # print(f"get_cave_leniently FAIL {caveid}")
+        try:
+            c = getCave("1623-"+caveid)
+            if c:
+                return c 
+        except:
+            return None
+    
 
 def getCaves(cave_id):
     """Only gets called if a call to getCave() raises a MultipleObjects exception
@@ -59,7 +73,8 @@ def getCaves(cave_id):
 
 
 def getCave(cave_id):
-    """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm.
+    """Returns a cave object when given a cave name or number. 
+    It is used by views including cavehref, ent, wallets and qm.
 
     TO DO: search GCavelookup first, which should raise a MultpleObjectsReturned exception if there
     are duplicates"""

core/views/scans.py

@@ -52,20 +52,15 @@ def populatewallet(w):
 
 
 def caveifywallet(w):
-    """Gets the cave from the list of survex files,
+    """Gets the caves from the list of survex files,
-    only selects one of them though. Only used for display.
+     """
-    FIX THIS to display many caves
-    """
     # print(f' - Caveify {w=}')
     blocknames = []
     blocks = SurvexBlock.objects.filter(scanswallet=w)
     for b in blocks:
         # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath
         if b.survexfile.cave:
-            w.caveobj = (
+            w.caves.add(b.survexfile.cave)
-                b.survexfile.cave
-            )  # just gets the last one, randomly. SHould make this a list or many:many ideally
-            w.cave = w.caveobj
         if b.name:
             blocknames.append(b.name)
 
@@ -120,11 +115,10 @@ def is_cave(wallet, id):
     if not id:
         return False
     Gcavelookup = GetCaveLookup()
-    id = id.strip("' []'")
     if id in Gcavelookup:
         return True
     else:
-        # Historic wallets used just 2 or 3 digits and were all 1623 area. So, just for these wallets, 
+        # Historic wallets  were all 1623 area. So, just for wallets, 
         # assume it is 1623-xxx
         if f"1623-{id}" in Gcavelookup:
             print(f" - Should modify wallet {wallet} to use 1623- prefix for cave <{id}>")
@@ -145,26 +139,26 @@ def fillblankothers(w):
 
     Gcavelookup = GetCaveLookup()
 
+    caveifywallet(w)
+    
     wcaveid = w.cave()
     if not wcaveid or wcaveid == "":
-        caveifywallet(w)
-    else:
         if type(wcaveid) == list:
             for i in wcaveid:
-                i = i.strip("' []'")
+                i = i.strip("' []\"")
                 if is_cave(w,i):
-                    w.caveobj = Gcavelookup[i] # just sets it to the last one found. nasty. bug waiting to happen
+                    w.caves.add(Gcavelookup[i])
         elif wcaveid.find(',') != -1:
             # it's a list of cave ids as a string
             ids = wcaveid.split(',')
             for i in ids:
-                i = i.strip("' []'")
+                i = i.strip("' []\"")
                 if is_cave(w,i):
-                    w.caveobj = Gcavelookup[i]  # just sets it to the last one found. nasty. bug waiting to happen     
+                    w.caves.add(Gcavelookup[i])
         else:
             if is_cave(w,wcaveid):
-                w.caveobj = Gcavelookup[wcaveid.strip("' []'")]
+                w.caves.add(Gcavelookup[i])
-   
+  
                 
 def fixsurvextick(w, ticks):
     ticks["S"] = w.fixsurvextick(ticks["S"])

core/views/wallets_edit.py

@@ -23,7 +23,7 @@ from troggle.core.models.troggle import DataIssue, Expedition
 from troggle.core.models.wallets import Wallet, YEAR_RANGE, make_valid_date
 
 from troggle.core.views.auth import login_required_if_public
-from troggle.core.views.caves import getCave
+from troggle.core.views.caves import getCave, get_cave_leniently
 from troggle.core.views.scans import caveifywallet, oldwallet
 from troggle.core.views.uploads import FilesForm
 
@@ -108,7 +108,8 @@ xlate = {
     "survex": "survex file",
 }
 
-
+    
+    
 def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
     """Taken from old script wallets.py and edited to make more comprehensible
     Loads the survex files names and processes all complaints
@@ -231,20 +232,29 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
         )
 
     # Find the cave, if it exists
+    # Just for wallets, we are lenient about whether the 1623- prefix has been written down.
     if waldata["cave"]:
+        caveobject = None
         try:
             caveid = waldata["cave"]
             if type(caveid) is list:
                 for i in caveid:
                     i = i.replace("/", "-")
-                    caveobject = getCave(i)  # only the last one gets recorded.. ouch.
+                    caveobject = get_cave_leniently(i)  
+                    w.caves.add(caveobject) # new many-to-many field
+                    #print(w.caves)
             else:
-                caveid = caveid # ?urk? why?
+                # either single cave or the square barckets have been removed
-                try:
+                ids = caveid.split(",")
-                    caveobject = getCave(caveid) # may fail if garbage value ,e.g. space, in wallet data
+                for i in ids:
-                except: 
+                    j = i.replace("'","").strip('[] "')
-                    caveobject = None
+                    #print(j)
-            print(f'getCave for id "{waldata["cave"]}" {caveobject}')
+                    try:
+                        caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data
+                        w.caves.add(caveobject)
+                    except: 
+                        pass
+            print(f'get_cave_leniently from "{waldata["cave"]}" => {caveobject}')
             # if not caveobject.url == waldata["description url"]:
             # complaints.append(f'The URL of cave description \"{waldata["description url"]}\" does not match the one on record for this cave which is: "{caveobject.url}". If the wallet is not for a cave, put a useful URL here.')
         except Cave.MultipleObjectsReturned:

parsers/caves.py

@@ -443,6 +443,7 @@ def validate_station(station):
     if dot == -1:
         print(dot)
         # no full stop found. Bad station identifier.
+        # should just skip really, and log an error
         raise
     else:
         return True

parsers/scans.py

@@ -24,6 +24,9 @@ def set_JSONwalletdate(wallet):
     we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here."""
     _ = wallet.date()  # don't need return value. Sets .walletdate as side effect
 
+def set_caves(wallet):
+     _ = wallet.allcaves()  # don't need return value. Just calling this saves it as w.caves This ONLY gets the list on the wallet JSON
+
 def load_all_scans():
     """This iterates through the scans directories (either here or on the remote server)
     and builds up the models we can access later.
@@ -114,6 +117,7 @@ def load_all_scans():
                     wallet = Wallet(fpath=fpath, walletname=walletname)
                     set_walletyear(wallet)
                     wallet.save()
+                    set_caves(wallet)
                     wallets[walletname] = wallet
 
                 singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet)
@@ -159,6 +163,7 @@ def load_all_scans():
                         # BUT can't check linked survex blocks as they haven't been imported yet
                         set_JSONwalletdate(wallet)
                         set_walletyear(wallet)
+                        set_caves(wallet)
                         if not created:
                             print(
                                 f"\n  - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?"

templates/personwallets.html

@@ -23,7 +23,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
 <p>Note that names in italics are copied from the related survex file block name.
 <br />
 <table width=95%>
-<tr><th>Wallet</th><th width=13%>Wallet Date</th><th>Wallet Name</th><th width=28%>Team</th><th width=8%>>Cave</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr>
+<tr><th>Wallet</th><th width=13%>Wallet Date</th><th>Wallet Name</th><th width=28%>Team</th><th width=8%>Cave(s)</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr>
 {% for wallet in manywallets|dictsort:"walletname" %}
   <tr>
     <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
@@ -37,15 +37,11 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
     {%for p in wallet.persons%}
         <a href="/person/{{p.slug}}">{{p.fullname}}</a>{%if not forloop.last %}, {% endif %}
         {% endfor %}</td>
-    <td style="padding:2px">
+    <td style="padding:2px; font-family: monospace; font-size: 90%;">
-    {% if wallet.cave %}
+    {% for c  in wallet.caves.all %}
-        {% if wallet.caveobj.slug %}
+        <a href="/cave/scans/{{c.slug}}">{{c}}</a>
-            <a href="/cave/scans/{{wallet.caveobj.slug}}">{{wallet.cave}}</a>
+    {% endfor %}
-        {% else %}
+    </td>
-            {{wallet.cave}}
-        {% endif %}
-    {% else %} 
-    {% endif %}</td>
     
     <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td>
     <td style="padding:2px">

templates/wallet_table.html

@@ -1,6 +1,6 @@
 <!-- this is an INCLUDED template wallet_table.html-->
 <table width=95%>
-<tr><th>Wallet</th><th width=15%>Wallet Date</th><th>Cave</th><th>Wallet Name</th><th>Survex survey length</th>
+<tr><th>Wallet</th><th width=15%>Wallet Date</th><th>Cave(s)</th><th>Wallet Name</th><th>Survex survey length</th>
 
 <!-- survex file-->
 <th style="font-family: monospace; font-size: 150%;" title="Survex file exists">S</th>
@@ -22,15 +22,11 @@
     <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
     
     <td style="padding:2px">{% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %}</td>
-    <td style="padding:2px">
+    <td style="padding:2px; font-family: monospace; font-size: 90%;">
-    {% if wallet.cave %}
+    {% for c  in wallet.caves.all %}
-        {% if wallet.caveobj.slug %}
+        <a href="/cave/scans/{{c.slug}}">{{c}}</a>
-            <a href="/cave/scans/{{wallet.caveobj.slug}}">{{wallet.cave}}</a>
+    {% endfor %}
-        {% else %}
+    </td>
-            {{wallet.cave}}
-        {% endif %}
-    {% else %} 
-    {% endif %}</td>
     <td style="padding:2px">{% if wallet.name %}{{wallet.name|truncatechars:20}}{% else %}<em>{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}</em>{% endif %}</td>
     <td style="padding:2px" align=center>
        {% for survexblock in wallet.survexblock_set.all %}

templates/walletform.html

@@ -149,12 +149,13 @@ and <em>also</em> the exported files in standard formats: svx, svg etc. See why
 
     <br>
     <span style="font-family: monospace;  font-size: 150%; ">
-    {% if cave %}<u>Cave ID</u>: 
+    {% for wallet in manywallets %}
-        {% if caveobject %}<b>{{cave}}</b></a> which implies "<a href="/{{caveobject.url|urlencode}}">{{caveobject}}</a>"<br> 
+     <u>Cave IDs</u>: 
-        {% else %}
+
-        <b>{{cave}}</b><br> 
+     {% for c in wallet.caves.all %}
-        {% endif %}   
+        <a href="/cave/scans/{{c.slug}}">{{c}}</a>
-    {% endif %}
+     {% endfor %}<br />
+
     {% if psg %}<u>Survey area</u>: <b>{{psg}}</b><br>{% endif %}
     {% if svxfiles %}<u>Survex files</u>: 
             {% for svx in svxfiles %}
@@ -163,7 +164,7 @@ and <em>also</em> the exported files in standard formats: svx, svg etc. See why
             <br>
     {% endif %}
 
-  
+    {% endfor %}
     </span>
     <span style="font-family: monospace;  font-size: 130%; ">
     

templates/yearwallets.html

@@ -32,7 +32,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
 <p>Total underground survey length: {{length_ug|floatformat:"1g"}} m
 <br />
 <table width=95%>
-<tr><th>Wallet</th><th width=13%>Wallet Date</th><th>Wallet Name</th><th width=25%>People</th><th width=8%>Cave</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr>
+<tr><th>Wallet</th><th width=13%>Wallet Date</th><th>Wallet Name</th><th width=25%>People</th><th width=8%>Cave(s)</th><th>Scans</th><th>Survex blocks</th><th>Drawings using these scans</th></tr>
 {% for wallet in manywallets|dictsort:"walletname" %}
   <tr>
     <td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
@@ -46,15 +46,12 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c
     {%for p in wallet.persons%}
         <a href="/person/{{p.slug}}">{{p.fullname}}</a>{%if not forloop.last %}, {% endif %}
         {% endfor %}</td>
-    <td style="padding:2px">
+
-    {% if wallet.cave %}
+    <td style="padding:2px; font-family: monospace; font-size: 90%;">
-        {% if wallet.caveobj.slug %}
+    {% for c  in wallet.caves.all %}
-            <a href="/cave/scans/{{wallet.caveobj.slug}}">{{wallet.cave}}</a>
+        <a href="/cave/scans/{{c.slug}}">{{c}}</a>
-        {% else %}
+    {% endfor %}
-            {{wallet.cave}}
+    </td>
-        {% endif %}
-    {% else %} 
-    {% endif %}</td>
     
     <td align="center" style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.singlescan_set.all|length}}</a></td>
     <td style="padding:2px">