refactoring duplicate code

core/models/wallets.py

@@ -105,6 +105,29 @@ class Wallet(models.Model):
     class Meta:
         ordering = ("walletname",)
 
+    @staticmethod
+    def input_to_list(stuff):
+        """With wallets we often have to deal with either a list object (from valid JSON parsing)
+        or a string which may or may not also be a list, but munged by user eror on a form etc.
+        This function returns a list, either the JSON list, or a list with a single object in it,
+        or a list of strings. It silently absorbs empty strings an consumes odd quotes and square
+        brackets. 
+        Always return a list, even if it is an empty list"""
+        if type(stuff) == list:
+            newstuff = []
+            for o in stuff:
+                if o: # not an empty string, None
+                   newstuff.append(o) 
+            return newstuff
+        if type(stuff) == str:
+            newstuff = stuff.split(",")
+            for s in newstuff:
+                s = s.strip('[] ').replace("'","").replace('"','').replace("/", "-").replace(" ", "_").strip('[] ')
+            return newstuff
+        if stuff: 
+            return [stuff] # single object, not a string, but now in a list.
+ 
+    
     def get_absolute_url(self):
         # we do not use URL_ROOT any more.    
         return reverse("singlewallet", kwargs={"path": re.sub("#", "%23", self.walletname)})
@@ -114,6 +137,8 @@ class Wallet(models.Model):
         Do it every time it is queried, to be sure the result is fresh.. well, no.
         Do it every time we have a new python instance.
         
+        Reads JSON date and sets w.walletdate
+        
         import DataIssue locally to prevent import cycle problem"""
         
         if hasattr(self, "JSONdata"):
@@ -170,8 +195,7 @@ class Wallet(models.Model):
         if not (waldata := self.get_json()): # WALRUS
             return None
         if waldata["survex file"]:
-            if not type(waldata["survex file"]) == list:  # a string also is a sequence type, so do it this way
+            waldata["survex file"] = Wallet.input_to_list(waldata["survex file"])
-                waldata["survex file"] = [waldata["survex file"]]
             for sx in waldata["survex file"]:
                 # this logic appears in several places, inc get_ticks(). and wallets_edit.py Refactor.
                 if sx != "":
@@ -182,28 +206,39 @@ class Wallet(models.Model):
                         DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
     
     def allcaves(self):
-        """Reads all the JSON data just to get the JSON date."""
+        """Called when parsing importing all data. Called on all new wallets, but before
+        the survex files are parsed"""
         if not (jsondata := self.get_json()): # WALRUS
              return None
-        cavelist = jsondata["cave"]
+        #cavelist = jsondata["cave"]
-        if type(cavelist) is list:
+        cavelist = Wallet.input_to_list(jsondata["cave"])
-                for i in cavelist:
+        for i in cavelist:
-                    if i != "":
+            try:
-                        i = i.replace("/", "-")
+                caveobject = get_cave_leniently(i)
-                        caveobject = get_cave_leniently(i)  
+                if caveobject:
-                        self.caves.add(caveobject) # new many-to-many field
+                    self.caves.add(caveobject)
-        else:
+            except: 
-            # either single cave or the square barckets have been removed and it s a singoe string
+                print(f"FAIL adding cave to wallet.caves  '{i}'")
-            ids = cavelist.split(",")
+                pass
-            for i in ids:
+            
-                j = i.replace("'","").replace("/", "-").strip('[] "')
+        # if type(cavelist) is list:
-                if i != "":
+                # for i in cavelist:
-                    try:
+                    # if i != "":
-                        caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data
+                        # i = i.replace("/", "-")
-                        self.caves.add(caveobject)
+                        # caveobject = get_cave_leniently(i)  
-                    except: 
+                        # self.caves.add(caveobject) # new many-to-many field
-                        print(f"FAIL adding cave to wallet.caves  '{j}'")
+        # else:
-                        pass
+            # # either single cave or the square barckets have been removed and it s a singoe string
+            # ids = cavelist.split(",")
+            # for i in ids:
+                # j = i.replace("'","").replace("/", "-").strip('[] "')
+                # if i != "":
+                    # try:
+                        # caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data
+                        # self.caves.add(caveobject)
+                    # except: 
+                        # print(f"FAIL adding cave to wallet.caves  '{j}'")
+                        # pass
     def year(self):
         """This gets the year syntactically without opening and reading the JSON"""
         if len(self.walletname) < 5:

core/views/scans.py

@@ -174,23 +174,17 @@ def parse_name_list(w):
     crew = GetPersonExpeditionNameLookup(expo)
     
     for n in namelist:
-        # if n.lower().startswith("lydia"):
-            # print(f"{w} {n=} ")
-            # for x in crew:
-                # if x.lower()==n.lower():
-                    # print(f"{w} {n=} {x=}")
-
         if n.lower() in crew:
             peeps.add(crew[n.lower()].person)
         else:
-            if n.startswith("*"): #ignore people flagged as guests or not-expo anyway, such as ARGE
+            if check := n.startswith("*"): #ignore people flagged as guests or not-expo anyway, such as ARGE
                 continue
             nobod = n.lower()
             if nobod == "unknown" or nobod == "nobody" or nobod == " " or nobod == "":
                 continue
             else:
                 wurl = f"/walletedit/{w.walletname.replace('#',':')}"
-                message = f"{w} name '{n.lower()}' NOT found in GetPersonExpeditionNameLookup({w.year()}) ?!"
+                message = f"{w} name '{n.lower()}' NOT found in GetPersonExpeditionNameLookup({w.year()}) n.startswith* = {check} ?!"
                 print(message)
                 DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)    
     return peeps

parsers/scans.py

@@ -13,15 +13,14 @@ contentsjson = "contents.json"
 
 git = settings.GIT
 
-# to do: Actually read all the JSON files and set the survex file field appropriately!
-
 
 def set_walletyear(wallet):
     _ = wallet.year()  # don't need return value. Just calling this saves it as w.walletyear Syntactic.
 
 def set_JSONwalletdate(wallet):
     """At this point in the import process, the survex files have not been imported so
-    we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here."""
+    we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here.
+    Actually, doing anything that reads the JSON sets .walletdate"""
     _ = wallet.date()  # don't need return value. Sets .walletdate as side effect
 
 def set_caves(wallet):
@@ -111,11 +110,11 @@ def load_all_scans():
                 else:
                     print("", flush=True, end="")
                     # Create the wallet object. But we don't have a date for it yet.
-                    wallet = Wallet(fpath=fpath, walletname=walletname)
+                    wallet = Wallet.objects.create(walletname=walletname, fpath=fpath)
                     wallets[walletname] = wallet
                     set_walletyear(wallet)
-                    wallet.save()
                     set_caves(wallet)
+                    wallet.save()
  
                 singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet)
                 singlescan.save()
@@ -156,15 +155,13 @@ def load_all_scans():
                         # The wallets found from JSON should all have dates already
                         wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
                         wallets[walletname] = wallet
-                        # Now also load the json 
-                        set_JSONwalletdate(wallet)
                         set_walletyear(wallet)
                         set_caves(wallet)
+                        wallet.save()
                         if not created:
                             print(
                                 f"\n  - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?"
                             )
-                            wallet.save()
     print(f"\n  - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets")
 
     # Only the 1999 wallets have filenames which mean that the walletyear will be unset:

templates/base.html

@@ -47,7 +47,7 @@
     
     <a href="{% url "dataissues" %}">Data Issues</a> |
     <a href="/handbook/computing/todo-data.html">tasks to do </a>  |
-    <a id="entsLink" href="{% url "eastings" %}">ents</a>  |
+    <a id="entsLink" href="{% url "entranceindex" %}">ents</a>  |
     <a id="folklink" href="/folk">expoers</a>  |
     <a id="caversLink" href="{% url "notablepersons" %}">survey lengths</a>  |
     <a href="{% url "stats" %}">statistics</a> |