findng variant data on survexblocks and setting wallet *ref

parsers/survex.py

@@ -2484,19 +2484,41 @@ def MakeFileRoot(svxpath):
     return fileroot
 
 def set_survexblocks(wallet):
-    if svxfiles := wallet.survexfiles():  # reads from JSON, should be cached
+    """Need to find the optimal Django way of doing this query. 
+    It's a mess now"""
+    if svxfiles := wallet.survexfiles():  # reads from JSON, should be cached already
         for svx  in svxfiles:
-            # svx is a string, need to resolve to a survexfile object
-            #o = SurvexFile.objects.get(path=svx)
-            blocks = SurvexBlock.objects.filter(survexfile__path=svx)
-            for b in blocks:
-                if b.scanswallet == wallet:
-                    pass
-                else:
-                    b.scanswallet = wallet
-                    b.save()
-                    # print(f"setting {wallet} on {b.survexfile} : {b}")
-
+            if svx:
+                if svx.endswith(".svx"):
+                    svx = svx.replace(".svx","")
+                try:
+                    # there are survex files we ignore when troggle parses, and some of these are referred to in wallets
+                    sfile = SurvexFile.objects.get(path=svx) #.select_related("survexblocks")
+                    # print(sfile)
+                except:
+                    continue
+                blocks = SurvexBlock.objects.filter(survexfile=sfile)
+                for b in blocks:
+                    try:
+                        if b.scanswallet == wallet:
+                            pass
+                        elif b.scanswallet:
+                            if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1):
+                                print(f"not set{wallet} on {b.survexfile} : {b} as already set to {b.scanswallet}")
+                        else:
+                                b.scanswallet = wallet
+                                b.save()
+                                if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1):
+                                    print(f"setting {wallet} on {b.survexfile} : {b}")
+                    except: 
+                        if not hasattr(b,"date"):
+                            print(f" Block {b} on {b.survexfile} HAS NO DATE SET ")
+                        elif not b.date:
+                            print(f" Block {b} on {b.survexfile} HAS NULL DATE  ")
+                        else:
+                            print(f" exception {wallet} on {b.survexfile} : {b}")
+                            
+ 
 def survexifywallets():
     """Gets the caves from the list of survexblocks
     
@@ -2514,7 +2536,7 @@ def survexifywallets():
             w.persons.add(spr.person)
        
     duration = time.time() - start
-    print(f" -  TIME: add people to wallets {duration:7.2f} s", file=sys.stderr)
+    print(f" -  {duration:7.2f} s to add people to wallets ", file=sys.stderr)
     start = time.time()
                 
     wallets = Wallet.objects.all()
@@ -2522,7 +2544,7 @@ def survexifywallets():
         set_survexblocks(w) # reads JSON, sets survexblocks if survexfiles specified on wallet JSON
    
     duration = time.time() - start
-    print(f" -  TIME: set survexblock:wallet using JSON survexfiles {duration:7.2f} s", file=sys.stderr)
+    print(f" -  {duration:7.2f} s to set survexblock:wallet using JSON survexfiles ", file=sys.stderr)
     start = time.time()
     
     for w in wallets:
@@ -2533,15 +2555,16 @@ def survexifywallets():
         w.save()
 
     duration = time.time() - start
-    print(f" -  TIME: add caves to wallets {duration:7.2f} s", file=sys.stderr)
+    print(f" -  {duration:7.2f} s to add caves to wallets ", file=sys.stderr)
     start = time.time()
     
         
-
+    # Find the survex blocks which are 'ours' i.e. ignore all those (ARGE etc) without expo people attached.
     cuccblocks = set()
     for spr in SurvexPersonRole.objects.all():
         cuccblocks.add(spr.survexblock)
         
+    # Because we have just run set_survexblocks(w), this should only complain if there is no *ref and no wallet that links to its parent file
     sentinelbad = Wallet.objects.get(walletname="1983#00")
     for b in cuccblocks:
         if b.date > date(2001, 1, 1): # do we care about older ones? 1999 certainly has different wallet system
@@ -2551,12 +2574,13 @@ def survexifywallets():
                         b.scanswallet = b.parent.scanswallet
                         continue
                 message = f"  ! *REF missing {b.date} {b.survexfile}.svx : '{b}'"
-                # print(message, file=sys.stderr)
+                if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1):
+                    print(message, file=sys.stderr)
                 url = get_offending_filename(b.survexfile.path)
                 DataIssue.objects.update_or_create(parser="ref", message=message, url=url)
 
     duration = time.time() - start
-    print(f" -  TIME: check missing *ref on survexblocks  {duration:7.2f} s", file=sys.stderr)
+    print(f" -  {duration:7.2f} s to check missing *ref on survexblocks  ", file=sys.stderr)
     start = time.time()