findng variant data on survexblocks and setting wallet *ref

core/models/survex.py

@@ -161,7 +161,7 @@ class SurvexBlock(models.Model):
 
     date = models.DateField(blank=True, null=True)
     expedition = models.ForeignKey("Expedition", blank=True, null=True, on_delete=models.SET_NULL)
-    # if the survexfile object is deleted, then all the suvex-blocks in it should be too,
+    # if the survexfile object is deleted, then all the survex-blocks in it should be too,
     # though a block can span more than one file...
     survexfile = models.ForeignKey("SurvexFile", blank=True, null=True, on_delete=models.CASCADE)
     # survexpath = models.CharField(max_length=200, blank=True, null=True)  No need for this anymore

core/models/wallets.py

@@ -283,7 +283,7 @@ class Wallet(models.Model):
             return None
             
         filelist = Wallet.input_to_list(jsondata["survex file"])
-        #print(f"'{self} {jsondata['survex file']}' => {filelist}")
+        # print(f"'{self} {jsondata['survex file']}' => {filelist}")
         return filelist
 
     def get_fnames(self):

databaseReset.py

@@ -53,6 +53,7 @@ from troggle.parsers.imports import (
     import_people,
     import_QMs,
     import_survex,
+    import_survex_checks,
     import_surveyscans,
 )
 
@@ -395,7 +396,8 @@ def usage():
              scans     - the survey scans in all the wallets (must run before survex)
              drawings  - read in the Tunnel & Therion files - which scans the survey scans too
              survex    - read in the survex files - all the survex blocks and entrances x/y/z 
-             ents       - read just the entrances x/y/z (must run after survex)
+             survex_ck - set caves and people on wallets, check wallets for *ref 
+             ents      - read just the entrances x/y/z (must run after survex)
 
              dumplogbooks - Not used. write out autologbooks (not working? use http://localhost:8000/controlpanel )
              logbook   - read a single logbook. Defautl set in python code
@@ -442,6 +444,8 @@ if __name__ == "__main__":
         jq.enq("reinit", reinit_db)
     elif "ents" in sys.argv:
         jq.enq("survex", import_ents)
+    elif "survex_ck" in sys.argv:
+        jq.enq("survex", import_survex_checks)
     elif "test2" in sys.argv:
         jq.enq("QMs", import_QMs)
         jq.enq("drawings", import_drawingsfiles)

parsers/imports.py

@@ -54,12 +54,17 @@ def import_survex():
     print(" - Survex Blocks")
     with transaction.atomic():
         troggle.parsers.survex.LoadSurvexBlocks()
-    print(" - Survex entrances x/y/z Positions")
     with transaction.atomic():
         troggle.parsers.survex.survexifywallets()
+    print(" - Survex entrances x/y/z Positions")
     with transaction.atomic():
         troggle.parsers.locations.LoadPositions()
 
+def import_survex_checks():
+    print(" - Survex: check wallet references and set persons, caves")
+    with transaction.atomic():
+        troggle.parsers.survex.survexifywallets()
+
 def import_ents():
     print(" - Survex entrances x/y/z Positions")
     with transaction.atomic():

parsers/survex.py

@@ -2484,18 +2484,40 @@ def MakeFileRoot(svxpath):
     return fileroot
 
 def set_survexblocks(wallet):
-    if svxfiles := wallet.survexfiles():  # reads from JSON, should be cached
+    """Need to find the optimal Django way of doing this query. 
+    It's a mess now"""
+    if svxfiles := wallet.survexfiles():  # reads from JSON, should be cached already
         for svx  in svxfiles:
-            # svx is a string, need to resolve to a survexfile object
+            if svx:
-            #o = SurvexFile.objects.get(path=svx)
+                if svx.endswith(".svx"):
-            blocks = SurvexBlock.objects.filter(survexfile__path=svx)
+                    svx = svx.replace(".svx","")
-            for b in blocks:
+                try:
-                if b.scanswallet == wallet:
+                    # there are survex files we ignore when troggle parses, and some of these are referred to in wallets
-                    pass
+                    sfile = SurvexFile.objects.get(path=svx) #.select_related("survexblocks")
-                else:
+                    # print(sfile)
-                    b.scanswallet = wallet
+                except:
-                    b.save()
+                    continue
-                    # print(f"setting {wallet} on {b.survexfile} : {b}")
+                blocks = SurvexBlock.objects.filter(survexfile=sfile)
+                for b in blocks:
+                    try:
+                        if b.scanswallet == wallet:
+                            pass
+                        elif b.scanswallet:
+                            if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1):
+                                print(f"not set{wallet} on {b.survexfile} : {b} as already set to {b.scanswallet}")
+                        else:
+                                b.scanswallet = wallet
+                                b.save()
+                                if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1):
+                                    print(f"setting {wallet} on {b.survexfile} : {b}")
+                    except: 
+                        if not hasattr(b,"date"):
+                            print(f" Block {b} on {b.survexfile} HAS NO DATE SET ")
+                        elif not b.date:
+                            print(f" Block {b} on {b.survexfile} HAS NULL DATE  ")
+                        else:
+                            print(f" exception {wallet} on {b.survexfile} : {b}")
+                            
  
 def survexifywallets():
     """Gets the caves from the list of survexblocks
@@ -2514,7 +2536,7 @@ def survexifywallets():
             w.persons.add(spr.person)
        
     duration = time.time() - start
-    print(f" -  TIME: add people to wallets {duration:7.2f} s", file=sys.stderr)
+    print(f" -  {duration:7.2f} s to add people to wallets ", file=sys.stderr)
     start = time.time()
                 
     wallets = Wallet.objects.all()
@@ -2522,7 +2544,7 @@ def survexifywallets():
         set_survexblocks(w) # reads JSON, sets survexblocks if survexfiles specified on wallet JSON
    
     duration = time.time() - start
-    print(f" -  TIME: set survexblock:wallet using JSON survexfiles {duration:7.2f} s", file=sys.stderr)
+    print(f" -  {duration:7.2f} s to set survexblock:wallet using JSON survexfiles ", file=sys.stderr)
     start = time.time()
     
     for w in wallets:
@@ -2533,15 +2555,16 @@ def survexifywallets():
         w.save()
 
     duration = time.time() - start
-    print(f" -  TIME: add caves to wallets {duration:7.2f} s", file=sys.stderr)
+    print(f" -  {duration:7.2f} s to add caves to wallets ", file=sys.stderr)
     start = time.time()
     
         
-
+    # Find the survex blocks which are 'ours' i.e. ignore all those (ARGE etc) without expo people attached.
     cuccblocks = set()
     for spr in SurvexPersonRole.objects.all():
         cuccblocks.add(spr.survexblock)
         
+    # Because we have just run set_survexblocks(w), this should only complain if there is no *ref and no wallet that links to its parent file
     sentinelbad = Wallet.objects.get(walletname="1983#00")
     for b in cuccblocks:
         if b.date > date(2001, 1, 1): # do we care about older ones? 1999 certainly has different wallet system
@@ -2551,12 +2574,13 @@ def survexifywallets():
                         b.scanswallet = b.parent.scanswallet
                         continue
                 message = f"  ! *REF missing {b.date} {b.survexfile}.svx : '{b}'"
-                # print(message, file=sys.stderr)
+                if b.date > date(2019, 1, 1) and b.date < date(2020, 1, 1):
+                    print(message, file=sys.stderr)
                 url = get_offending_filename(b.survexfile.path)
                 DataIssue.objects.update_or_create(parser="ref", message=message, url=url)
 
     duration = time.time() - start
-    print(f" -  TIME: check missing *ref on survexblocks  {duration:7.2f} s", file=sys.stderr)
+    print(f" -  {duration:7.2f} s to check missing *ref on survexblocks  ", file=sys.stderr)
     start = time.time()
 
 

templates/yearwallets.html

@@ -8,6 +8,7 @@
 plans and elevations. It also contains scans of centre-line survex output on which
 hand-drawn passage sections are drawn. These hand-drawn passages will eventually be 
 traced to produce Tunnel or Therion drawings and eventually the final complete cave survey.
+<p>Link to <a href="/expedition/{{year}}">logbooks and calendar for {{year}}</a>.
 
 {% include 'wallet_new.html' %}