Duplicate survex files in lists all done properly

parsers/survex.py

@@ -49,7 +49,7 @@ METRESINFEET = 3.28084
 UNSEENS = "_unseens.svx"
 
 stop_dup_warning = False
-dup_includes = 1
+dup_includes = 0
 debugprint = False  # Turns on debug printout for just one *include file
 debugprinttrigger = "!"
 
@@ -302,7 +302,7 @@ class LoadingSurvex:
     stacksvxfiles = []
     svxfileslist = []
     svxdirs = {}
-    uniquefile = {}
+    uniquefile = {} # each key is a survex path, and its value is a list of parent files
     expos = {}
     survexdict = {}  # each key is a directory, and its value is a list of files
     lineno = 0
@@ -319,6 +319,9 @@ class LoadingSurvex:
         "fixedpts/gps/gps00raw",
         "",
     ]
+    TREE = "tree"
+    ODDS = "oddments"
+    svxpass = TREE
     includedfilename = ""
     currentsurvexblock = None
     currentsurvexfile = None
@@ -1168,8 +1171,9 @@ class LoadingSurvex:
         Inspects the parent folder of the survexfile and uses that to decide if this is 
         a cave we know.
         
-        If we see a duplicate cave, this is too late. It has already been included into the
+        If we see a duplicate cave, this is TOO LATE. It has already been included into the
-        long linear file. This needs to be prevented when the long linear file is created.
+        long linear file. We prevent duplication when the long linear file is created, so
+        if we see a duplicate here, it is a serious error.
         
         The survexblock passed-in is not necessarily the parent. FIX THIS.
         """
@@ -1188,7 +1192,7 @@ class LoadingSurvex:
         newfile, created = SurvexFile.objects.update_or_create(path=svxid)
         if not created:
             dup_includes += 1
-            message = f" ! DUP SurvexFile '{svxid}' create attempt in LoadSurvexFile()"
+            message = f" ! DUPLICATE SurvexFile '{svxid}' create attempt in LoadSurvexFile()"
             print(message)
             # print(message, file=sys.stderr)
             stash_data_issue(parser="survex", message=message, url=f"/survexfile/{svxid}")
@@ -1766,18 +1770,18 @@ class LoadingSurvex:
         want to process them again. For the _unseens this is not an error, but for the main
         *include tree it is an error.
         """
- 
         if incpath in self.uniquefile:
             self.uniquefile[incpath].append(parent)
             
-            message = (
+            if self.svxpass == self.TREE:
-                f" DUP:  non-unique survex filepath, '{incpath}' -  #{len(self.uniquefile[incpath])} '{self.uniquefile[incpath]}'"
+                message = (
-            )
+                    f" DUP:  skipping non-unique survex filepath, '{incpath}' -  #{len(self.uniquefile[incpath])} '{self.uniquefile[incpath]}'"
-            print(message)
+                )
-            # stash_data_issue(parser='survex', message=message)
+                print(message)
-            for p in self.uniquefile[incpath]:
+                stash_data_issue(parser='survex', message=message)
-                if p in self.uniquefile:
+                for p in self.uniquefile[incpath]:
-                    print(f"{p} <- {self.uniquefile[p]}")
+                    if p in self.uniquefile:
+                        print(f"{p} <- {self.uniquefile[p]}")
             return False
         else:
             self.uniquefile[incpath] = [parent]
@@ -1933,9 +1937,11 @@ def FindAndLoadSurvex(survexblockroot):
 
     pr = cProfile.Profile()
     pr.enable()
+    svx_scan.svxpass = svx_scan.TREE
     # ----------------------------------------------------------------
     svx_scan.PushdownStackScan(survexblockroot, survexfileroot.path, finrootname, flinear, fcollate)
     # ----------------------------------------------------------------
+    svx_scan.svxpass = ""
     pr.disable()
     with open("PushdownStackScan.prof", "w") as f:
         ps = pstats.Stats(pr, stream=f)
@@ -2043,11 +2049,13 @@ def FindAndLoadSurvex(survexblockroot):
     finrootname = fullpathtotop
     fcollate.write(f";*include {UNSEENS}\n")
     flinear.write(f"{omit_scan.depthinclude:2} {indent} *include {unseensroot}\n")
+    omit_scan.svxpass = omit_scan.ODDS
     # stop_dup_warning = True
     # ----------------------------------------------------------------
     omit_scan.PushdownStackScan(survexblockroot, unseensroot, finrootname, flinear, fcollate)
     # ----------------------------------------------------------------
     # stop_dup_warning = False
+    omit_scan.svxpass = ""
 
     flinear.write(f"{omit_scan.depthinclude:2} {indent} *edulcni {unseensroot}\n")
     fcollate.write(f";*edulcni {UNSEENS}\n")
@@ -2221,6 +2229,7 @@ def LoadSurvexBlocks():
     store_data_issues()
     # duration = time.time() - start
     # print(f" - TIME: {duration:7.2f} s", file=sys.stderr)
-    print(f" - Duplicate *includes = {dup_includes}")
+    if dup_includes > 0:
+        print(f" - ERROR: There are {dup_includes} duplicate *includes in the final list")
     print(" - Loaded All Survex Blocks.")