Fix SurvexDirectory import

core/models_survex.py

@@ -16,8 +16,9 @@ class SurvexDirectory(models.Model):
     class Meta:
         ordering = ('id',)
 
-    def __str__(self):
+    # Don't change from the default as that breaks troggle webpages and internal referencing!
-        return "[SurvexDirectory:"+str(self.path)  + "-" + str(self.primarysurvexfile.path) + "-" + str(self.cave)+"]" 
+    # def __str__(self):
+        # return "[SurvexDirectory:"+str(self.path)  + "-" + str(self.primarysurvexfile.path) + "-" + str(self.cave)+"]" 
 
 
 
@@ -29,8 +30,9 @@ class SurvexFile(models.Model):
     class Meta:
         ordering = ('id',)
 
-    def __str__(self):
+    # Don't change from the default as that breaks troggle webpages and internal referencing!
-        return "[SurvexFile:"+str(self.path) + "-" + str(self.survexdirectory) + "-" + str(self.cave)+"]" 
+    # def __str__(self):
+        # return "[SurvexFile:"+str(self.path) + "-" + str(self.survexdirectory) + "-" + str(self.cave)+"]" 
 
     def exists(self):
         fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx")
@@ -117,10 +119,15 @@ class SurvexBlock(models.Model):
     class Meta:
         ordering = ('id',)
 
+    # Don't change from the original as that breaks troggle webpages and internal referencing!
+    # def __str__(self):
+        # return "[SurvexBlock:"+ str(self.name) + "-path:" + \
+            # str(self.survexpath) + "-cave:" + \
+            # str(self.cave) + "]" 
     def __str__(self):
-        return "[SurvexBlock:"+ str(self.name) + "-path:" + \
+        return self.name and str(self.name) or 'no name'
-            str(self.survexpath) + "-cave:" + \
+    
-            str(self.cave) + "]" 
+
 
 
     def isSurvexBlock(self): # Function used in templates

parsers/survex.py

@@ -45,7 +45,7 @@ class LoadingSurvex():
     rx_qm      = re.compile(r'(?i)^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')
 #   remember there is also QM_PATTERN used in views_other and set in settings.py
 
-    rx_cave    = re.compile(r'(?i)caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)')
+    rx_cave    = re.compile(r'(?i)caves-(\d\d\d\d)/([-\d\w]+|\d\d\d\d-?\w+-\d+)')
     rx_comment = re.compile(r'([^;]*?)\s*(?:;\s*(.*))?\n?$')
     rx_comminc = re.compile(r'(?i)^\*include[\s]*([-\w/]*).*$') # inserted by linear collate ;*include
     rx_commcni = re.compile(r'(?i)^\*edulcni[\s]*([-\w/]*).*$') # inserted by linear collate ;*edulcni
@@ -64,7 +64,6 @@ class LoadingSurvex():
     stacksvxfiles = []
     svxfileslist = []
     svxdirs = {}
-    svxcaves = {}
     lineno = 0
     insp = ""
     callcount = 0
@@ -75,6 +74,7 @@ class LoadingSurvex():
     currentcave = None
 
     def __init__(self):
+        self.caveslist = GetCaveLookup()
         pass
 
     def LoadSurvexIgnore(self, survexblock, line, cmd):
@@ -306,29 +306,29 @@ class LoadingSurvex():
         pass
 
     def IdentifyCave(self, cavepath):
-        if cavepath in self.svxcaves:
+        if cavepath.lower() in self.caveslist:
-            print('    - Cave FAST matched for %s' % cavepath)
+            return self.caveslist[cavepath.lower()]
-            return self.svxcaves[cavepath]
 
         path_match = self.rx_cave.search(cavepath)
         if path_match:
             sluggy = '{}-{}'.format(path_match.group(1), path_match.group(2))
-            print('    - Attempting cave match for %s' % sluggy)
+            guesses = [sluggy.lower(), path_match.group(2).lower()]
-            cave = GetCaveLookup().get(sluggy)
+            for g in guesses:
-            # Below is how it has been done for years: very fuzzy & slow searches
+                if g in self.caveslist:
-            # ..and wrong!
+                    self.caveslist[cavepath] = self.caveslist[g]
-            #cave = models_caves.getCaveByReference(sluggy)
+                    return self.caveslist[g]
-            if cave:
+            print('    ! Failed to find cave for {}'.format(cavepath.lower()))
-                self.currentcave = cave
-                self.svxcaves[cavepath] = cave
-                print('    - Cave matched for %s' % cavepath)
-                return cave
         else:
-                print('    ! Failed to set cave for {} or {}'.format(cavepath, sluggy))
+            print('    ! No regex cave match for %s' % cavepath.lower())
-        else:
-            print('    ! No regex cave match for %s' % cavepath)
             return None
 
+    def GetSurvexDirectory(self, headpath):
+        if not headpath:
+            return self.svxdirs[""]
+        if headpath.lower() not in self.svxdirs:
+            self.svxdirs[headpath.lower()] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile) 
+        return self.svxdirs[headpath.lower()]
+
     def LoadSurvexFile(self, includelabel):
         """Creates SurvexFile in the database, and SurvexDirectory if needed
         with links to 'cave'
@@ -338,18 +338,20 @@ class LoadingSurvex():
         depth = " " * self.depthbegin
         print("{:2}{}   - NEW survexfile:'{}'".format(self.depthbegin, depth, includelabel))
 
-        headpath, tail = os.path.split(includelabel)
-        if headpath not in self.svxdirs:
-            self.svxdirs[headpath] = models_survex.SurvexDirectory(path=headpath, primarysurvexfile=self.currentsurvexfile) 
-        newsurvexdirectory = self.svxdirs[headpath]
-        
         newsurvexfile = models_survex.SurvexFile(path=includelabel)
+        headpath, tail = os.path.split(includelabel)
+        newsurvexdirectory = self.GetSurvexDirectory(headpath)
         newsurvexfile.survexdirectory = newsurvexdirectory
 
         cave = self.IdentifyCave(headpath)
         if cave:
             newsurvexdirectory.cave = cave
             newsurvexfile.cave   = cave
+        # else:
+            # message = " ! Cannot identify cave from {} when creating sfile & sdirectory".format(headpath)
+            # print(message)
+            # print(message,file=sys.stderr)
+            # models.DataIssue.objects.create(parser='survex', message=message)
         self.currentsurvexfile.save() # django insists on this although it is already saved !?
         try:
             newsurvexdirectory.save()
@@ -366,7 +368,6 @@ class LoadingSurvex():
         self.LoadSurvexFile(svxid)
         self.stacksvxfiles.append(self.currentsurvexfile)
 
-
     def ProcessEdulcniLine(self, edulcni):
         """Saves the current survexfile in the db
         """
@@ -803,6 +804,8 @@ def FindAndLoadSurvex(survexblockroot):
    
     print('\n - Loading All Survex Blocks (LinearRecursive)',file=sys.stderr)
     svx_load = LoadingSurvex()
+    
+    svx_load.svxdirs[""] = survexfileroot.survexdirectory
     with open(collatefilename, "r") as fcollate:
             svxlines = fcollate.read().splitlines()
     #----------------------------------------------------------------
@@ -834,6 +837,15 @@ def FindAndLoadSurvex(survexblockroot):
     sys.stdout = stdout_orig
     return (survexlegsnumber, survexlegsalllength)
 
+def MakeSurvexFileRoot():
+    survexfileroot = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
+    survexfileroot.save()
+    survexdirectoryroot = models_survex.SurvexDirectory(path=settings.EXPOWEB, cave=None, primarysurvexfile=survexfileroot)
+    survexdirectoryroot.save()
+    survexfileroot.survexdirectory = survexdirectoryroot
+    survexfileroot.save() # mutually dependent objects need a double-save like this
+
+    return survexfileroot
 
 def LoadSurvexBlocks():
 
@@ -846,9 +858,7 @@ def LoadSurvexBlocks():
     print(" - survex Data Issues flushed")
     models.DataIssue.objects.filter(parser='survex').delete()
 
-    survexfileroot = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
+    survexfileroot = MakeSurvexFileRoot()
-    survexfileroot.save()
-    survexfileroot.SetDirectory()
     survexblockroot = models_survex.SurvexBlock(name=ROOTBLOCK, survexpath="", cave=None, survexfile=survexfileroot, 
             legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
     # this is the first so id=1
@@ -872,12 +882,11 @@ def LoadSurvexBlocks():
 poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
 
 def LoadPos():
-    """Run cavern to produce a complete .3d file, then run 3dtopos to produce a table of 
+    """First load the survex stations for entrances and fixed points (about 600) into the database.
+    Run cavern to produce a complete .3d file, then run 3dtopos to produce a table of 
     all survey point positions. Then lookup each position by name to see if we have it in the database 
-    and if we do, then save the x/y/z coordinates.
+    and if we do, then save the x/y/z coordinates. This gives us coordinates of the entrances.
     If we don't have it in the database, print an error message and discard it.
-    This is ONLY ever used for entrance and fixedpts locations for the prospecting map:
-    about 600 points out of 32,000.
     """
     topdata = settings.SURVEX_DATA + settings.SURVEX_TOPNAME
     print((' - Generating a list of Pos from %s.svx and then loading...' % (topdata)))
@@ -913,17 +922,21 @@ def LoadPos():
             for sid in mappoints:
                 if id.endswith(sid):
                     blockpath = "." + id[:-len(sid)].strip(".")
+                    # But why are we doing this? Why do we need the survexblock id for each of these ?
+                    # ..because mostly they don't actually appear in any SVX file. We should match them up
+                    # via the cave data, not by this half-arsed syntactic match which almost never works. PMS.
+                    if False:
                         try:
                             sbqs = models_survex.SurvexBlock.objects.filter(survexpath=blockpath)
                             if len(sbqs)==1:
                                 sb = sbqs[0]
                             if len(sbqs)>1:
-                            message = ' ! MULTIPLE SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
+                                message = " ! MULTIPLE SurvexBlocks {:3} matching Entrance point {} {} '{}'".format(len(sbqs), blockpath, sid, id)
                                 print(message)
                                 models.DataIssue.objects.create(parser='survex', message=message)
                                 sb = sbqs[0]
                             elif len(sbqs)<=0:
-                            message = ' ! ZERO SurvexBlocks matching Entrance point {} {}'.format(blockpath, sid)
+                                message = " ! ZERO SurvexBlocks matching Entrance point {} {} '{}'".format(blockpath, sid, id)
                                 print(message)
                                 models.DataIssue.objects.create(parser='survex', message=message)
                                 sb = survexblockroot
@@ -932,7 +945,7 @@ def LoadPos():
                             print(message)
                             models.DataIssue.objects.create(parser='survex', message=message)
                     try:
-                        ss = models_survex.SurvexStation(name=id, block=sb)
+                        ss = models_survex.SurvexStation(name=id, block=survexblockroot)
                         ss.x = float(x)
                         ss.y = float(y)
                         ss.z = float(z)