From ecf92e2079924cbb058347b4d1d075bd52598b78 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip@Muscogee.localdomain>
Date: Tue, 28 Apr 2020 21:50:53 +0100
Subject: [PATCH] getting the LoadPos to work better

---
 .gitignore        |  2 ++
 parsers/survex.py | 29 ++++++++++++++++++-----------
 settings.py       |  2 +-
 3 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/.gitignore b/.gitignore
index b43cb7e..39d4835 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,5 @@ import_profile.json
 import_times.json
 ignored-files.log
 tunnel-import.log
+posnotfound
+troggle.sqlite-journal
diff --git a/parsers/survex.py b/parsers/survex.py
index 0484b86..5720b11 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -10,7 +10,7 @@ from django.utils.timezone import make_aware
 
 import re
 import os
-from datetime import datetime
+from datetime import datetime, timedelta
 
 line_leg_regex = re.compile(r"[\d\-+.]+$")
 
@@ -405,20 +405,25 @@ def LoadPos():
         updtsvx = os.path.getmtime(topdata + ".svx")
         updtcache = os.path.getmtime(cachefile)
         age = updtcache - updtsvx
-        print('   svx: %s    cache: %s    cache age: %s' % (updtsvx, updtcache, age ))
+        print('   svx: %s    cache: %s    cache age: %s' % (updtsvx, updtcache, str(timedelta(seconds=age) )))
         if age < 0 :
             print "   cache is stale."
             os.remove(cachefile)
         else:
-            print "   cache is fresh."
+            print "   cache is fresh. Reading..."
             try:
-                f = open(cachefile, "r")
-                for line in f:
-                    notfoundbefore[line] +=1 # should not be duplicates
+                with open(cachefile, "r") as f:
+                    for line in f:
+                        l = line.rstrip()
+                        if l in notfoundbefore:
+                            notfoundbefore[l] +=1 # should not be duplicates
+                            print " DUPLICATE ", line, notfoundbefore[l]
+                        else:
+                            notfoundbefore[l] =1
             except:
                 print "   FAILURE READ opening cache file %s" % (cachefile)
-            f.close()
-
+                raise
+            
     
     notfoundnow =[]
     found = 0
@@ -434,7 +439,7 @@ def LoadPos():
         if r:
             x, y, z, name = r.groups() # easting, northing, altitude
             if name in notfoundbefore:
-                skip[name] += 1
+                skip[name] = 1
             else:
                 try:
                     ss = models.SurvexStation.objects.lookup(name)
@@ -446,16 +451,18 @@ def LoadPos():
                 except:
                     #print "%s in %s.pos not found in lookup of SurvexStation.objects" % (name, settings.SURVEX_TOPNAME)
                     notfoundnow.append(name)
-    print " - %s stations NOT found in lookup of SurvexStation.objects. %s found. %s skipper." % (len(notfoundnow),found, skip)
+    print " - %s stations NOT found in lookup of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip))
 
     if found > 10: # i.e. a previous cave import has been done
         try:
             with open(cachefile, "w") as f:
-                print "   cache file opened"
+                c = len(notfoundnow)+len(skip)
                 for i in notfoundnow:
                     f.write("%s\n" % i)
                 for j in skip:
                     f.write("%s\n" % j) # NB skip not notfoundbefore
+                print('   Not-found cache file written: %s entries' % c)
         except:
             print "   FAILURE WRITE opening cache file %s" % (cachefile)
+            raise
 
diff --git a/settings.py b/settings.py
index 6710e72..de615e4 100644
--- a/settings.py
+++ b/settings.py
@@ -7,7 +7,7 @@ import django
 BASE_DIR = os.path.dirname(os.path.dirname(__file__))
 
 # Django settings for troggle project.
-DEBUG = True
+DEBUG = False
 TEMPLATE_DEBUG = DEBUG
 
 ALLOWED_HOSTS = [u'expo.survex.com']