From 092c8bb91340685843e1353cfce7c8e397277e70 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@klebos.com>
Date: Fri, 12 Jun 2020 00:34:53 +0100
Subject: [PATCH] stopped storing survex legs

---
 core/views_logbooks.py      | 39 +++++++++++++++++++++++++++++++++++++
 parsers/survex.py           | 35 +++++++++++++++++++++++++++------
 templates/experimental.html | 28 ++++++++++++++++++++++++++
 3 files changed, 96 insertions(+), 6 deletions(-)
 create mode 100644 templates/experimental.html

diff --git a/core/views_logbooks.py b/core/views_logbooks.py
index b430796..4c36d04 100644
--- a/core/views_logbooks.py
+++ b/core/views_logbooks.py
@@ -217,6 +217,45 @@ def pathsreport(request):
         "bypathslist":bypathslist, 
         "ncodes":ncodes})
 
+
+    
+def experimental(request):
+    blockroots = models.SurvexBlock.objects.filter(name="root")
+    if len(blockroots)>1:
+        print(" ! more than one root survexblock {}".format(len(blockroots)))
+        for sbr in blockroots:
+                print("{} {} {} {}".format(sbr.id, sbr.name, sbr.text, sbr.date))
+    sbr = blockroots[0]
+    totalsurvexlength = sbr.totalleglength
+    try:
+        nimportlegs = int(sbr.text)
+    except:
+        print("{} {} {} {}".format(sbr.id, sbr.name, sbr.text, sbr.date))
+        nimportlegs = -1
+
+    legsbyexpo = [ ]
+    addupsurvexlength = 0
+    for expedition in Expedition.objects.all():
+        survexblocks = expedition.survexblock_set.all()
+        #survexlegs = [ ]
+        legsyear=0
+        survexleglength = 0.0
+        for survexblock in survexblocks:
+            #survexlegs.extend(survexblock.survexleg_set.all())
+            survexleglength += survexblock.totalleglength
+            try:
+                legsyear += int(survexblock.text)
+            except:
+                pass
+        addupsurvexlength += survexleglength
+        legsbyexpo.append((expedition, {"nsurvexlegs":legsyear, "survexleglength":survexleglength}))
+    legsbyexpo.reverse() 
+
+    #removing survexleg objects completely
+    #survexlegs = models.SurvexLeg.objects.all()
+    #totalsurvexlength = sum([survexleg.tape  for survexleg in survexlegs])
+    return render(request, 'experimental.html', { "nsurvexlegs":nimportlegs, "totalsurvexlength":totalsurvexlength, "addupsurvexlength":addupsurvexlength, "legsbyexpo":legsbyexpo })
+
 @login_required_if_public
 def newLogbookEntry(request, expeditionyear, pdate = None, pslug = None):
     expedition = Expedition.objects.get(year=expeditionyear)
diff --git a/parsers/survex.py b/parsers/survex.py
index b2d1e50..56017f3 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -21,8 +21,12 @@ A 'survexscansfolder' is what we today call a "survey scans folder" or a "wallet
 """
 
 line_leg_regex = re.compile(r"[\d\-+.]+$")
+survexlegsalllength = 0.0
+survexlegsnumber = 0
 
 def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
+    global survexlegsalllength
+    global survexlegsnumber
     # The try catches here need replacing as they are relatively expensive
     ls = sline.lower().split()
     ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]])
@@ -34,13 +38,14 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
     if stardata["type"] == "normal":
         try:
             survexleg.tape = float(ls[stardata["tape"]])
+            survexlegsnumber += 1
         except ValueError:
             print(("! Tape misread in", survexblock.survexfile.path))
             print(("  Stardata:", stardata))
             print(("  Line:", ls))
             message = ' ! Value Error: Tape misread in line %s in %s' % (ls, survexblock.survexfile.path)
             models.DataIssue.objects.create(parser='survex', message=message)
-            survexleg.tape = 1000
+            survexleg.tape = 0
         try:
             lclino = ls[stardata["clino"]]
         except:
@@ -86,15 +91,20 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
             survexleg.cave = cave
 
         # only save proper legs
-        survexleg.save()
+        # No need to save as we are measuring lengths only on parsing now.
+        # delete the object so that django autosaving doesn't save it.
+        survexleg = None
+        #survexleg.save()
 
     itape = stardata.get("tape")
     if itape:
         try:
             survexblock.totalleglength += float(ls[itape])
+            survexlegsalllength += float(ls[itape])
         except ValueError:
             print("! Length not added")
-        survexblock.save()
+        # No need to save as we are measuring lengths only on parsing now.
+        #survexblock.save()
 
 
 def LoadSurvexEquate(survexblock, sline):
@@ -144,6 +154,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
     teammembers = [ ]
     global insp
     global callcount
+    global survexlegsnumber
 
     # uncomment to print out all files during parsing 
     print(insp+"  - Reading file: " + survexblock.survexfile.path + " <> " + survexfile.path)
@@ -337,6 +348,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
                 else:
                     print((insp+'    - No match (b) for %s' % newsvxpath))
 
+                previousnlegs = survexlegsnumber
                 name = line.lower()
                 print((insp+'   - Begin found for: ' + name))
                 # print(insp+'Block cave: ' + str(survexfile.cave))
@@ -356,7 +368,11 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
             if iblankbegins:
                 iblankbegins -= 1
             else:
-                survexblock.text = "".join(textlines)
+                #survexblock.text = "".join(textlines)
+                # .text not used, using it for number of legs per block
+                legsinblock = survexlegsnumber - previousnlegs
+                print("LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber))
+                survexblock.text = str(legsinblock)
                 survexblock.save()
                 # print(insp+'   - End found: ')
                 endstamp = datetime.now()
@@ -437,6 +453,8 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
         # print(insp+'   - Time to process: ' + str(timetaken))
 
 def LoadAllSurvexBlocks():
+    global survexlegsalllength
+    global survexlegsnumber
 
     print(' - Flushing All Survex Blocks...')
 
@@ -464,14 +482,17 @@ def LoadAllSurvexBlocks():
     survexfile.SetDirectory()
 
     #Load all
-    survexblockroot = models_survex.SurvexBlock(name="root", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0)
+    # this is the first so id=1
+    survexblockroot = models.SurvexBlock(name="root", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0)
     survexblockroot.save()
     fin = survexfile.OpenFile()
     textlines = [ ]
     # The real work starts here
     RecursiveLoad(survexblockroot, survexfile, fin, textlines)
     fin.close()
-    survexblockroot.text = "".join(textlines)
+    survexblockroot.totalleglength = survexlegsalllength
+    survexblockroot.text = str(survexlegsnumber)
+    #survexblockroot.text = "".join(textlines) these are all blank
     survexblockroot.save()
     
     # Close the file
@@ -481,6 +502,8 @@ def LoadAllSurvexBlocks():
     
     # Restore sys.stdout to our old saved file handler
     sys.stdout = stdout_orig
+    print(" - total number of survex legs: {}m".format(survexlegsnumber))
+    print(" - total leg lengths loaded: {}m".format(survexlegsalllength))
     print(' - Loaded All Survex Blocks.')
 
 
diff --git a/templates/experimental.html b/templates/experimental.html
new file mode 100644
index 0000000..3e5d1f1
--- /dev/null
+++ b/templates/experimental.html
@@ -0,0 +1,28 @@
+{% extends "base.html" %}
+{% load wiki_markup %}
+{% load link %}
+
+{% block title %}Experimental{% endblock %}
+
+{% block content %}
+
+<h1>Expo Experimental</h1>
+
+<p>Number of survey legs: {{nsurvexlegs}}<br />
+Total length: {{totalsurvexlength}} m on importing survex files.<br />
+Total length: {{addupsurvexlength}} m adding up all the years below.</p>
+
+<table>
+<tr><th>Year</th><th>Surveys</th><th>Survey Legs</th><th>Total length</th></tr>
+{% for legs in legsbyexpo %}
+<tr>
+  <td>{{legs.0.year}}</td>
+  <td>{{legs.0.survexblock_set.all|length}}</td>
+  <td>{{legs.1.nsurvexlegs}}</td>
+  <td>{{legs.1.survexleglength}}</td>
+</tr>
+{% endfor %}
+</table>
+
+{% endblock %}
+