move *_views files to /views/*

core/views/caves.py

@@ -0,0 +1,541 @@
+import os
+import string
+import subprocess
+import re
+import settings
+import urllib.parse
+
+from PIL import Image, ImageDraw, ImageFont
+from django import forms
+from django.conf import settings
+from django.urls import reverse
+from django.http import HttpResponse, HttpResponseRedirect
+from django.shortcuts import get_object_or_404, render
+from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
+
+import troggle.settings as settings
+#import troggle.core.models as models
+from troggle.core.models import Expedition, DataIssue
+from troggle.core.models_caves import CaveSlug, Cave, CaveAndEntrance, QM, EntranceSlug, Entrance, Area, SurvexStation, GetCaveLookup
+from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
+from troggle.helper import login_required_if_public
+
+class MapLocations(object):
+    p = [
+        ("laser.0_7", "BNase", "Reference", "Bräuning Nase laser point"),
+        ("226-96", "BZkn", "Reference", "Bräuning Zinken trig point"),
+        ("vd1","VD1","Reference", "VD1 survey point"),
+        ("laser.kt114_96","HSK","Reference", "Hinterer Schwarzmooskogel trig point"), 
+        ("2000","Nipple","Reference", "Nipple (Weiße Warze)"),
+        ("3000","VSK","Reference", "Vorderer Schwarzmooskogel summit"),
+        ("topcamp", "OTC", "Reference", "Old Top Camp"),
+        ("laser.0", "LSR0", "Reference", "Laser Point 0"),
+        ("laser.0_1", "LSR1", "Reference", "Laser Point 0/1"),
+        ("laser.0_3", "LSR3", "Reference", "Laser Point 0/3"),
+        ("laser.0_5", "LSR5", "Reference", "Laser Point 0/5"),
+        ("225-96", "BAlm", "Reference", "Bräuning Alm trig point")
+    ]
+    def points(self):
+        for ent in Entrance.objects.all():
+            if ent.best_station():
+                try:
+                    k = ent.caveandentrance_set.all()[0].cave
+                except:
+                    message = " ! Failed to get Cave linked to Entrance:{} from:{} best:{}".format(ent.name, ent.filename, ent.best_station())
+                    DataIssue.objects.create(parser='entrances', message=message)
+                    print(message)
+                    raise
+                try:
+                    areaName = k.getArea().short_name
+                except:
+                    message = " ! Failed to get Area on cave '{}' linked to Entrance:{} from:{} best:{}".format(cave, ent.name, ent.filename, ent.best_station())
+                    DataIssue.objects.create(parser='entrances', message=message)
+                    print(message)
+                    raise
+                self.p.append((ent.best_station(), "%s-%s" % (areaName, str(ent)[5:]), ent.needs_surface_work(), str(ent)))
+        return self.p
+
+    def __str__(self):
+        return "{} map locations".format(len(self.p))
+
+def getCave(cave_id):
+    '''Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm.
+    
+    TO DO: search GCavelookup first, which should raise a MultpleObjectsReturned exception if there
+    are duplicates'''
+    try:
+        cave = Cave.objects.get(kataster_number=cave_id)
+        return cave
+    except Cave.MultipleObjectsReturned as ex:
+        raise MultipleObjectsReturned("Duplicate kataster number")  from ex # propagate this up
+
+    except Cave.DoesNotExist as ex:
+        Gcavelookup = GetCaveLookup() # dictionary makes strings to Cave objects
+        if cave_id in Gcavelookup:
+            return Gcavelookup[cave_id] 
+        else:
+            raise ObjectDoesNotExist("No cave found with this identifier in any id field") from ex # propagate this up           
+    except:
+        raise ObjectDoesNotExist("No cave found with this identifier in any id field")
+
+def pad5(x):
+   return "0" * (5 -len(x.group(0))) + x.group(0) 
+def padnumber(x):
+   return re.sub("\d+", pad5, x)
+def numericalcmp(x, y):
+   return cmp(padnumber(x), padnumber(y))
+
+def caveKey(x):
+    """python3 function for sort. Done in a hurry.
+    Note that cave kataster numbers are not always integers.
+    This needs to be fixed make a decent sort order.
+    """ 
+    if not x.kataster_number:
+        return "~"
+    return x.kataster_number
+
+def getnotablecaves():
+    notablecaves = []
+    for kataster_number in settings.NOTABLECAVESHREFS:
+        try:
+            cave = Cave.objects.get(kataster_number=kataster_number)
+            notablecaves.append(cave)
+        except:
+            print(" ! FAILED to get only one cave per kataster_number OR invalid number for: "+kataster_number)
+            caves = Cave.objects.all().filter(kataster_number=kataster_number)
+            for c in caves:
+                print(c.kataster_number, c.slug())
+                if c.slug() != None:
+                    notablecaves.append(c)
+    return notablecaves
+
+
+def caveindex(request):
+    caves = Cave.objects.all()
+    caves1623 = list(Cave.objects.filter(area__short_name = "1623"))
+    caves1626 = list(Cave.objects.filter(area__short_name = "1626"))
+    caves1623.sort(key=caveKey)
+    caves1626.sort(key=caveKey)
+    return render(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':getnotablecaves(), 'cavepage': True})
+
+def cave3d(request, cave_id=''):
+    try:
+        cave = getCave(cave_id)
+    except Cave.MultipleObjectsReturned: # entirely the wrong action, REPLACE with the right display
+        caves = Cave.objects.filter(kataster_number=cave_id) 
+        return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves })
+    
+    survexfilename = settings.SURVEX_DATA + cave.survex_file
+    threedfilename = settings.THREEDCACHEDIR + '%s.3d' % cave_id
+    if True or os.path.getmtime(survexfilename) > os.path.getmtime(threedfilename):
+        subprocess.call(["cavern", "--output=%s" % threedfilename, survexfilename])
+    test_file = open(threedfilename, 'rb')
+    response = HttpResponse(content=test_file, content_type='application/3d')#mimetype is replaced by content_type for django 1.7
+    response['Content-Disposition'] = 'attachment; filename=%s.3d' % cave_id
+    # response['X-Sendfile'] = "%s.3d" % cave_id
+    # It's usually a good idea to set the 'Content-Length' header too.
+    # You can also set any other required headers: Cache-Control, etc.
+    return response
+
+def cave(request, cave_id='', offical_name=''):
+    try:
+        cave=getCave(cave_id) 
+    except MultipleObjectsReturned:
+        caves = Cave.objects.filter(kataster_number=cave_id) 
+        return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves }) # not the right template, needs a specific one
+    except ObjectDoesNotExist:
+        return render(request, 'svxcavesingle404.html', {'settings': settings, "cave":cave_id })
+    except:
+        return render(request, 'svxcavesingle404.html', {'settings': settings })
+
+    if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
+        return render(request, 'nonpublic.html', {'instance': cave, 'cavepage': True, 'cave_id': cave_id})
+    else:
+        return render(request,'cave.html', {'settings': settings, 'cave': cave, 'cavepage': True, 'cave_id': cave_id})
+
+def caveEntrance(request, slug):
+    cave = Cave.objects.get(caveslug__slug = slug)
+    if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
+        return render(request,'nonpublic.html', {'instance': cave})
+    else:
+        return render(request,'cave_entrances.html', {'cave': cave})
+
+def caveDescription(request, slug):
+    cave = Cave.objects.get(caveslug__slug = slug)
+    if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
+        return render(request,'nonpublic.html', {'instance': cave})
+    else:
+        return render(request,'cave_uground_description.html', {'cave': cave})
+
+def caveQMs(request, slug):
+    cave = Cave.objects.get(caveslug__slug = slug)
+    if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
+        return render(request,'nonpublic.html', {'instance': cave})
+    else:
+        return render(request,'cave_qms.html', {'cave': cave})
+def caveLogbook(request, slug):
+    cave = Cave.objects.get(caveslug__slug = slug)
+    if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
+        return render(request,'nonpublic.html', {'instance': cave})
+    else:
+        return render(request,'cave_logbook.html', {'cave': cave})
+
+def caveSlug(request, slug):
+    cave = Cave.objects.get(caveslug__slug = slug)
+    if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
+        return render(request,'nonpublic.html', {'instance': cave, 'cave_editable': slug})
+    else:
+        return render(request,'cave.html', {'cave': cave, 'cave_editable': slug})
+
+@login_required_if_public
+def edit_cave(request, slug=None):
+    if  slug is not None:
+        cave = Cave.objects.get(caveslug__slug = slug)
+    else:
+        cave = Cave()
+    if request.POST:
+        form = CaveForm(request.POST, instance=cave)
+        ceFormSet = CaveAndEntranceFormSet(request.POST)
+        versionControlForm = VersionControlCommentForm(request.POST)
+        if form.is_valid() and ceFormSet.is_valid() and versionControlForm.is_valid():
+            cave = form.save(commit = False)
+            if slug is None:
+                for a in form.cleaned_data["area"]:
+                    if a.kat_area():
+                        myArea = a.kat_area()
+                if form.cleaned_data["kataster_number"]:
+                    myslug =  "%s-%s" % (myArea, form.cleaned_data["kataster_number"])
+                else:
+                    myslug =  "%s-%s" % (myArea, form.cleaned_data["unofficial_number"])
+            else:
+                myslug = slug
+            cave.filename = myslug + ".html"
+            cave.save()
+            form.save_m2m()
+            if slug is None:
+              cs = CaveSlug(cave = cave, slug = myslug, primary = True)
+              cs.save()
+            ceinsts = ceFormSet.save(commit=False)
+            for ceinst in ceinsts:
+                ceinst.cave = cave
+                ceinst.save()
+            cave.writeDataFile()
+            return HttpResponseRedirect("/" + cave.url)    
+    else:
+        form = CaveForm(instance=cave)
+        ceFormSet = CaveAndEntranceFormSet(queryset=cave.caveandentrance_set.all())
+        versionControlForm = VersionControlCommentForm()
+        
+    return render(request, 
+                               'editcave2.html', 
+                               {'form': form,
+                                'caveAndEntranceFormSet': ceFormSet,
+                                'versionControlForm': versionControlForm
+                                })
+
+@login_required_if_public
+def editEntrance(request, caveslug, slug=None):
+    cave = Cave.objects.get(caveslug__slug = caveslug) 
+    if  slug is not None:
+        entrance = Entrance.objects.get(entranceslug__slug = slug)
+    else:
+        entrance = Entrance()
+    if request.POST:
+        form = EntranceForm(request.POST, instance = entrance)
+        versionControlForm = VersionControlCommentForm(request.POST)
+        if slug is None:
+            entletter = EntranceLetterForm(request.POST)
+        else:
+            entletter = None
+        if form.is_valid() and versionControlForm.is_valid() and (slug is not None or entletter.is_valid()):
+            entrance = form.save(commit = False)
+            if slug is None:
+                slugname = cave.slug() + entletter.cleaned_data["entrance_letter"]
+                entrance.cached_primary_slug = slugname
+                entrance.filename = slugname + ".html"
+            entrance.save()
+            if slug is None:
+                es = EntranceSlug(entrance = entrance, slug = slugname, primary = True)
+                es.save()
+                el = entletter.save(commit = False)
+                el.cave = cave
+                el.entrance = entrance
+                el.save()
+            entrance.writeDataFile()
+            return HttpResponseRedirect("/" + cave.url) 
+    else:
+        form = EntranceForm(instance = entrance)
+        versionControlForm = VersionControlCommentForm()
+        if slug is None:
+            entletter = EntranceLetterForm(request.POST)
+        else:
+            entletter = None
+    return render(request, 
+                               'editentrance.html', 
+                               {'form': form,
+                                'versionControlForm': versionControlForm,
+                                'entletter': entletter
+                                })
+
+def qm(request,cave_id,qm_id,year,grade=None):
+    year=int(year)
+    try:
+        qm=getCave(cave_id).get_QMs().get(number=qm_id,found_by__date__year=year) 
+        return render(request,'qm.html',locals())
+    except Cave.MultipleObjectsReturned: # entirely the wrong action, REPLACE with the right display
+        caves = Cave.objects.filter(kataster_number=cave_id) 
+        return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves })
+
+    except QM.DoesNotExist:
+        url=urllib.parse.urljoin(settings.URL_ROOT, r'/admin/core/qm/add/'+'?'+  r'number=' + qm_id)
+        if grade:
+            url += r'&grade=' + grade
+        return HttpResponseRedirect(url)
+
+def ent(request, cave_id, ent_letter):
+    cave = Cave.objects.filter(kataster_number = cave_id)[0]
+    cave_and_ent = CaveAndEntrance.objects.filter(cave = cave).filter(entrance_letter = ent_letter)[0]
+    return render(request,'entrance.html', {'cave': cave,
+                                                'entrance': cave_and_ent.entrance,
+                                                'letter': cave_and_ent.entrance_letter,})
+
+def entranceSlug(request, slug):
+    entrance = Entrance.objects.get(entranceslug__slug = slug)
+    if entrance.non_public and not request.user.is_authenticated():
+        return render(request,'nonpublic.html', {'instance': entrance})
+    else:
+        return render(request,'entranceslug.html', {'entrance': entrance})
+
+def surveyindex(request):
+    surveys=Survey.objects.all()
+    expeditions=Expedition.objects.order_by("-year")
+    return render(request,'survey.html',locals())
+
+# def cave_description(request, cavedescription_name):
+    # cave_description = get_object_or_404(CaveDescription, short_name = cavedescription_name)
+    # return render(request,'cave_description.html', locals())
+
+def get_entrances(request, caveslug):
+    cave = Cave.objects.get(caveslug__slug = caveslug)
+    return render(request,'options.html', {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]})
+
+def get_qms(request, caveslug):
+    cave = Cave.objects.get(caveslug__slug = caveslug)
+    return render(request,'options.html', {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]})
+
+areanames = [
+	#('',     'Location unclear'),
+	('1a',   '1a – Plateau: around Top Camp'),
+	('1b',   '1b – Western plateau near 182'),
+	('1c',   '1c – Eastern plateau near 204 walk-in path'),
+	('1d',   '1d – Further plateau around 76'),
+	('2a',   '2a – Southern Schwarzmooskogel near 201 path and the Nipple'),
+	('2b',   '2b – Eishöhle area'),
+	('2b or 4 (unclear)', '2b or 4 (unclear)'),
+	('2c',   '2c – Kaninchenhöhle area'),
+	('2d',   '2d – Steinbrückenhöhle area'),
+	('3',    '3 – Bräuning Alm'),
+	('4',	'4 – Kratzer valley'),
+	('5',	'5 – Schwarzmoos-Wildensee'),
+	('6',	'6 – Far plateau'),
+	('1626 or 6 (borderline)', '1626 or 6 (borderline)'),
+	('7',	'7 – Egglgrube'),
+	('8a',	'8a – Loser south face'),
+	('8b',	'8b – Loser below Dimmelwand'),
+	('8c',	'8c – Augst See'),
+	('8d',	'8d – Loser-Hochganger ridge'),
+	('9',	'9 – Gschwandt Alm'),
+	('10',	'10 – Altaussee'),
+	('11',	'11 – Augstbach')
+	]
+
+
+def prospecting(request):
+    #for key, name in areanames:
+    #    print key, Area.objects.get(short_name = key)
+    areas = []
+    for key, name in areanames:
+        a = Area.objects.get(short_name = key)
+        caves = list(a.cave_set.all())
+        caves.sort(key=caveKey)
+        areas.append((name, a, caves))
+    return render(request, 'prospecting.html', {"areas": areas})
+    
+# Parameters for big map and zoomed subarea maps:
+# big map first (zoom factor ignored)
+
+maps = {
+# id	left		top			right		bottom		zoom
+#		G&K			G&K			G&K			G&K			factor
+"all": [33810.4,	85436.5,	38192.0,		81048.2,	0.35,
+		"All"],
+"40":  [36275.6,	82392.5,	36780.3,	81800.0,	3.0,
+		"Eishöhle"],
+"76":  [35440.0,	83220.0,	36090.0,	82670.0,	1.3,
+		"Eislufthöhle"],
+"204": [36354.1,	84154.5,	37047.4,	83300,	3.0,
+		"Steinbrückenhöhle"],
+"tc":  [35230.0,	82690.0,	36110.0,	82100.0,	3.0,
+		"Near Top Camp"],
+"grieß":
+       [36000.0,	86300.0,	38320.0,	84400.0,	4.0,
+		"Grießkogel Area"],
+}
+
+for n in list(maps.keys()):
+    L, T, R, B, S, name = maps[n]
+    W = (R-L)/2
+    H = (T-B)/2
+    for i in range(2):
+        for j in range(2):
+            maps["%s%i%i" % (n, i, j)] = [L + i * W, T - j * H, L + (i + 1) * W, T - (j + 1) * H, S, name]
+# Keys in the order in which we want the maps output
+mapcodes = ["all", "grieß","40", "76", "204", "tc"] 
+# Field codes
+L = 0
+T = 1
+R = 2
+B = 3
+ZOOM = 4
+DESC = 5
+
+areacolours = {
+	'1a' : '#00ffff',
+	'1b' : '#ff00ff',
+	'1c' : '#ffff00',
+	'1d' : '#ffffff',
+	'2a' : '#ff0000',
+	'2b' : '#00ff00',
+	'2c' : '#008800',
+	'2d' : '#ff9900',
+	'3'  : '#880000',
+	'4'  : '#0000ff',
+	'6'  : '#000000', # doubles for surface fixed pts, and anything else
+	'7'  : '#808080'
+	}
+
+for FONT in [
+		"/usr/share/fonts/truetype/freefont/FreeSans.ttf",
+		"/usr/X11R6/lib/X11/fonts/truetype/arial.ttf",
+		"/mnt/c/windows/fonts/arial.ttf",
+		"C:\WINNT\Fonts\ARIAL.TTF"
+		]:
+	if os.path.isfile(FONT): break
+TEXTSIZE = 16
+CIRCLESIZE =8
+LINEWIDTH = 2
+myFont = ImageFont.truetype(FONT, TEXTSIZE)
+
+def mungecoord(x, y, mapcode, img):
+	# Top of Zinken is 73 1201 = dataset 34542 81967
+	# Top of Hinter is 1073 562 = dataset 36670 83317
+	# image is 1417 by 2201
+	# FACTOR1 = 1000.0 / (36670.0-34542.0)
+	# FACTOR2 = (1201.0-562.0) / (83317 - 81967)
+	# FACTOR = (FACTOR1 + FACTOR2)/2 
+	# The factors aren't the same as the scanned map's at a slight angle. I
+	# can't be bothered to fix this.  Since we zero on the Hinter it makes
+	# very little difference for caves in the areas round 76 or 204.
+	# xoffset = (x - 36670)*FACTOR
+	# yoffset = (y - 83317)*FACTOR
+	# return (1073 + xoffset, 562 - yoffset)
+
+	m = maps[mapcode]
+	factorX, factorY = img.size[0] / (m[R] - m[L]), img.size[1] / (m[T] - m[B])
+	return ((x - m[L]) * factorX, (m[T] - y) * factorY)
+	
+COL_TYPES = {True: "red",
+             False: "#dddddd",
+             "Reference": "#dddddd"}
+	
+def plot(surveypoint, number,  point_type, label, mapcode, draw, img):
+    try:
+        ss = SurvexStation.objects.lookup(surveypoint)
+        E, N = ss.x, ss.y
+        shortnumber = number.replace("—","")
+        (x,y) = list(map(int, mungecoord(E, N, mapcode, img)))
+        #imgmaps[maparea].append( [x-4, y-SIZE/2, x+4+draw.textsize(shortnumber)[0], y+SIZE/2, shortnumber, label] )
+        draw.rectangle([(x+CIRCLESIZE, y-TEXTSIZE/2), (x+CIRCLESIZE*2+draw.textsize(shortnumber)[0], y+TEXTSIZE/2)], fill="#ffffff")
+        draw.text((x+CIRCLESIZE * 1.5,y-TEXTSIZE/2), shortnumber, fill="#000000")
+        draw.ellipse([(x-CIRCLESIZE,y-CIRCLESIZE),(x+CIRCLESIZE,y+CIRCLESIZE)], fill=COL_TYPES[point_type], outline="#000000")
+    except:
+        pass
+
+def prospecting_image(request, name):
+    # We should replace all this with something that exports an overlay for Google Maps and OpenStreetView
+    mainImage = Image.open(os.path.join(settings.SURVEY_SCANS, "location_maps", "pguidemap.jpg"))
+    if settings.PUBLIC_SITE and not request.user.is_authenticated():
+        mainImage = Image.new("RGB", mainImage.size, '#ffffff')
+    m = maps[name]
+    #imgmaps = []
+    if name == "all":
+        img = mainImage
+    else:
+        M = maps['all']
+        W, H = mainImage.size
+        l = int((m[L] - M[L]) / (M[R] - M[L]) * W)
+        t = int((m[T] - M[T]) / (M[B] - M[T]) * H)
+        r = int((m[R] - M[L]) / (M[R] - M[L]) * W)
+        b = int((m[B] - M[T]) / (M[B] - M[T]) * H)
+        img = mainImage.crop((l, t, r, b))
+        w = int(round(m[ZOOM] * (m[R] - m[L]) / (M[R] - M[L]) * W))
+        h = int(round(m[ZOOM] * (m[B] - m[T]) / (M[B] - M[T]) * H))
+        img = img.resize((w, h), Image.BICUBIC)
+    draw = ImageDraw.Draw(img)
+    draw.setfont(myFont)
+    if name == "all":
+        for maparea in list(maps.keys()):
+            if maparea == "all":
+                continue
+            localm = maps[maparea]
+            l,t = mungecoord(localm[L], localm[T], "all", img)
+            r,b = mungecoord(localm[R], localm[B], "all", img)
+            text = maparea + " map"
+            textlen = draw.textsize(text)[0] + 3
+            draw.rectangle([l, t, l+textlen, t+TEXTSIZE+2], fill='#ffffff')
+            draw.text((l+2, t+1), text, fill="#000000")
+            #imgmaps.append( [l, t, l+textlen, t+SIZE+2, "submap" + maparea, maparea + " subarea map"] )
+            draw.line([l, t, r, t], fill='#777777', width=LINEWIDTH)
+            draw.line([l, b, r, b], fill='#777777', width=LINEWIDTH)
+            draw.line([l, t, l, b], fill='#777777', width=LINEWIDTH)
+            draw.line([r, t, r, b], fill='#777777', width=LINEWIDTH)
+            draw.line([l, t, l+textlen, t], fill='#777777', width=LINEWIDTH)
+            draw.line([l, t+TEXTSIZE+2, l+textlen, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH)
+            draw.line([l, t, l, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH)
+            draw.line([l+textlen, t, l+textlen, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH)
+    #imgmaps[maparea] = []
+    # Draw scale bar
+    m100 = int(100 / (m[R] - m[L]) * img.size[0])
+    draw.line([10, TEXTSIZE*3, 10, TEXTSIZE*2], fill='#000000', width=LINEWIDTH)
+    draw.line([10, TEXTSIZE*2, 10+m100, TEXTSIZE*2], fill='#000000', width=LINEWIDTH)
+    draw.line([10+m100, TEXTSIZE * 3, 10+m100, TEXTSIZE*2], fill='#000000', width=LINEWIDTH)
+    label = "100m"
+    draw.text([10 + (m100 - draw.textsize(label)[0]) / 2, TEXTSIZE/2], label, fill='#000000')
+
+    for p in MapLocations.points():
+        surveypoint, number,  point_type, label = p
+        plot(surveypoint, number,  point_type, label, name, draw, img)
+
+    for (N, E, D, num) in [(35975.37,    83018.21,    100,"177"),    # Calculated from bearings
+                           (35350.00,    81630.00,    50,    "71"),    # From Auer map
+                           (36025.00,    82475.00,    50,    "146"),    # From mystery map
+                           (35600.00,    82050.00,    50,    "35"),    # From Auer map
+                           (35650.00,    82025.00,    50,    "44"),    # From Auer map
+                           (36200.00,    82925.00,    50,    "178"),    # Calculated from bearings
+                           (35232.64,    82910.37,    25,    "181"),    # Calculated from bearings
+                           (35323.60,    81357.83,    50,    "74")    # From Auer map
+                           ]:
+        (N,E,D) = list(map(float, (N, E, D)))
+        maparea = Cave.objects.get(kataster_number = num).getArea().short_name
+        lo = mungecoord(N-D, E+D, name, img)
+        hi = mungecoord(N+D, E-D, name, img)
+        lpos = mungecoord(N-D, E, name, img)
+        draw.ellipse([lo,hi], outline="#000000")
+        draw.ellipse([lo[0]+1, lo[1]+1, hi[0]-1, hi[1]-1], outline=areacolours[maparea])
+        draw.ellipse([lo[0]+2, lo[1]+2, hi[0]-2, hi[1]-2], outline=areacolours[maparea])
+        draw.rectangle([lpos[0],lpos[1]-TEXTSIZE/2, lpos[0] + draw.textsize(name)[0], lpos[1]+TEXTSIZE/2], fill="#ffffff")
+        draw.text((lpos[0], lpos[1]-TEXTSIZE/2), num, fill="#000000")
+    response = HttpResponse(content_type = "image/png")
+    del draw
+    img.save(response, "PNG")
+    return response

core/views/expo.py

@@ -0,0 +1,254 @@
+import os
+import re
+from pathlib import Path
+from urllib.parse import urljoin, unquote as urlunquote
+from urllib.request import urlopen
+
+from django.shortcuts import render, redirect
+from django.http import HttpResponse, HttpResponseRedirect, Http404
+from django.urls import reverse, resolve
+from django.template import Context, loader
+from django.views.decorators.csrf import ensure_csrf_cookie
+from django.contrib import admin
+
+import django.forms as forms
+
+from troggle.helper import login_required_if_public
+from troggle.core.models_caves import Cave
+import troggle.core.views.caves
+import troggle.settings as settings
+
+'''Formerly a separate package  'flatpages' written by Martin Green 2011. 
+This was NOT django.contrib.flatpages which stores HTML in the database, so the name was chnaged to expopages.
+Then it was incorporated into troggle directly, rather than being an unnecessary external package.
+'''
+
+def expofiles_redirect(request, path):
+    '''This is used only when running as a test system without a local copy of /expofiles/
+    '''
+    return redirect(urljoin('http://expo.survex.com/expofiles/', path))
+
+def expofilessingle(request, filepath):
+    '''sends a single binary file to the user,
+    '''
+    fn=urlunquote(filepath)
+    fn = Path(settings.EXPOFILES,filepath)
+    if fn.is_dir():
+        return expofilesdir(request, Path(fn), Path(filepath))
+    # print(" - expofilessingle {}:{}:{}:".format(filepath, fn, getmimetype(fn)))
+    return HttpResponse(content=open(fn, "rb"),content_type=getmimetype(filepath)) # any file
+
+def expofilesdir(request, dirpath, filepath):
+    '''does a directory display. If there is an index.html file we should display that.
+    - dirpath is a Path() and it does not have /expofiles/ in it
+    '''
+    # print(" - expofilesdir {}".format(dirpath))
+    urlpath = 'expofiles' / Path(filepath)
+    fileitems = []
+    diritems = []
+    for f in dirpath.iterdir():
+        if f.is_dir():
+             diritems.append((urlpath / f.parts[-1], str(f.parts[-1])))
+        else:
+            # if f.parts[-1].lower() == 'index.htm' or f.parts[-1].lower() == 'index.html': # css cwd problem
+                # return HttpResponse(content=open(f, "rb"),content_type=getmimetype(filepath)) # any file
+                # return expofilessingle(request, str(Path(filepath /  f.parts[-1])))             
+            fileitems.append((Path(urlpath) / f.parts[-1], str(f.parts[-1]), getmimetype(f)))
+    return render(request, 'dirdisplay.html', { 'filepath': urlpath, 'fileitems':fileitems, 'diritems': diritems,'settings': settings })
+
+def expowebpage(request, expowebpath, path):
+    '''Adds memnus and serves an HTML page
+    '''
+    if not Path(expowebpath / path).is_file():
+        return render(request, 'pagenotfound.html', {'path': path})
+        
+    with open(os.path.normpath(expowebpath / path), "rb") as o:   
+        html = o.read()
+    
+    m = re.search(rb'(.*)<\s*head([^>]*)>(.*)<\s*/head\s*>(.*)<\s*body([^>]*)>(.*)<\s*/body\s*>(.*)', html, re.DOTALL + re.IGNORECASE)
+    if m:
+        preheader, headerattrs, head, postheader, bodyattrs, body, postbody = m.groups()
+    else:
+        return HttpResponse(html + "HTML Parsing failure: Page could not be split into header and body: failed in expowebpage in views.expo.py")
+    m = re.search(rb"<title>(.*)</title>", head, re.DOTALL + re.IGNORECASE)
+    if m:
+        title, = m.groups()
+    else:
+        title = ""
+    m = re.search(rb"<meta([^>]*)noedit", head, re.DOTALL + re.IGNORECASE)
+    if m:
+        editable = False
+    else:
+        editable = True
+    
+    has_menu = False
+    menumatch = re.match(rb'(.*)<div id="menu">', body, re.DOTALL + re.IGNORECASE)
+    if menumatch:
+        has_menu = True
+    menumatch = re.match(rb'(.*)<ul id="links">', body, re.DOTALL + re.IGNORECASE)
+    if menumatch:
+        has_menu = True
+    return render(request, 'flatpage.html', {'editable': editable, 'path': path, 'title': title, 
+                'body': body, 'homepage': (path == "index.htm"), 'has_menu': has_menu})
+    
+
+def expopage(request, path):
+    '''Either renders an HTML page from expoweb with all the menus,
+    or serves an unadorned binary file with mime type
+    '''
+    #print(" - EXPOPAGES delivering the file: '{}':{} as MIME type: {}".format(request.path, path,getmimetype(path)),flush=True)
+
+    if path.startswith("noinfo") and settings.PUBLIC_SITE and not request.user.is_authenticated():
+        return HttpResponseRedirect(urljoin(reverse("auth_login"),'?next={}'.format(request.path)))
+
+    if path.startswith("admin/"):
+        # don't even attempt to handle these sorts of mistakes
+        return HttpResponseRedirect("/admin/")
+
+    expowebpath = Path(settings.EXPOWEB)
+
+    if path == "":
+        return expowebpage(request, expowebpath, "index.htm")
+
+    if path.endswith(".htm") or path.endswith(".html"):
+        return expowebpage(request, expowebpath, path)
+    
+    if Path(expowebpath / path ).is_dir():
+        for p in ["index.html", "index.htm", "default.html"]:
+            try:
+                o = open(os.path.normpath(expowebpath / path / p), "rb")
+            except IOError:
+                pass
+            else: # no exception, so file was found
+                return expowebpage(request, expowebpath, Path(path) / p)
+        return render(request, 'pagenotfound.html', {'path': Path(path) / "index.html"})
+        
+    if path.endswith("/"):
+        # we already know it is not a directory.
+        # the final / may have been appended by middleware if there was no page without it
+        # do not redirect to a file path without the slash as we may get in a loop. Let the user fix it:
+        return render(request, 'dirnotfound.html', {'path': path, 'subpath': path[0:-1]})
+   
+    if path.startswith('site_media'): # BUT we may have missing files, directories or .html here too?!
+        # print(" - MEDIA_ROOT: {}  ...{}".format(settings.MEDIA_ROOT, path))
+        npath = path.replace("site_media", settings.MEDIA_ROOT)
+        filetobeopened = os.path.normpath(npath)
+    elif path.startswith("static"):
+        # print(" - STATIC_ROOT: {}  ...{}".format(settings.MEDIA_ROOT, path))
+        npath = path.replace("static", settings.MEDIA_ROOT)
+        filetobeopened = os.path.normpath(npath)
+    else:
+        filetobeopened = os.path.normpath(expowebpath / path)
+    
+    try:
+        return HttpResponse(content=open(filetobeopened, "rb"), content_type=getmimetype(path))
+    except IOError:
+        return render(request, 'pagenotfound.html', {'path': path})
+ 
+ 
+
+def getmimetype(path):
+    path = str(path)
+    if path.lower().endswith(".css"): return "text/css"
+    if path.lower().endswith(".txt"): return "text/css"
+    if path.lower().endswith(".js"): return "application/javascript"
+    if path.lower().endswith(".json"): return "application/javascript"
+    if path.lower().endswith(".ico"): return "image/vnd.microsoft.icon"
+    if path.lower().endswith(".png"): return "image/png"
+    if path.lower().endswith(".tif"): return "image/tif"
+    if path.lower().endswith(".gif"): return "image/gif"
+    if path.lower().endswith(".jpeg"): return "image/jpeg"
+    if path.lower().endswith(".jpg"): return "image/jpeg"
+    if path.lower().endswith("svg"): return "image/svg+xml"
+    if path.lower().endswith("xml"): return "application/xml" # we use "text/xhtml" for tunnel files
+    if path.lower().endswith(".pdf"): return "application/pdf"
+    if path.lower().endswith(".ps"): return "application/postscript"
+    if path.lower().endswith(".svx"): return "application/x-survex-svx"
+    if path.lower().endswith(".3d"): return "application/x-survex-3d"
+    if path.lower().endswith(".pos"): return "application/x-survex-pos"
+    if path.lower().endswith(".err"): return "application/x-survex-err"
+    if path.lower().endswith(".odt"): return "application/vnd.oasis.opendocument.text"
+    if path.lower().endswith(".ods"): return "application/vnd.oasis.opendocument.spreadsheet"
+    if path.lower().endswith(".docx"): return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    if path.lower().endswith(".xslx"): return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+    if path.lower().endswith(".gz"): return "application/x-7z-compressed"
+    if path.lower().endswith(".7z"): return "application/x-7z-compressed"
+    if path.lower().endswith(".zip"): return "application/zip"
+    return ""
+
+@login_required_if_public
+@ensure_csrf_cookie
+def editexpopage(request, path):
+    try:
+        r = Cave.objects.get(url = path)
+        return troggle.core.views.caves.editCave(request, r.cave.slug)
+    except Cave.DoesNotExist:
+        pass
+
+
+    try:
+        filepath = Path(settings.EXPOWEB) / path
+        o = open(filepath, "r")
+        html = o.read()
+        autogeneratedmatch = re.search(r"\<\!--\s*(.*?(Do not edit|auto-generated).*?)\s*--\>", html, re.DOTALL + re.IGNORECASE)
+        if autogeneratedmatch:
+            return HttpResponse(autogeneratedmatch.group(1))
+        m = re.search(r"(.*)<head([^>]*)>(.*)</head>(.*)<body([^>]*)>(.*)</body>(.*)", html, re.DOTALL + re.IGNORECASE)
+        if m:
+            filefound = True
+            preheader, headerargs, head, postheader, bodyargs, body, postbody = m.groups()
+            linksmatch = re.match(r'(.*)(<ul\s+id="links">.*)', body, re.DOTALL + re.IGNORECASE)
+            if linksmatch:
+                body, links = linksmatch.groups()
+#            if re.search(r"iso-8859-1", html):
+#                body = str(body, "iso-8859-1")
+        else:
+            return HttpResponse("Page could not be split into header and body")
+    except IOError:
+        print("### File not found ### ", filepath)
+        filefound = False
+        
+
+    if request.method == 'POST': # If the form has been submitted...
+        flatpageForm = FlatPageForm(request.POST) # A form bound to the POST data
+        if flatpageForm.is_valid():# Form valid therefore write file
+            print("### \n", str(flatpageForm)[0:300])
+            print("### \n csrfmiddlewaretoken: ",request.POST['csrfmiddlewaretoken'])
+            if filefound:
+                headmatch =  re.match(r"(.*)<title>.*</title>(.*)", head, re.DOTALL + re.IGNORECASE)
+                if headmatch:
+                    head = headmatch.group(1) + "<title>" + flatpageForm.cleaned_data["title"] + "</title>" + headmatch.group(2)
+                else:
+                    head = "<title>" + flatpageForm.cleaned_data["title"] + "</title>"
+            else:
+                head = "<title>" + flatpageForm.cleaned_data["title"] + "</title>"
+                preheader = "<html>"
+                headerargs = ""
+                postheader = ""
+                bodyargs = ""
+                postbody = "</html>" 
+            body = flatpageForm.cleaned_data["html"]
+            body = body.replace("\r", "")
+            result = "%s<head%s>%s</head>%s<body%s>\n%s</body>%s" % (preheader, headerargs, head, postheader, bodyargs, body, postbody)
+            f = open(filepath, "w")
+            f.write(result)
+            f.close()
+            return HttpResponseRedirect(reverse('flatpage', args=[path])) # Redirect after POST
+    else:
+        if filefound:
+            m = re.search(r"<title>(.*)</title>", head, re.DOTALL + re.IGNORECASE)
+            if m: 
+                title, = m.groups()
+            else:
+                title = ""
+            flatpageForm = FlatPageForm({"html": body, "title": title})
+        else:
+            body = "### File not found ###\n"    + str(filepath)
+            flatpageForm = FlatPageForm({"html": body, "title": "Missing"})
+    return render(request, 'editexpopage.html', {'path': path, 'form': flatpageForm, })
+
+class FlatPageForm(forms.Form):
+    title = forms.CharField(widget=forms.TextInput(attrs={'size':'60'}))
+
+    #html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 20}))
+    html = forms.CharField(widget=forms.Textarea(attrs={"cols":80, "rows":20}))

core/views/logbooks.py

@@ -0,0 +1,270 @@
+import datetime
+import os.path
+import re
+
+import django.db.models
+from django.db.models import Min, Max
+from django.urls import reverse
+from django.http import HttpResponse, HttpResponseRedirect
+from django.shortcuts import render
+from django.template import Context, loader
+from django.template.defaultfilters import slugify
+from django.utils import timezone
+from django.views.generic.list import ListView
+
+import troggle.core.models as models
+import troggle.parsers.logbooks as logbookparsers
+from troggle.core.forms import getTripForm  # , get_name, PersonForm
+from troggle.core.models import Expedition, Person, PersonExpedition
+from troggle.core.models_caves import LogbookEntry, PersonTrip
+from troggle.core.models_survex import SurvexBlock
+from troggle.helper import login_required_if_public
+from troggle.parsers.logbooks import LoadLogbookForExpedition
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
+import troggle.settings as settings
+
+# Django uses Context, not RequestContext when you call render
+# to_response. We always want to use RequestContext, so that
+# django adds the context from settings.TEMPLATE_CONTEXT_PROCESSORS.
+# This way we automatically get necessary settings variables passed
+# to each template. So we use a custom method, render_response
+# instead of render_to_response. Hopefully future Django releases
+# will make this unnecessary.
+
+# from troggle.alwaysUseRequestContext import render_response
+
+# Deprecated in 1.11.29
+# @django.db.models.permalink #this allows the nice get_absolute_url syntax we are using
+
+def getNotablePersons():
+    notablepersons = []
+    for person in Person.objects.all():
+            if person.bisnotable():
+                notablepersons.append(person)
+    return notablepersons
+
+
+def personindex(request):
+    persons = Person.objects.all()
+    # From what I can tell, "persons" seems to be the table rows, while "personss" is the table columns. - AC 16 Feb 09
+    personss = [ ]
+    ncols = 4
+    nc = int((len(persons) + ncols - 1) / ncols)
+    for i in range(ncols):
+        personss.append(persons[i * nc: (i + 1) * nc])
+    
+    notablepersons = []
+    for person in Person.objects.all():
+            if person.bisnotable():
+                notablepersons.append(person)
+
+    return render(request,'personindex.html', {'persons': persons, 'personss':personss, 'notablepersons':notablepersons})
+
+
+def expedition(request, expeditionname):
+    this_expedition = Expedition.objects.get(year=int(expeditionname))
+    expeditions = Expedition.objects.all()
+    personexpeditiondays = [ ]
+    dateditems = list(this_expedition.logbookentry_set.all()) + list(this_expedition.survexblock_set.all())
+    dates = sorted(set([item.date for item in dateditems]))
+    for personexpedition in this_expedition.personexpedition_set.all():
+        prow = [ ]
+        for date in dates:
+            pcell = { "persontrips": PersonTrip.objects.filter(personexpedition=personexpedition, 
+                                                                logbook_entry__date=date) }
+            pcell["survexblocks"] = set(SurvexBlock.objects.filter(survexpersonrole__personexpedition=personexpedition, 
+                                                                    date = date))
+            prow.append(pcell)
+        personexpeditiondays.append({"personexpedition":personexpedition, "personrow":prow})
+        
+    if "reload" in request.GET:
+        LoadLogbookForExpedition(this_expedition)
+    return render(request,'expedition.html', {'expedition': this_expedition, 'expeditions':expeditions, 'personexpeditiondays':personexpeditiondays, 'settings':settings, 'dateditems': dateditems })
+
+def get_absolute_url(self):
+    return ('expedition', (expedition.year))
+
+# replaced by statistics page
+# class ExpeditionListView(ListView): # django thus expects a template called "expedition_list.html" 
+# # from the name of the object not the name of the class.
+    # model = Expedition
+
+
+class Expeditions_tsvListView(ListView): 
+    """This uses the Django built-in shortcut mechanism 
+    It defaults to use a template with name <app-label>/<model-name>_list.html.
+    https://www.agiliq.com/blog/2017/12/when-and-how-use-django-listview/
+    https://developer.mozilla.org/en-US/docs/Learn/Server-side/Django/Generic_views
+    Either a queryset variable  or set_queryset() function is used, but not needed
+    if you want all the obejcts of a particaulr type in which case just set model = <object>
+    """
+    template_name = 'core/expeditions_tsv_list.html' # if not present then uses core/expedition_list.html 
+    #queryset = Expedition.objects.all()
+    #context_object_name = 'expedition'
+    model = Expedition # equivalent to .objects.all() for a queryset
+
+class Expeditions_jsonListView(ListView): 
+    template_name = 'core/expeditions_json_list.html' 
+    model = Expedition 
+
+
+def person(request, first_name='', last_name='', ):
+    this_person = Person.objects.get(first_name = first_name, last_name = last_name)
+    
+    # This is for removing the reference to the user's profile, in case they set it to the wrong person
+    if request.method == 'GET':
+        if request.GET.get('clear_profile')=='True':
+            this_person.user=None
+            this_person.save()
+            return HttpResponseRedirect(reverse('profiles_select_profile'))
+    
+    return render(request,'person.html', {'person': this_person, })
+
+
+def GetPersonChronology(personexpedition):
+    '''Horrible bug here whern ther eis more than one survex block per day, it duplicates the entry but gets it wrong
+    Fortunately this is just the display on this page which is wroing, no bad calculations get into the database.
+    '''
+    res = { }
+    for persontrip in personexpedition.persontrip_set.all():
+        a = res.setdefault(persontrip.logbook_entry.date, { })
+        a.setdefault("persontrips", [ ]).append(persontrip)
+
+    for personrole in personexpedition.survexpersonrole_set.all():
+        a = res.setdefault(personrole.survexblock.date, { })
+        a.setdefault("personroles", [ ]).append(personrole.survexblock)
+    
+    # build up the tables
+    rdates = sorted(list(res.keys()))
+
+    res2 = [ ]
+    for rdate in rdates:
+        persontrips   = res[rdate].get("persontrips", [])
+        personroles   = res[rdate].get("personroles", [])
+        for n in range(max(len(persontrips), len(personroles) )):
+            res2.append(((n == 0 and rdate or "--"), (n < len(persontrips) and persontrips[n]), (n < len(personroles) and personroles[n]) ))
+            
+    return res2
+
+
+def personexpedition(request, first_name='',  last_name='', year=''):
+    person = Person.objects.get(first_name = first_name, last_name = last_name)
+    this_expedition = Expedition.objects.get(year=year)
+    personexpedition = person.personexpedition_set.get(expedition=this_expedition)
+    personchronology = GetPersonChronology(personexpedition)
+    return render(request,'personexpedition.html', {'personexpedition': personexpedition, 'personchronology':personchronology})
+
+
+def logbookentry(request, date, slug):
+    this_logbookentry = LogbookEntry.objects.filter(date=date, slug=slug)
+
+    if len(this_logbookentry)>1:
+        return render(request, 'object_list.html',{'object_list':this_logbookentry})
+    else:
+        this_logbookentry=this_logbookentry[0]
+        return render(request, 'logbookentry.html', {'logbookentry': this_logbookentry})
+
+
+def logbookSearch(request, extra):
+    query_string = ''
+    found_entries = None
+    if ('q' in request.GET) and request.GET['q'].strip():
+        query_string = request.GET['q']
+    entry_query = search.get_query(query_string, ['text','title',])
+    found_entries = LogbookEntry.objects.filter(entry_query)
+
+    return render(request,'logbooksearch.html',
+                          { 'query_string': query_string, 'found_entries': found_entries, })
+                          #context_instance=RequestContext(request))
+
+def personForm(request,pk):
+    person=Person.objects.get(pk=pk)
+    form=PersonForm(instance=person)
+    return render(request,'personform.html', {'form':form,})
+
+# tried to delete all this, and the reference in urls.py, but got impenetrable django error message
+# @login_required_if_public
+# def newLogbookEntry(request, expeditionyear, pdate = None, pslug = None):
+    # expedition = Expedition.objects.get(year=expeditionyear)
+    # PersonTripFormSet, TripForm = getTripForm(expedition)
+    # if pslug and pdate:
+        # previousdate = datetime.date(*[int(x) for x in pdate.split("-")])
+        # previouslbe = LogbookEntry.objects.get(slug = pslug, date = previousdate, expedition = expedition)
+        # assert previouslbe.filename
+    # if request.method == 'POST': # If the form has been submitted...
+        # tripForm = TripForm(request.POST) # A form bound to the POST data
+        # personTripFormSet = PersonTripFormSet(request.POST)
+        # if tripForm.is_valid() and personTripFormSet.is_valid(): # All validation rules pass
+            # dateStr = tripForm.cleaned_data["date"].strftime("%Y-%m-%d")
+            # directory = os.path.join(settings.EXPOWEB, 
+                                    # "years", 
+                                    # expedition.year, 
+                                    # "autologbook")
+            # filename = os.path.join(directory, 
+                                    # dateStr + "." + slugify(tripForm.cleaned_data["title"])[:50] + ".html")
+            # if not os.path.isdir(directory):
+                # os.mkdir(directory)
+            # if pslug and pdate:
+                # delLogbookEntry(previouslbe)
+            # f = open(filename, "w")
+            # template = loader.get_template('dataformat/logbookentry.html')
+            # context = Context({'trip': tripForm.cleaned_data, 
+                               # 'persons': personTripFormSet.cleaned_data,
+                               # 'date': dateStr,
+                               # 'expeditionyear': expeditionyear})
+            # f.write(template.render(context))
+            # f.close()
+            # print((logbookparsers.parseAutoLogBookEntry(filename)))
+            # return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST
+    # else:
+        # if pslug and pdate:
+            # if previouslbe.cave:
+                # tripForm = TripForm(initial={"date": previousdate,
+                                             # "title": previouslbe.title,
+                                             # "cave": previouslbe.cave.reference(),
+                                             # "location": None,
+                                             # "caveOrLocation": "cave",
+                                             # "html": previouslbe.text})
+            # else:
+                # tripForm = TripForm(initial={"date": previousdate,
+                                             # "title": previouslbe.title,
+                                             # "cave": None,
+                                             # "location": previouslbe.place,
+                                             # "caveOrLocation": "location",
+                                             # "html": previouslbe.text})
+            # personTripFormSet = PersonTripFormSet(initial=[{"name": get_name(py.personexpedition), 
+                                                            # "TU": py.time_underground, 
+                                                            # "author": py.is_logbook_entry_author}
+                                                           # for py in previouslbe.persontrip_set.all()])
+        # else:        
+            # tripForm = TripForm() # An unbound form
+            # personTripFormSet = PersonTripFormSet()
+
+    # return render(request, 'newlogbookentry.html', {
+        # 'tripForm': tripForm,
+        # 'personTripFormSet': personTripFormSet,
+
+    # })
+
+# @login_required_if_public
+# def deleteLogbookEntry(request, expeditionyear, date = None, slug = None):
+    # expedition = Expedition.objects.get(year=expeditionyear)
+    # previousdate = datetime.date(*[int(x) for x in date.split("-")])
+    # previouslbe = LogbookEntry.objects.get(slug = slug, date = previousdate, expedition = expedition)
+    # delLogbookEntry(previouslbe)
+    # return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST
+
+# def delLogbookEntry(lbe):
+    # for pt in lbe.persontrip_set.all():
+        # pt.delete()
+    # lbe.delete()
+    # os.remove(lbe.filename)
+
+def get_people(request, expeditionslug):
+    exp = Expedition.objects.get(year = expeditionslug)
+    return render(request,'options.html', {"items": [(pe.slug, pe.name) for pe in exp.personexpedition_set.all()]})
+
+def get_logbook_entries(request, expeditionslug):
+    exp = Expedition.objects.get(year = expeditionslug)
+    return render(request,'options.html', {"items": [(le.slug, "%s - %s" % (le.date, le.title)) for le in exp.logbookentry_set.all()]})

core/views/other.py

@@ -0,0 +1,197 @@
+import re
+
+from django import forms
+from django.conf import settings
+from django.urls import reverse
+from django.db.models import Q
+from django.http import HttpResponse, HttpResponseRedirect
+from django.shortcuts import render
+from django.template import Context, loader
+
+import troggle.parsers.imports
+from troggle.core.models import Expedition, Person, PersonExpedition
+from troggle.core.models_caves import LogbookEntry, QM, Cave, PersonTrip
+from troggle.helper import login_required_if_public
+from troggle.core.forms import UploadFileForm
+
+print("** importing troggle/core/views.other.py")
+
+"""Utility functions and code to serve the control panel and individual user's 
+progress and task list (deprecated as we do not have individual user login).
+
+Also has code to download a logbook in a choice of formats (why?!) and to
+download all QMs (not working)
+"""
+
+def troggle404(request): # cannot get this to work. Handler404 in urls.py not right syntax
+    '''Custom 404 page to be used even when Debug=True
+    https://blog.juanwolf.fr/posts/programming/how-to-create-404-page-django/
+    '''
+    context = RequestContext(request)
+    #context['caves'] = Cave.objects.all()
+    return render(request, ('404.html', context.flatten()))
+
+def showrequest(request):
+    return HttpResponse(request.GET)
+
+def frontpage(request): 
+    '''never seen in practice'''
+    # bthe messages system does a popup on this page if there is a recent message, e.g. from the admin site actions.
+    # via django.contrib.messages.middleware.MessageMiddleware 
+    # this is set in the templates.
+    if request.user.is_authenticated():
+        return render(request,'tasks.html')
+
+    expeditions =  Expedition.objects.order_by("-year")
+    logbookentry = LogbookEntry
+    cave = Cave
+    #from django.contrib.admin.templatetags import log
+    return render(request,'frontpage.html', locals())
+
+
+def controlPanel(request):
+    jobs_completed=[]
+    if request.method=='POST':
+        if request.user.is_superuser:
+            # NONE of this works now that databaseReset (now parsers.imports) has been so extensively rewritten
+            reinit_db()
+            import_caves()
+            import_people()
+            import_surveyscans()
+            import_logbooks()
+            import_QMs()
+            import_tunnelfiles()
+            import_survexblks()
+            import_survexpos()
+        else:
+            if request.user.is_authenticated(): #The user is logged in, but is not a superuser.
+                return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'error':'You must be a superuser to use that feature.'})
+            else:
+                return HttpResponseRedirect(reverse('auth_login'))
+
+    return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
+
+
+def downloadLogbook(request,year=None,extension=None,queryset=None):
+    
+    if year:
+        current_expedition=Expedition.objects.get(year=year)
+        logbook_entries=LogbookEntry.objects.filter(expedition=current_expedition)
+        filename='logbook'+year
+    elif queryset:
+        logbook_entries=queryset
+        filename='logbook'
+    else:
+        response = HttpResponse(content_type='text/plain')
+        return response(r"Error: Logbook downloader doesn't know what year you want")
+    
+    if 'year' in request.GET:
+        year=request.GET['year']
+    if 'extension' in request.GET:
+        extension=request.GET['extension']
+
+    if extension =='txt':
+        response = HttpResponse(content_type='text/plain')
+        style='2008'
+    elif extension == 'html':
+        response = HttpResponse(content_type='text/html')
+        style='2005'
+        
+    template='logbook'+style+'style.'+extension
+    response['Content-Disposition'] = 'attachment; filename='+filename+'.'+extension 
+    t=loader.get_template(template)
+    c=Context({'logbook_entries':logbook_entries})
+    response.write(t.render(c))
+    return response
+
+
+def downloadQMs(request):
+    # Note to self: use get_cave method for the below
+    if request.method=='GET':
+        try:
+            cave=Cave.objects.get(kataster_number=request.GET['cave_id'])
+        except Cave.DoesNotExist:
+            cave=Cave.objects.get(name=request.GET['cave_id'])
+
+    from export import toqms
+
+    response = HttpResponse(content_type='text/csv')
+    response['Content-Disposition'] = 'attachment; filename=qm.csv'
+    toqms.writeQmTable(response,cave)
+    return response
+    
+def ajax_test(request):
+    post_text = request.POST['post_data']
+    return HttpResponse("{'response_text': '"+post_text+" recieved.'}", 
+                                   content_type="application/json")
+                                   
+def eyecandy(request):
+    return 
+
+def ajax_QM_number(request):
+    res=""
+    if request.method=='POST':
+        cave=Cave.objects.get(id=request.POST['cave'])
+        print(cave)
+        exp=Expedition.objects.get(pk=request.POST['year'])
+        print(exp)
+        res=cave.new_QM_number(exp.year)
+
+    return HttpResponse(res)
+
+
+print(" -  newFile() is next in troggle/core/views.other.py")
+
+@login_required_if_public
+def newFile(request, pslug = None):
+    if pslug:
+        previousfile = LogbookEntry.objects.get(slug = pslug, date = previousdate, expedition = expedition)
+        #assert previousfile.filename
+    if request.method == 'POST': # If the form has been submitted...
+        tripForm = TripForm(request.POST) # A form bound to the POST data
+        personTripFormSet = PersonTripFormSet(request.POST)
+        if tripForm.is_valid() and personTripFormSet.is_valid(): # All validation rules pass
+            dateStr = tripForm.cleaned_data["date"].strftime("%Y-%m-%d")
+            directory = os.path.join(settings.EXPOWEB, 
+                                   "years", 
+                                   expedition.year, 
+                                   "autologbook")
+            filename = os.path.join(directory, 
+                                   dateStr + "." + slugify(tripForm.cleaned_data["title"])[:50] + ".html")
+            if not os.path.isdir(directory):
+                os.mkdir(directory)
+            if pslug and pdate:
+                delLogbookEntry(previouslbe)
+            f = open(filename, "w")
+            template = loader.get_template('dataformat/logbookentry.html')
+            context = Context({'trip': tripForm.cleaned_data, 
+                              'persons': personTripFormSet.cleaned_data,
+                              'date': dateStr,
+                              'expeditionyear': expeditionyear})
+            f.write(template.render(context))
+            f.close()
+            print(logbookparsers.parseAutoLogBookEntry(filename))
+            return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST
+    else:
+        if pslug:
+            pass
+        else:        
+            fileform = UploadFileForm() # An unbound form
+
+    return render(request, 'editfile.html', {'fileForm': fileform, })
+
+@login_required_if_public
+def deleteFile(request, expeditionyear, date = None, slug = None):
+    expedition = Expedition.objects.get(year=expeditionyear)
+    previousdate = datetime.date(*[int(x) for x in date.split("-")])
+    previouslbe = LogbookEntry.objects.get(slug = slug, date = previousdate, expedition = expedition)
+    delLogbookEntry(previouslbe)
+    return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST
+
+def delFile(f):
+    for pt in lbe.persontrip_set.all():
+        pt.delete()
+    lbe.delete()
+    os.remove(lbe.filename)
+
+print("** Finished importing troggle/core/views.other.py")

core/views/persons.py

@@ -0,0 +1,269 @@
+import datetime
+import os.path
+import re
+
+import django.db.models
+from django.db.models import Min, Max
+from django.urls import reverse
+from django.http import HttpResponse, HttpResponseRedirect
+from django.shortcuts import render
+from django.template import Context, loader
+from django.template.defaultfilters import slugify
+from django.utils import timezone
+from django.views.generic.list import ListView
+
+import troggle.core.models as models
+import troggle.parsers.logbooks as logbookparsers
+from troggle.core.forms import getTripForm  # , get_name, PersonForm
+from troggle.core.models import Expedition, Person, PersonExpedition
+from troggle.core.models_caves import LogbookEntry, PersonTrip
+from troggle.core.models_survex import SurvexBlock
+from troggle.helper import login_required_if_public
+from troggle.parsers.logbooks import LoadLogbookForExpedition
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
+import troggle.settings as settings
+
+# Django uses Context, not RequestContext when you call render
+# to_response. We always want to use RequestContext, so that
+# django adds the context from settings.TEMPLATE_CONTEXT_PROCESSORS.
+# This way we automatically get necessary settings variables passed
+# to each template. So we use a custom method, render_response
+# instead of render_to_response. Hopefully future Django releases
+# will make this unnecessary.
+
+# from troggle.alwaysUseRequestContext import render_response
+
+# Deprecated in 1.11.29
+# @django.db.models.permalink #this allows the nice get_absolute_url syntax we are using
+
+def getNotablePersons():
+    notablepersons = []
+    for person in Person.objects.all():
+            if person.bisnotable():
+                notablepersons.append(person)
+    return notablepersons
+
+
+def personindex(request):
+    persons = Person.objects.all()
+    # From what I can tell, "persons" seems to be the table rows, while "personss" is the table columns. - AC 16 Feb 09
+    personss = [ ]
+    ncols = 4
+    nc = int((len(persons) + ncols - 1) / ncols)
+    for i in range(ncols):
+        personss.append(persons[i * nc: (i + 1) * nc])
+    
+    notablepersons = []
+    for person in Person.objects.all():
+            if person.bisnotable():
+                notablepersons.append(person)
+
+    return render(request,'personindex.html', {'persons': persons, 'personss':personss, 'notablepersons':notablepersons})
+
+
+def expedition(request, expeditionname):
+    this_expedition = Expedition.objects.get(year=int(expeditionname))
+    expeditions = Expedition.objects.all()
+    personexpeditiondays = [ ]
+    dateditems = list(this_expedition.logbookentry_set.all()) + list(this_expedition.survexblock_set.all())
+    dates = sorted(set([item.date for item in dateditems]))
+    for personexpedition in this_expedition.personexpedition_set.all():
+        prow = [ ]
+        for date in dates:
+            pcell = { "persontrips": PersonTrip.objects.filter(personexpedition=personexpedition, 
+                                                                logbook_entry__date=date) }
+            pcell["survexblocks"] = set(SurvexBlock.objects.filter(survexpersonrole__personexpedition=personexpedition, 
+                                                                    date = date))
+            prow.append(pcell)
+        personexpeditiondays.append({"personexpedition":personexpedition, "personrow":prow})
+        
+    if "reload" in request.GET:
+        LoadLogbookForExpedition(this_expedition)
+    return render(request,'expedition.html', {'expedition': this_expedition, 'expeditions':expeditions, 'personexpeditiondays':personexpeditiondays, 'settings':settings, 'dateditems': dateditems })
+
+def get_absolute_url(self):
+    return ('expedition', (expedition.year))
+
+class ExpeditionListView(ListView): # django thus expects a template called "expedition_list.html" 
+# from the name of the object not the name of the class.
+    model = Expedition
+
+
+class Expeditions_tsvListView(ListView): 
+    """This uses the Django built-in shortcut mechanism 
+    It defaults to use a template with name <app-label>/<model-name>_list.html.
+    https://www.agiliq.com/blog/2017/12/when-and-how-use-django-listview/
+    https://developer.mozilla.org/en-US/docs/Learn/Server-side/Django/Generic_views
+    Either a queryset variable  or set_queryset() function is used, but not needed
+    if you want all the obejcts of a particaulr type in which case just set model = <object>
+    """
+    template_name = 'core/expeditions_tsv_list.html' # if not present then uses core/expedition_list.html 
+    #queryset = Expedition.objects.all()
+    #context_object_name = 'expedition'
+    model = Expedition # equivalent to .objects.all() for a queryset
+
+class Expeditions_jsonListView(ListView): 
+    template_name = 'core/expeditions_json_list.html' 
+    model = Expedition 
+
+
+def person(request, first_name='', last_name='', ):
+    this_person = Person.objects.get(first_name = first_name, last_name = last_name)
+    
+    # This is for removing the reference to the user's profile, in case they set it to the wrong person
+    if request.method == 'GET':
+        if request.GET.get('clear_profile')=='True':
+            this_person.user=None
+            this_person.save()
+            return HttpResponseRedirect(reverse('profiles_select_profile'))
+    
+    return render(request,'person.html', {'person': this_person, })
+
+
+def GetPersonChronology(personexpedition):
+    '''Horrible bug here whern there is more than one survex block per day, it duplicates the entry but gets it wrong
+    Fortunately this is just the display on this page which is wroing, no bad calculations get into the database.
+    '''
+    res = { }
+    for persontrip in personexpedition.persontrip_set.all():
+        a = res.setdefault(persontrip.logbook_entry.date, { })
+        a.setdefault("persontrips", [ ]).append(persontrip)
+
+    for personrole in personexpedition.survexpersonrole_set.all():
+        a = res.setdefault(personrole.survexblock.date, { })
+        a.setdefault("personroles", [ ]).append(personrole.survexblock)
+    
+    # build up the tables
+    rdates = sorted(list(res.keys()))
+
+    res2 = [ ]
+    for rdate in rdates:
+        persontrips   = res[rdate].get("persontrips", [])
+        personroles   = res[rdate].get("personroles", [])
+        for n in range(max(len(persontrips), len(personroles) )):
+            res2.append(((n == 0 and rdate or "--"), (n < len(persontrips) and persontrips[n]), (n < len(personroles) and personroles[n]) ))
+            
+    return res2
+
+
+def personexpedition(request, first_name='',  last_name='', year=''):
+    person = Person.objects.get(first_name = first_name, last_name = last_name)
+    this_expedition = Expedition.objects.get(year=year)
+    personexpedition = person.personexpedition_set.get(expedition=this_expedition)
+    personchronology = GetPersonChronology(personexpedition)
+    return render(request,'personexpedition.html', {'personexpedition': personexpedition, 'personchronology':personchronology})
+
+
+def logbookentry(request, date, slug):
+    this_logbookentry = LogbookEntry.objects.filter(date=date, slug=slug)
+
+    if len(this_logbookentry)>1:
+        return render(request, 'object_list.html',{'object_list':this_logbookentry})
+    else:
+        this_logbookentry=this_logbookentry[0]
+        return render(request, 'logbookentry.html', {'logbookentry': this_logbookentry})
+
+
+def logbookSearch(request, extra):
+    query_string = ''
+    found_entries = None
+    if ('q' in request.GET) and request.GET['q'].strip():
+        query_string = request.GET['q']
+    entry_query = search.get_query(query_string, ['text','title',])
+    found_entries = LogbookEntry.objects.filter(entry_query)
+
+    return render(request,'logbooksearch.html',
+                          { 'query_string': query_string, 'found_entries': found_entries, })
+                          #context_instance=RequestContext(request))
+
+def personForm(request,pk):
+    person=Person.objects.get(pk=pk)
+    form=PersonForm(instance=person)
+    return render(request,'personform.html', {'form':form,})
+
+# tried to delete all this, and the reference in urls.py, but got impenetrable django error message
+# @login_required_if_public
+# def newLogbookEntry(request, expeditionyear, pdate = None, pslug = None):
+    # expedition = Expedition.objects.get(year=expeditionyear)
+    # PersonTripFormSet, TripForm = getTripForm(expedition)
+    # if pslug and pdate:
+        # previousdate = datetime.date(*[int(x) for x in pdate.split("-")])
+        # previouslbe = LogbookEntry.objects.get(slug = pslug, date = previousdate, expedition = expedition)
+        # assert previouslbe.filename
+    # if request.method == 'POST': # If the form has been submitted...
+        # tripForm = TripForm(request.POST) # A form bound to the POST data
+        # personTripFormSet = PersonTripFormSet(request.POST)
+        # if tripForm.is_valid() and personTripFormSet.is_valid(): # All validation rules pass
+            # dateStr = tripForm.cleaned_data["date"].strftime("%Y-%m-%d")
+            # directory = os.path.join(settings.EXPOWEB, 
+                                    # "years", 
+                                    # expedition.year, 
+                                    # "autologbook")
+            # filename = os.path.join(directory, 
+                                    # dateStr + "." + slugify(tripForm.cleaned_data["title"])[:50] + ".html")
+            # if not os.path.isdir(directory):
+                # os.mkdir(directory)
+            # if pslug and pdate:
+                # delLogbookEntry(previouslbe)
+            # f = open(filename, "w")
+            # template = loader.get_template('dataformat/logbookentry.html')
+            # context = Context({'trip': tripForm.cleaned_data, 
+                               # 'persons': personTripFormSet.cleaned_data,
+                               # 'date': dateStr,
+                               # 'expeditionyear': expeditionyear})
+            # f.write(template.render(context))
+            # f.close()
+            # print((logbookparsers.parseAutoLogBookEntry(filename)))
+            # return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST
+    # else:
+        # if pslug and pdate:
+            # if previouslbe.cave:
+                # tripForm = TripForm(initial={"date": previousdate,
+                                             # "title": previouslbe.title,
+                                             # "cave": previouslbe.cave.reference(),
+                                             # "location": None,
+                                             # "caveOrLocation": "cave",
+                                             # "html": previouslbe.text})
+            # else:
+                # tripForm = TripForm(initial={"date": previousdate,
+                                             # "title": previouslbe.title,
+                                             # "cave": None,
+                                             # "location": previouslbe.place,
+                                             # "caveOrLocation": "location",
+                                             # "html": previouslbe.text})
+            # personTripFormSet = PersonTripFormSet(initial=[{"name": get_name(py.personexpedition), 
+                                                            # "TU": py.time_underground, 
+                                                            # "author": py.is_logbook_entry_author}
+                                                           # for py in previouslbe.persontrip_set.all()])
+        # else:        
+            # tripForm = TripForm() # An unbound form
+            # personTripFormSet = PersonTripFormSet()
+
+    # return render(request, 'newlogbookentry.html', {
+        # 'tripForm': tripForm,
+        # 'personTripFormSet': personTripFormSet,
+
+    # })
+
+# @login_required_if_public
+# def deleteLogbookEntry(request, expeditionyear, date = None, slug = None):
+    # expedition = Expedition.objects.get(year=expeditionyear)
+    # previousdate = datetime.date(*[int(x) for x in date.split("-")])
+    # previouslbe = LogbookEntry.objects.get(slug = slug, date = previousdate, expedition = expedition)
+    # delLogbookEntry(previouslbe)
+    # return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST
+
+# def delLogbookEntry(lbe):
+    # for pt in lbe.persontrip_set.all():
+        # pt.delete()
+    # lbe.delete()
+    # os.remove(lbe.filename)
+
+def get_people(request, expeditionslug):
+    exp = Expedition.objects.get(year = expeditionslug)
+    return render(request,'options.html', {"items": [(pe.slug, pe.name) for pe in exp.personexpedition_set.all()]})
+
+def get_logbook_entries(request, expeditionslug):
+    exp = Expedition.objects.get(year = expeditionslug)
+    return render(request,'options.html', {"items": [(le.slug, "%s - %s" % (le.date, le.title)) for le in exp.logbookentry_set.all()]})

core/views/statistics.py

@@ -0,0 +1,143 @@
+import datetime
+import os.path
+import re
+from collections import OrderedDict
+
+import django.db.models
+from django.db.models import Min, Max
+from django.shortcuts import render
+from django.template import Context, loader
+from django.template.defaultfilters import slugify
+from django.utils import timezone
+#from django.views.generic.list import ListView
+
+from troggle.core.models import Expedition, Person, PersonExpedition
+from troggle.core.models_caves import Cave, LogbookEntry
+from troggle.core.models_survex import SurvexBlock
+
+import troggle.settings as settings
+
+
+def pathsreport(request):
+    pathsdict = OrderedDict()
+    try:
+        pathsdict = {
+#       "BOGUS" : str( settings.BOGUS),
+        "JSLIB_URL" : str( settings.JSLIB_URL),
+#       "CSSLIB_URL" : str( settings.CSSLIB_URL),
+        "CAVEDESCRIPTIONS" : str( settings.CAVEDESCRIPTIONS),
+        "DIR_ROOT" : str( settings.DIR_ROOT),
+        "ENTRANCEDESCRIPTIONS" : str( settings.ENTRANCEDESCRIPTIONS),
+        "EXPOUSER_EMAIL" : str( settings.EXPOUSER_EMAIL),
+        "EXPOUSERPASS" : str("<redacted>"),
+        "EXPOUSER" : str( settings.EXPOUSER),
+        "EXPOWEB" : str( settings.EXPOWEB),
+        "EXPOWEB_URL" : str( settings.EXPOWEB_URL),
+        "FILES" : str( settings.FILES),
+        "LOGFILE" : str( settings.LOGFILE),
+        "LOGIN_REDIRECT_URL" : str( settings.LOGIN_REDIRECT_URL),
+        "MEDIA_ROOT" : str( settings.MEDIA_ROOT),
+        "MEDIA_URL" : str( settings.MEDIA_URL),
+        "PHOTOS_URL" : str( settings.PHOTOS_URL),
+        "PYTHON_PATH" : str( settings.PYTHON_PATH),
+        "REPOS_ROOT_PATH" : str( settings.REPOS_ROOT_PATH),
+        "ROOT_URLCONF" : str( settings.ROOT_URLCONF),
+        "STATIC_URL" : str( settings.STATIC_URL),
+        "SURVEX_DATA" : str( settings.SURVEX_DATA),
+        "SURVEY_SCANS" : str( settings.SURVEY_SCANS),
+        "SURVEYS" : str( settings.SURVEYS),
+        "SURVEYS_URL" : str( settings.SURVEYS_URL),
+        "SURVEXPORT" : str( settings.SURVEXPORT),
+        "THREEDCACHEDIR" : str( settings.THREEDCACHEDIR),
+        "TUNNEL_DATA" : str( settings.TUNNEL_DATA),
+        "URL_ROOT" : str( settings.URL_ROOT)
+        }
+    except:
+        pathsdict["! EXCEPTION !"] = "missing or exta string constant in troggle/settings"
+
+    pathstype = OrderedDict()
+    try:
+        pathstype = {
+#       "BOGUS" : type(settings.BOGUS),
+        "JSLIB_URL" : type(settings.JSLIB_URL),
+#       "CSSLIB_URL" : type(settings.CSSLIB_URL),
+        "CAVEDESCRIPTIONS" : type(settings.CAVEDESCRIPTIONS),
+        "DIR_ROOT" : type(settings.DIR_ROOT),
+        "ENTRANCEDESCRIPTIONS" : type(settings.ENTRANCEDESCRIPTIONS),
+        "EXPOUSER_EMAIL" : type(settings.EXPOUSER_EMAIL),
+        "EXPOUSERPASS" : type(settings.EXPOUSERPASS),
+        "EXPOUSER" : type(settings.EXPOUSER),
+        "EXPOWEB" : type(settings.EXPOWEB),
+        "EXPOWEB_URL" : type(settings.EXPOWEB_URL),
+        "FILES" : type(settings.FILES),
+        "LOGFILE" : type(settings.LOGFILE),
+        "LOGIN_REDIRECT_URL" : type(settings.LOGIN_REDIRECT_URL),
+        "MEDIA_ROOT" : type(settings.MEDIA_ROOT),
+        "MEDIA_URL" : type(settings.MEDIA_URL),
+        "PHOTOS_URL" : type(settings.PHOTOS_URL),
+        "PYTHON_PATH" : type(settings.PYTHON_PATH),
+        "REPOS_ROOT_PATH" : type(settings.REPOS_ROOT_PATH),
+        "ROOT_URLCONF" : type(settings.ROOT_URLCONF),
+        "STATIC_URL" : type(settings.STATIC_URL),
+        "SURVEX_DATA" : type(settings.SURVEX_DATA),
+        "SURVEY_SCANS" : type(settings.SURVEY_SCANS),
+        "SURVEYS" : type(settings.SURVEYS),
+        "SURVEYS_URL" : type(settings.SURVEYS_URL),
+        "SURVEXPORT" : type(settings.SURVEXPORT),
+        "THREEDCACHEDIR" : type(settings.THREEDCACHEDIR),
+        "TUNNEL_DATA" : type(settings.TUNNEL_DATA),
+        "URL_ROOT" : type(settings.URL_ROOT)
+        }
+    except:
+        pathstype["! EXCEPTION !"] = "missing or exta string constant in troggle/settings"
+
+    # settings are unique by paths are not
+    ncodes = len(pathsdict)
+    bycodeslist = sorted(pathsdict.items()) # a list of tuples
+    bycodeslist2 = []
+
+    for k, p in bycodeslist:
+        bycodeslist2.append((k, p, str(pathstype[k])))
+    
+    bypaths = sorted(pathsdict.values()) # a list 
+    bypathslist = []
+    
+    for p in bypaths:
+        for k in pathsdict.keys():
+            if pathsdict[k] == p:
+                bypathslist.append((p, k, str(pathstype[k])))
+                del pathsdict[k]
+                break
+
+    return render(request, 'pathsreport.html', { 
+        "pathsdict":pathsdict, 
+        "bycodeslist":bycodeslist2, 
+        "bypathslist":bypathslist, 
+        "ncodes":ncodes})
+
+def stats(request):
+    statsDict={}
+    statsDict['expoCount'] = "{:,}".format(Expedition.objects.count())
+    statsDict['caveCount'] = "{:,}".format(Cave.objects.count())
+    statsDict['personCount'] = "{:,}".format(Person.objects.count())
+    statsDict['logbookEntryCount'] = "{:,}".format(LogbookEntry.objects.count())
+
+    legsbyexpo = [ ]
+    addupsurvexlength = 0
+    for expedition in Expedition.objects.all():
+        survexblocks = expedition.survexblock_set.all()
+        legsyear=0
+        survexleglength = 0.0
+        for survexblock in survexblocks:
+            survexleglength += survexblock.legslength
+            try:
+                legsyear += int(survexblock.legsall)
+            except:
+                pass
+        addupsurvexlength += survexleglength
+        legsbyexpo.append((expedition, {"nsurvexlegs": "{:,}".format(legsyear), 
+                "survexleglength":"{:,.0f}".format(survexleglength)}))
+    legsbyexpo.reverse() 
+
+    renderDict = {**statsDict, **{ "addupsurvexlength":addupsurvexlength/1000, "legsbyexpo":legsbyexpo }} # new syntax
+    return render(request,'statistics.html',  renderDict)

core/views/survex.py

@@ -0,0 +1,440 @@
+import re
+import os
+import datetime
+import difflib
+from pathlib import Path
+
+from django import forms
+from django.http import HttpResponseRedirect, HttpResponse, Http404
+from django.shortcuts import render
+from django.template.context_processors import csrf
+from django.views.decorators.csrf import ensure_csrf_cookie
+
+from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
+
+import troggle.settings as settings
+import parsers.survex
+from troggle.core.models import Expedition, Person, PersonExpedition 
+from troggle.core.models_survex import SurvexBlock, SurvexPersonRole, SurvexFile, SurvexDirectory
+from troggle.core.models_caves import Cave, PersonTrip, LogbookEntry
+from troggle.parsers.people import GetPersonExpeditionNameLookup
+
+survexdatasetpath = Path(settings.SURVEX_DATA)
+
+survextemplatefile = """; *** THIS IS A TEMPLATE FILE NOT WHAT YOU MIGHT BE EXPECTING ***
+
+*** DO NOT SAVE THIS FILE WITHOUT RENAMING IT !! ***
+;[Stuff in square brackets is example text to be replaced with real data, 
+; removing the square brackets]
+
+*begin [surveyname]
+
+; stations linked into other surveys (or likely to)
+*export [1 8 12 34]
+
+; Cave:
+; Area in cave/QM: 
+*title ""
+*date [2040.07.04]            ;  <-- CHANGE THIS DATE
+*team Insts [Fred Fossa]
+*team Notes [Brenda Badger]
+*team Pics [Luke Lynx]
+*team Tape [Albert Aadvark]
+*instrument [SAP #+Laser Tape/DistoX/Compass # ; Clino #]
+; Calibration: [Where, readings]
+*ref [2040#00]                ;  <-- CHANGE THIS TOO
+; the #number is on the clear pocket containing the original notes
+
+; if using a tape:
+*calibrate tape +0.0	; +ve if tape was too short, -ve if too long
+
+; Centreline data
+*data normal from to length bearing gradient ignoreall
+[ 1   2    5.57	    034.5     -12.8 ]
+
+;-----------
+;recorded station details (leave commented out)
+;(NP=Nail Polish, LHW/RHW=Left/Right Hand Wall)
+;Station Left Right Up Down Description
+;[Red] nail varnish markings
+[;1   0.8   0     5.3   1.6  ; NP on boulder. pt 23 on foo survey ]
+[;2   0.3   1.2   6     1.2  ; NP '2' LHW ]
+[;3   1.3   0     3.4   0.2  ; Rock on floor - not refindable ]
+
+
+;LRUDs arranged into passage tubes
+;new *data command for each 'passage',
+;repeat stations and adjust numbers as needed
+*data passage station left right up down 
+;[ 1   0.8   0     5.3   1.6 ]
+;[ 2   0.3   1.2   6     1.2 ]
+*data passage station left right up down
+;[ 1   1.3   1.5   5.3   1.6 ]
+;[ 3   2.4   0     3.4   0.2 ]
+
+
+;-----------
+;Question Mark List  ;(leave commented-out)
+; The nearest-station is the name of the survey and station which are nearest to
+; the QM. The resolution-station is either '-' to indicate that the QM hasn't
+; been checked; or the name of the survey and station which push that QM. If a
+; QM doesn't go anywhere, set the resolution-station to be the same as the
+; nearest-station. Include any relevant details of how to find or push the QM in
+; the textual description.
+;Serial number   grade(A/B/C/X)  nearest-station  resolution-station  description
+;[ QM1    A    surveyname.3    -    description of QM ]
+;[ QM2    B    surveyname.5    -    description of QM ]
+
+;------------
+;Cave description  ;(leave commented-out)
+;freeform text describing this section of the cave
+
+*end [surveyname]
+"""
+
+
+class SvxForm(forms.Form):
+    dirname = forms.CharField(widget=forms.TextInput(attrs={"readonly":True}))
+    filename = forms.CharField(widget=forms.TextInput(attrs={"readonly":True}))
+    datetime = forms.DateTimeField(widget=forms.TextInput(attrs={"readonly":True}))
+    outputtype = forms.CharField(widget=forms.TextInput(attrs={"readonly":True}))
+    code = forms.CharField(widget=forms.Textarea(attrs={"cols":150, "rows":36}))
+    
+    def GetDiscCode(self):
+        fname = survexdatasetpath / (self.data['filename'] + ".svx")
+        if not os.path.isfile(fname):
+            print(">>> >>> WARNING - svx file not found, showing TEMPLATE SVX",fname, flush=True)
+            return survextemplatefile
+        fin = open(fname, "rt",encoding='utf8',newline='')
+        svxtext = fin.read() 
+        fin.close()
+        return svxtext
+            
+    def DiffCode(self, rcode):
+        code = self.GetDiscCode()
+        difftext = difflib.unified_diff(code.splitlines(), rcode.splitlines())
+        difflist = [ diffline.strip()  for diffline in difftext  if not re.match(r"\s*$", diffline) ]
+        return difflist
+
+    def SaveCode(self, rcode):
+        fname = survexdatasetpath / (self.data['filename'] + ".svx")
+        if not os.path.isfile(fname):
+            if re.search(r"\[|\]", rcode):   
+                return "Error: remove all []s from the text. They are only template guidance."
+            mbeginend = re.search(r"(?s)\*begin\s+(\w+).*?\*end\s+(\w+)", rcode)
+            if not mbeginend:
+                return "Error: no begin/end block here"
+            if mbeginend.group(1) != mbeginend.group(2):
+                return "Error: mismatching begin/end labels"
+
+        # Make this create new survex folders if needed
+        try:
+            fout = open(fname, "wt", encoding='utf8',newline='\n')
+        except FileNotFoundError:
+            pth = os.path.dirname(self.data['filename'])
+            newpath = survexdatasetpath / pth
+            if not os.path.exists(newpath):
+                os.makedirs(newpath) 
+            fout = open(fname, "wt", encoding='utf8',newline='\n')
+
+        # javascript seems to insert CRLF on WSL1 whatever you say. So fix that:
+        res = fout.write(rcode.replace("\r",""))
+        fout.close()
+        return "SAVED ."
+
+    def Process(self):
+        print(">>>>....\n\n\n....Processing\n\n\n")
+        froox = os.fspath(survexdatasetpath / (self.data['filename'] + ".svx"))
+        froog = os.fspath(survexdatasetpath / (self.data['filename'] + ".log"))
+        cwd = os.getcwd()
+        os.chdir(os.path.split(froox)[0])
+        os.system(settings.CAVERN + " --log " + froox )
+        os.chdir(cwd)
+        fin = open(froog, "rt",encoding='utf8')
+        log = fin.read()
+        fin.close()
+        log = re.sub("(?s).*?(Survey contains)", "\\1", log)
+        return log
+
+
+@ensure_csrf_cookie
+def svx(request, survex_file):
+    '''Displays a singhle survex file in an textarea window (using a javascript online editor to enable 
+    editing) with buttons which allow SAVE, check for DIFFerences from saved, and RUN (which runs the
+    cavern executable and displays the output below the main textarea). Requires CSRF to be set upcorrect;ly, and requires permission to write to  the filesystem.
+    '''
+    # get the basic data from the file given in the URL
+    dirname = os.path.split(survex_file)[0]
+    dirname += "/"
+    nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    outputtype = "normal"
+    form = SvxForm({'filename':survex_file, 'dirname':dirname, 'datetime':nowtime, 'outputtype':outputtype}) 
+    
+    # if the form has been returned
+    difflist = [ ]
+    logmessage = ""
+    message = ""
+
+    if request.method == 'POST': # If the form has been submitted...
+        rform = SvxForm(request.POST) # 
+        if rform.is_valid(): # All validation rules pass (how do we check it against the filename and users?)
+            rcode = rform.cleaned_data['code']
+            outputtype = rform.cleaned_data['outputtype']
+            difflist = form.DiffCode(rcode)
+            #print(">>>> ", rform.data)
+            
+            if "revert" in rform.data:
+                pass
+            if "process" in rform.data:
+                if not difflist:
+                    message = "OUTPUT FROM PROCESSING"
+                    logmessage = form.Process()
+                    print(logmessage)
+                else:
+                    message = "SAVE FILE FIRST"
+                    form.data['code'] = rcode
+            if "save" in rform.data:
+                if request.user.is_authenticated():
+                    message = form.SaveCode(rcode)
+                else:
+                    message = "You do not have authority to save this file. Please log in."
+                if message != "SAVED":
+                    form.data['code'] = rcode
+            if "diff" in rform.data:
+                form.data['code'] = rcode
+    
+    #process(survex_file)
+    if 'code' not in form.data:    
+        form.data['code'] = form.GetDiscCode()
+    
+    if not difflist:
+        difflist.append("none")
+    if message:
+        difflist.insert(0, message)
+    
+    #print [ form.data['code'] ]
+    svxincludes = re.findall(r'\*include\s+(\S+)(?i)', form.data['code'] or "")
+    
+    vmap = {'settings': settings,
+            'has_3d': os.path.isfile(survexdatasetpath / survex_file / ".3d"),
+            'title': survex_file,
+            'svxincludes': svxincludes,
+            'difflist': difflist,
+            'logmessage':logmessage,
+            'form':form}
+    # vmap.update(csrf(request)) # this now refreshes to the wrong value, now that we user render(request, 
+    
+    if outputtype == "ajax":
+        return render(request, 'svxfiledifflistonly.html', vmap)
+        
+    return render(request, 'svxfile.html', vmap)
+
+def svxraw(request, survex_file):
+    '''Used for rendering .log files from survex outputtype'''
+    svx = open(os.path.join(survexdatasetpath /  survex_file / ".svx"), "rt",encoding='utf8')
+    return HttpResponse(svx, content_type="text")
+
+
+# The cavern running function
+def process(survex_file):
+    cwd = os.getcwd()
+    os.chdir(os.path.split(os.fspath(survexdatasetpath /  survex_file))[0])
+    os.system(settings.CAVERN + " --log " + survexdatasetpath /  survex_file / ".svx")
+    os.chdir(cwd)
+
+
+def threed(request, survex_file):
+    process(survex_file)
+    try:
+        threed = open(survexdatasetpath /  survex_file / ".3d", "rt",encoding='utf8')
+        return HttpResponse(threed, content_type="model/3d")
+    except:
+        log = open(survexdatasetpath /  survex_file / ".log", "rt",encoding='utf8')
+        return HttpResponse(log, content_type="text")
+
+
+def log(request, survex_file):
+    process(survex_file)
+    log = open(survexdatasetpath /  survex_file / ".log", "rt",encoding='utf8')
+    return HttpResponse(log, content_type="text")
+
+
+def err(request, survex_file):
+    process(survex_file)
+    err = open(survexdatasetpath /  survex_file / ".err", "rt",encoding='utf8')
+    return HttpResponse(err, content_type="text")
+
+
+def identifycavedircontents(gcavedir):
+    # find the primary survex file in each cave directory
+    # this should be in a configuration, not buried in the code...
+    name = os.path.split(gcavedir)[1]
+    subdirs = [ ]
+    subsvx = [ ]
+    primesvx = None
+    for f in os.listdir(gcavedir): # These may get outdated as data gets tidied up. This should not be in the code!
+        if name == "204" and (f in ["skel.svx", "template.svx", "204withents.svx"]):
+            pass
+        elif name == "136" and (f in ["136-noents.svx"]):
+            pass
+        elif name == "115" and (f in ["115cufix.svx", "115fix.svx"]):
+            pass
+        
+        elif os.path.isdir(os.path.join(gcavedir, f)):
+            if f[0] != ".":
+                subdirs.append(f)
+        elif f[-4:] == ".svx":
+            nf = f[:-4]
+            
+            if nf.lower() == name.lower() or nf[:3] == "all" or (name, nf) in [("resurvey2005", "145-2005"), ("cucc", "cu115")]:
+                if primesvx:
+                    if nf[:3] == "all":
+                        #assert primesvx[:3] != "all", (name, nf, primesvx, gcavedir, subsvx)
+                        primesvx = nf
+                    else:
+                        #assert primesvx[:3] == "all", (name, nf, primesvx, gcavedir, subsvx)
+                        pass
+                else:
+                    primesvx = nf
+            else:
+                subsvx.append(nf)
+        else:
+            pass
+            #assert re.match(".*?(?:.3d|.log|.err|.txt|.tmp|.diff|.e?spec|~)$", f), (gcavedir, f)
+    subsvx.sort()
+    #assert primesvx, (gcavedir, subsvx)
+    if primesvx:
+        subsvx.insert(0, primesvx)
+    return subdirs, subsvx
+
+ 
+def get_survexareapath(area):
+    return survexdatasetpath /  str("caves-" + area)
+
+# direct local non-database browsing through the svx file repositories
+# every time the page is viewed! Should cache this.
+def survexcaveslist(request):
+    '''This reads the entire list of caves in the Loser repo directory and produces a complete report.
+    It can find caves which have not yet been properly registered in the system by Databasereset.py because
+    someone may have uploaded the survex files without doing the rest of the integration process.
+    '''
+    # TO DO - filter out the non-public caves  from display UNLESS LOGGED INS
+    onefilecaves = [ ]
+    multifilecaves = [ ]
+    subdircaves = [ ]
+    fnumlist = [ ]
+    
+    for area in ["1623", "1626", "1624", "1627"]:
+        cavesdir = get_survexareapath(area)
+        arealist = sorted([ (area, -int(re.match(r"\d*", f).group(0) or "0"), f)  for f in os.listdir(cavesdir) ])
+        fnumlist += arealist
+    
+    #print(fnumlist)
+    
+    # go through the list and identify the contents of each cave directory
+    for area, num, cavedir in fnumlist:
+        
+        # these have sub dirs /cucc/ /arge/ /old/ but that is no reason to hide them in this webpage
+        # so these are now treated the same as 142 and 113 which also have a /cucc/ sub dir
+        #if cavedir in ["144", "40"]: 
+        #    continue
+
+        # This all assumes that the first .svx file has the same name as the cave name, 
+        # which usually but not always true. e.g. caves-1623/78/allkaese.svx not caves-1623/78/78.svx
+        # which is why we now also pass through the cavedir
+        cavesdir = get_survexareapath(area)
+        gcavedir = os.path.join(cavesdir, cavedir)
+        if os.path.isdir(gcavedir) and cavedir[0] != ".":
+            subdirs, subsvx = identifycavedircontents(gcavedir)
+            
+            caveid = check_cave_registered(area, cavedir) # should do this only once per database load or it will be slow
+            survdirobj = [ ]
+            for lsubsvx in subsvx:
+                survdirobj.append(("caves-" +area+ "/" +cavedir+"/"+lsubsvx, lsubsvx))
+            
+            # caves with subdirectories
+            if subdirs:
+                subsurvdirs = [ ]
+                for subdir in subdirs:
+                    dsubdirs, dsubsvx = identifycavedircontents(os.path.join(gcavedir, subdir))
+                    # assert not dsubdirs # handle case of empty sub directory
+                    lsurvdirobj = [ ]
+                    for lsubsvx in dsubsvx:
+                        lsurvdirobj.append(("caves-" +area+ "/" +cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))
+                    if len(dsubsvx) >= 1:
+                        subsurvdirs.append((subdir,lsurvdirobj[0], lsurvdirobj[0:])) # list now includes the first item too
+                subdircaves.append((cavedir, (survdirobj[0], survdirobj[1:]), subsurvdirs))
+            
+            # multifile caves
+            elif len(survdirobj) > 1:
+                multifilecaves.append((survdirobj[0], cavedir, survdirobj[1:]))
+            # single file caves
+            elif len(survdirobj) == 1:
+                onefilecaves.append(survdirobj[0])
+    
+    return render(request, 'svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves })
+
+def survexcavesingle(request, survex_cave):
+    '''parsing all the survex files of a single cave and showing that it's consistent and can find all 
+    the files and people.   Should explicity fix the kataster number thing.
+    kataster numbers are not unique across areas. Fix this.
+    
+    Should use getCave() from models_caves
+    '''
+    sc = survex_cave
+    try:
+        cave = Cave.objects.get(kataster_number=sc) # This may not be unique.
+        return render(request, 'svxcavesingle.html', {'settings': settings, "cave":cave })
+
+    except ObjectDoesNotExist:
+        # can get here if the survex file is in a directory labelled with unofficial number not kataster number.
+        # maybe - and _ mixed up, or CUCC-2017- instead of 2017-CUCC-, or CUCC2015DL01 . Let's not get carried away..
+        for unoff in [sc, sc.replace('-','_'), sc.replace('_','-'), sc.replace('-',''), sc.replace('_','')]:
+            try:
+                cave = Cave.objects.get(unofficial_number=unoff) # return on first one we find
+                return render(request, 'svxcavesingle.html', {'settings': settings, "cave":cave })
+            except ObjectDoesNotExist:
+                continue # next attempt in for loop
+        return render(request, 'svxcavesingle404.html', {'settings': settings, "cave":sc })
+
+    except MultipleObjectsReturned:
+        caves = Cave.objects.filter(kataster_number=survex_cave) 
+        return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves })
+
+    except:
+        return render(request, 'svxcavesingle404.html', {'settings': settings, "cave":sc })
+
+ 
+def check_cave_registered(area, survex_cave):
+    '''Checks whether a cave has been properly registered when it is found in the Loser repo
+    This should be called by Databasereset not here in a view
+    Currently Caves are only registered if they are listed in :expoweb: settings.CAVEDESCRIPTIONS
+    so we need to add in any more here.
+    
+    This function runs but does not seem to be used?!
+    '''
+    try:
+        cave = Cave.objects.get(kataster_number=survex_cave) 
+        return str(cave)
+
+    except MultipleObjectsReturned:
+        caves = Cave.objects.filter(kataster_number=survex_cave) 
+        for c in caves: 
+            if str(c) == area + "-" + survex_cave :
+                return str(c) # just get the first that matches
+        return None # many returned but none in correct area
+ 
+    except ObjectDoesNotExist:
+        pass
+
+    try:
+        cave = Cave.objects.get(unofficial_number=survex_cave) # should be unique!
+        if cave.kataster_number:
+            return str(cave)
+        else:
+            return None
+    except ObjectDoesNotExist:
+        pass
+        
+    return None
+        

core/views/surveys.py

@@ -0,0 +1,96 @@
+import os, stat
+import re
+from pathlib import Path
+from urllib.parse import urljoin, unquote as urlunquote
+from urllib.request import urlopen
+
+from django.conf import settings
+from django.shortcuts import render
+from django.http import HttpResponse, Http404
+
+from troggle.core.models_survex import ScansFolder, SingleScan, SurvexBlock, TunnelFile
+from troggle.core.views.expo import getmimetype
+import parsers.surveys
+
+'''Some of these views serve files as binary blobs, and simply set the mime type based on the file extension,
+as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked 
+by looking inside the file before being served.
+
+ need to check if inavlid query string is invalid, or produces multiple replies
+ and render a user-friendly error page.
+'''
+
+def surveyscansfolder(request, path):
+    #print [ s.walletname  for s in ScansFolder.objects.all() ]
+    scansfolder = ScansFolder.objects.get(walletname=urlunquote(path))  # need to check if inavlid query string and produce friendly error
+    return render(request, 'scansfolder.html', { 'scansfolder':scansfolder, 'settings': settings })
+
+def surveyscansingle(request, path, file):
+    '''sends a single binary file to the user,
+    '''
+    scansfolder = ScansFolder.objects.get(walletname=urlunquote(path)) # need to check if inavlid query string and produce friendly error
+    singlescan = SingleScan.objects.get(scansfolder=scansfolder, name=file)
+    # print(" - surveyscansingle {}:{}:{}:".format(path, file, getmimetype(file)))
+    return HttpResponse(content=open(singlescan.ffile,"rb"), content_type=getmimetype(file)) # any type of image
+
+
+def surveyscansfolders(request):
+    manyscansfolders = ScansFolder.objects.all()
+    return render(request, 'manyscansfolders.html', { 'manyscansfolders':manyscansfolders, 'settings': settings })
+    
+    
+def tunneldata(request):
+    tunnelfiles = TunnelFile.objects.all()
+    return render(request, 'tunnelfiles.html', { 'tunnelfiles':tunnelfiles, 'settings': settings })
+    
+
+def tunnelfilesingle(request, path):
+    '''sends a single binary file to the user, We should have a renderer that syntax-colours this Tunnel xml 
+    '''
+    tunnelfile = TunnelFile.objects.get(tunnelpath=urlunquote(path)) # need to check if inavlid query string and produce friendly error
+    tfile = Path(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
+    return HttpResponse(content=open(tfile), content_type="text/xhtml") # for display not download
+    
+def tunnelfileupload(request, path):
+    tunnelfile = TunnelFile.objects.get(tunnelpath=urlunquote(path)) # need to check if inavlid query string and produce friendly error
+    tfile = Path(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
+    
+    project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"]
+    print((project, user, tunnelversion))
+    
+    
+    if not (len(list(request.FILES.values())) == 1): # "only one file to upload"
+         return HttpResponse(content="Error: more than one file selected for upload", content_type="text/plain")
+       
+    uploadedfile = list(request.FILES.values())[0]
+        
+    if uploadedfile.field_name != "sketch":
+        return HttpResponse(content="Error: non-sketch file uploaded", content_type="text/plain")
+    if uploadedfile.content_type != "text/plain":
+        return HttpResponse(content="Error: non-plain content type", content_type="text/plain")
+        
+    # could use this to add new files
+    if os.path.split(path)[1] != uploadedfile.name:  
+        return HttpResponse(content="Error: name disagrees", content_type="text/plain")
+    
+    orgsize = tunnelfile.filesize   # = os.stat(tfile)[stat.ST_SIZE]
+        
+    ttext = uploadedfile.read()
+    
+    # could check that the user and projects agree here
+    
+    fout = open(tfile, "w")
+    fout.write(ttext)
+    fout.close()
+    
+    # redo its settings of 
+    parsers.surveys.SetTunnelfileInfo(tunnelfile)
+    tunnelfile.save()
+    
+    uploadedfile.close()
+    message = "File size %d overwritten with size %d" % (orgsize, tunnelfile.filesize)
+    return HttpResponse(content=message, content_type="text/plain")
+
+    
+    
+