From df3917a6771d2e2df3d35fde1b99a6f1d06f9589 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@klebos.com>
Date: Thu, 28 May 2020 02:20:50 +0100
Subject: [PATCH] Expunge cavetab
---
core/views_other.py | 29 ++--
parsers/QMs.py | 17 ++-
parsers/cavetab.py | 264 ------------------------------------
parsers/logbooks.py | 21 ++-
templates/controlPanel.html | 21 ---
urls.py | 1 -
6 files changed, 31 insertions(+), 322 deletions(-)
delete mode 100644 parsers/cavetab.py
diff --git a/core/views_other.py b/core/views_other.py
index 47a071a..4da8e70 100644
--- a/core/views_other.py
+++ b/core/views_other.py
@@ -1,17 +1,20 @@
-from troggle.core.models import Cave, Expedition, Person, LogbookEntry, PersonExpedition, PersonTrip, QM
-#from troggle.core.forms import UploadFileForm, DPhoto
-from django.conf import settings
-from django import forms
-from django.template import loader, Context
-from django.db.models import Q
-from django.shortcuts import render
-import databaseReset
import re
-from django.http import HttpResponse, HttpResponseRedirect
+
+from django import forms
+from django.conf import settings
from django.core.urlresolvers import reverse
+from django.db.models import Q
+from django.http import HttpResponse, HttpResponseRedirect
+from django.shortcuts import render
+from django.template import Context, loader
+
+import databaseReset
from troggle.core.models import *
+from troggle.core.models import (QM, Cave, Expedition, LogbookEntry, Person,
+ PersonExpedition, PersonTrip)
from troggle.helper import login_required_if_public
+
def showrequest(request):
return HttpResponse(request.GET)
@@ -71,18 +74,10 @@ def controlPanel(request):
return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
-def downloadCavetab(request):
- from export import tocavetab
- response = HttpResponse(content_type='text/csv')
- response['Content-Disposition'] = 'attachment; filename=CAVETAB2.CSV'
- tocavetab.writeCaveTab(response)
- return response
-
def downloadSurveys(request):
from export import tosurveys
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename=Surveys.csv'
- tosurveys.writeCaveTab(response)
return response
def downloadLogbook(request,year=None,extension=None,queryset=None):
diff --git a/parsers/QMs.py b/parsers/QMs.py
index a5ecd2f..578e422 100644
--- a/parsers/QMs.py
+++ b/parsers/QMs.py
@@ -1,11 +1,15 @@
# -*- coding: UTF-8 -*-
import csv
-from django.conf import settings
-from troggle.core.models import QM, LogbookEntry, Cave
+import os
+import re
from datetime import *
+
+from django.conf import settings
+
+from troggle.core.models import QM, Cave, LogbookEntry
from utils import save_carefully
-import re, os
+
def deleteQMs():
QM.objects.all().delete()
@@ -17,19 +21,19 @@ def parseCaveQMs(cave,inputFile):
try:
steinBr=Cave.objects.get(official_name="Steinbrückenhöhle")
except Cave.DoesNotExist:
- print("Steinbruckenhoehle is not in the database. Please run parsers.cavetab first.")
+ print("Steinbruckenhoehle is not in the database. Please run parsers.")
return
elif cave=='hauch':
try:
hauchHl=Cave.objects.get(official_name="Hauchhöhle")
except Cave.DoesNotExist:
- print("Hauchhoele is not in the database. Please run parsers.cavetab first.")
+ print("Hauchhoele is not in the database. Please run parsers.")
return
elif cave =='kh':
try:
kh=Cave.objects.get(official_name="Kaninchenhöhle")
except Cave.DoesNotExist:
- print("KH is not in the database. Please run parsers.cavetab first.")
+ print("KH is not in the database. Please run parsers.")
parse_KH_QMs(kh, inputFile=inputFile)
return
@@ -114,4 +118,3 @@ parseCaveQMs(cave='stein',inputFile=r"1623/204/qm.csv")
parseCaveQMs(cave='hauch',inputFile=r"1623/234/qm.csv")
parseCaveQMs(cave='kh', inputFile="1623/161/qmtodo.htm")
#parseCaveQMs(cave='balkonhoehle',inputFile=r"1623/264/qm.csv")
-
diff --git a/parsers/cavetab.py b/parsers/cavetab.py
deleted file mode 100644
index 8d257a7..0000000
--- a/parsers/cavetab.py
+++ /dev/null
@@ -1,264 +0,0 @@
-# -*- coding: utf-8 -*-
-import troggle.core.models as models
-from django.conf import settings
-import csv, time, re, os, logging
-from utils import save_carefully
-from django.core.urlresolvers import reverse
-import troggle.flatpages.models
-
-##format of CAVETAB2.CSV is
-KatasterNumber = 0
-KatStatusCode = 1
-Entrances = 2
-UnofficialNumber = 3
-MultipleEntrances = 4
-AutogenFile = 5
-LinkFile = 6
-LinkEntrance = 7
-Name = 8
-UnofficialName = 9
-Comment = 10
-Area = 11
-Explorers = 12
-UndergroundDescription = 13
-Equipment = 14
-QMList = 15
-KatasterStatus = 16
-References = 17
-UndergroundCentreLine = 18
-UndergroundDrawnSurvey = 19
-SurvexFile = 20
-Length = 21
-Depth = 22
-Extent = 23
-Notes = 24
-EntranceName = 25
-TagPoint = 26
-OtherPoint = 27
-DescriptionOfOtherPoint = 28
-ExactEntrance = 29
-TypeOfFix = 30
-GPSpreSA = 31
-GPSpostSA = 32
-Northing = 33
-Easting = 34
-Altitude = 35
-Bearings = 36
-Map = 37
-Location = 38
-Approach = 39
-EntranceDescription = 40
-PhotoOfLocation = 41
-Marking = 42
-MarkingComment = 43
-Findability = 44
-FindabilityComment = 45
-
-def LoadCaveTab():
-
- cavetab = open(os.path.join(settings.EXPOWEB, "noinfo", "CAVETAB2.CSV"),'rU')
- caveReader = csv.reader(cavetab)
- next(caveReader) # Strip out column headers
-
- logging.info("Beginning to import caves from "+str(cavetab)+"\n"+"-"*60+"\n")
-
- for katArea in ['1623', '1626']:
- if not models.Area.objects.filter(short_name = katArea):
- newArea = models.Area(short_name = katArea)
- newArea.save()
- logging.info("Added area "+str(newArea.short_name)+"\n")
- area1626 = models.Area.objects.filter(short_name = '1626')[0]
- area1623 = models.Area.objects.filter(short_name = '1623')[0]
-
- counter=0
- for line in caveReader :
- if line[Area] == 'nonexistent':
- continue
- entranceLetters=[] #Used in caves that have mulitlple entrances, which are not described on seperate lines
- if line[MultipleEntrances] == 'yes' or line[MultipleEntrances]=='': #When true, this line contains an actual cave, otherwise it is an extra entrance.
- args = {}
- defaultArgs = {}
-
- def addToArgs(CSVname, modelName):
- if line[CSVname]:
- args[modelName] = line[CSVname]
-
- def addToDefaultArgs(CSVname, modelName): #This has to do with the non-destructive import. These arguments will be passed as the "default" dictionary in a get_or_create
- if line[CSVname]:
- defaultArgs[modelName] = line[CSVname]
-
- # The attributes added using "addToArgs" will be used to look up an existing cave. Those added using "addToDefaultArgs" will not.
- addToArgs(KatasterNumber, "kataster_number")
- addToDefaultArgs(KatStatusCode, "kataster_code")
- addToArgs(UnofficialNumber, "unofficial_number")
- addToArgs(Name, "official_name")
- addToDefaultArgs(Comment, "notes")
- addToDefaultArgs(Explorers, "explorers")
- addToDefaultArgs(UndergroundDescription, "underground_description")
- addToDefaultArgs(Equipment, "equipment")
- addToDefaultArgs(KatasterStatus, "kataster_status")
- addToDefaultArgs(References, "references")
- addToDefaultArgs(UndergroundCentreLine, "underground_centre_line")
- addToDefaultArgs(UndergroundDrawnSurvey, "survey")
- addToDefaultArgs(Length, "length")
- addToDefaultArgs(Depth, "depth")
- addToDefaultArgs(Extent, "extent")
- addToDefaultArgs(SurvexFile, "survex_file")
- addToDefaultArgs(Notes, "notes")
- addToDefaultArgs(AutogenFile, "url")
- if line[Area] == "1626":
- if line[KatasterNumber] != "":
- slug = line[Area] + "-" + line[KatasterNumber]
- else:
- slug = line[Area] + "-" + line[UnofficialNumber]
- else:
- if line[KatasterNumber] != "":
- slug = "1623" + "-" + line[KatasterNumber]
- else:
- slug = "1623" + "-" + line[UnofficialNumber]
- args["filename"] = "%s.html" % slug
- #The following adds the legacy_file_path. This is always in either Autogen file or Link file
- for header in (AutogenFile,LinkFile):
- if line[header]:
- addToDefaultArgs(header,"description_file")
- break
-
- #The following checks if this cave is non-public i.e. we don't have rights to display it online.
- #Noinfo was the name of the old password protected directory, so if it has that then we will
- #set the non_public field of the model instance to true.
- defaultArgs["non_public"]=line[AutogenFile].startswith('noinfo') or line[LinkFile].startswith('noinfo')
- newCave, created = save_carefully(models.Cave, lookupAttribs=args, nonLookupAttribs=defaultArgs)
- cs = models.CaveSlug(slug = slug, cave = newCave, primary = True)
- cs.save()
- logging.info("Added cave "+str(newCave)+"\n")
-
- #If we created a new cave, add the area to it. This does mean that if a cave's identifying features have not changed, areas will not be updated from csv.
- if created and line[Area]:
- if line[Area] == "1626":
- newCave.area.add(area1626)
- else:
- area = models.Area.objects.filter(short_name = line[Area])
- if area:
- newArea = area[0]
- else:
- newArea = models.Area(short_name = line[Area], parent = area1623)
- newArea.save()
- newCave.area.add(newArea)
- newCave.area.add(area1623)
- elif created:
- newCave.area.add(area1623)
-
- newCave.save()
-
- logging.info("Added area "+line[Area]+" to cave "+str(newCave)+"\n")
- if created and line[UnofficialName]:
- newUnofficialName = models.OtherCaveName(cave = newCave, name = line[UnofficialName])
- newUnofficialName.save()
-
- logging.info("Added unofficial name "+str(newUnofficialName)+" to cave "+str(newCave)+"\n")
-
-
- if created and line[MultipleEntrances] == '' or \
- line[MultipleEntrances] == 'entrance' or \
- line[MultipleEntrances] == 'last entrance':
- args = {}
- def addToArgs(CSVname, modelName):
- if line[CSVname]:
- args[modelName] = line[CSVname]
- def addToArgsViaDict(CSVname, modelName, dictionary):
- if line[CSVname]:
- args[modelName] = dictionary[line[CSVname]]
-
- if line[Entrances]:
- entrance_letter = line[Entrances]
- else:
- entrance_letter = ''
-
- if not line[LinkFile]:
- addToArgs(Map, 'map_description')
- addToArgs(Location, 'location_description')
- addToArgs(Approach, 'approach')
- addToArgs(EntranceDescription, 'entrance_description')
- if line[MultipleEntrances] == 'entrance' or \
- line[MultipleEntrances] == 'last entrance':
- addToArgs(UndergroundDescription, 'underground_description')
- addToArgs(AutogenFile, 'url')
- addToArgs(EntranceName, 'name')
- addToArgs(Explorers, 'explorers')
- addToArgs(PhotoOfLocation, 'photo')
- addToArgsViaDict(Marking, 'marking', {"Paint": "P",
- "Paint (?)": "P?",
- "Tag": "T",
- "Tag (?)": "T?",
- "Retagged": "R",
- "Retag": "R",
- "Spit": "S",
- "Spit (?)": "S?",
- "Unmarked": "U",
- "": "?",
- })
-
- addToArgs(MarkingComment, 'marking_comment')
- addToArgsViaDict(Findability, 'findability', {"Surveyed": "S",
- "Lost": "L",
- "Refindable": "R",
- "": "?",
- "?": "?",
- })
- addToArgs(FindabilityComment, 'findability_description')
- addToArgs(Easting, 'easting')
- addToArgs(Northing, 'northing')
- addToArgs(Altitude, 'alt')
- addToArgs(DescriptionOfOtherPoint, 'other_description')
- addToArgs(TagPoint, 'tag_station')
- addToArgs(ExactEntrance, 'exact_station')
- addToArgs(OtherPoint, 'other_station')
- addToArgs(OtherPoint, 'other_description')
- if line[GPSpreSA]:
- addToArgs(GPSpreSA, 'other_station')
- args['other_description'] = 'pre selective availability GPS'
- if line[GPSpostSA]:
- addToArgs(GPSpostSA, 'other_station')
- args['other_description'] = 'post selective availability GPS'
- addToArgs(Bearings, 'bearings')
- args["filename"] = "%s.html" % (slug + entrance_letter)
-
- newEntrance = models.Entrance(**args)
- newEntrance.save()
- es = models.EntranceSlug(slug = slug + entrance_letter, entrance = newEntrance, primary = True)
- es.save()
-
-
- logging.info("Added entrance "+str(newEntrance)+"\n")
-
-
- newCaveAndEntrance = models.CaveAndEntrance(cave = newCave,
- entrance = newEntrance,
- entrance_letter = entrance_letter)
- newCaveAndEntrance.save()
-
- logging.info("Added CaveAndEntrance "+str(newCaveAndEntrance)+"\n")
- if line[AutogenFile] != "":
- f = flatpages.models.EntranceRedirect(originalURL = line[AutogenFile], entrance = newEntrance)
- f.save()
-
-
-# lookup function modelled on GetPersonExpeditionNameLookup
-Gcavelookup = None
-def GetCaveLookup():
- global Gcavelookup
- if Gcavelookup:
- return Gcavelookup
- Gcavelookup = {"NONEPLACEHOLDER":None}
- for cave in models.Cave.objects.all():
- Gcavelookup[cave.official_name.lower()] = cave
- if cave.kataster_number:
- Gcavelookup[cave.kataster_number] = cave
- if cave.unofficial_number:
- Gcavelookup[cave.unofficial_number] = cave
-
- Gcavelookup["tunnocks"] = Gcavelookup["258"]
- Gcavelookup["hauchhole"] = Gcavelookup["234"]
- return Gcavelookup
-
-
diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index ded90e4..e2f0ba0 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -1,21 +1,18 @@
#.-*- coding: utf-8 -*-
-from django.conf import settings
-import troggle.core.models as models
-
-from parsers.people import GetPersonExpeditionNameLookup
-from parsers.cavetab import GetCaveLookup
-
-from django.template.defaultfilters import slugify
-from django.utils.timezone import get_current_timezone
-from django.utils.timezone import make_aware
-
import csv
-import re
-import datetime, time
+import datetime
import os
import pickle
+import re
+import time
+from django.conf import settings
+from django.template.defaultfilters import slugify
+from django.utils.timezone import get_current_timezone, make_aware
+
+import troggle.core.models as models
+from parsers.people import GetPersonExpeditionNameLookup
from utils import save_carefully
#
diff --git a/templates/controlPanel.html b/templates/controlPanel.html
index 41c9409..7b35545 100644
--- a/templates/controlPanel.html
+++ b/templates/controlPanel.html
@@ -41,10 +41,6 @@
<td>people from folk.csv using parsers\people.py</td>
<td><input type="checkbox" name="import_people"/></td>
</tr>
- <tr>
- <td>caves from cavetab2.csv using parsers\cavetab.py</td>
- <td> <input type="checkbox" class="parser" name="import_cavetab"/></td>
- </tr>
<tr>
<td>logbook entries using parsers\logbooks.py</td>
<td><input type="checkbox" name="import_logbooks"/></td>
@@ -77,23 +73,6 @@
<td>Export onto server</td>
<td>Export and Download</td>
</th>
-<tr>
- <td>
- caves to cavetab2.csv
- </td>
- <td>
- <form name="export" method="post" action="">
- <p>Overwrite the existing CAVETAB2.CSV file with one generated by Troggle.</p>
- <input name="export_cavetab" type="submit" value="Update {{settings.EXPOWEB}}noinfo/CAVETAB2.CSV" />
- </form>
- </td>
- <td>
- <form name="export" method="get" action="{% url "downloadcavetab" %}">
- <p>Download a CAVETAB2.CSV file which is dynamically generated by Troggle.</p>
- <input name="download_cavetab" type="submit" value="Download CAVETAB2.CSV" />
- </form>
- </td>
-</tr>
<tr>
<td>
diff --git a/urls.py b/urls.py
index 36354cd..cf32a93 100644
--- a/urls.py
+++ b/urls.py
@@ -89,7 +89,6 @@ actualurlpatterns = patterns('',
# Is all this lot out of date ? Maybe the logbooks work?
url(r'^controlpanel/?$', views_other.controlPanel, name="controlpanel"),
- url(r'^CAVETAB2\.CSV/?$', views_other.downloadCavetab, name="downloadcavetab"),
url(r'^Surveys\.csv/?$', views_other.downloadSurveys, name="downloadsurveys"),
url(r'^logbook(?P<year>\d\d\d\d)\.(?P<extension>.*)/?$',views_other.downloadLogbook),
url(r'^logbook/?$',views_other.downloadLogbook, name="downloadlogbook"),