From df3917a6771d2e2df3d35fde1b99a6f1d06f9589 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Thu, 28 May 2020 02:20:50 +0100 Subject: [PATCH] Expunge cavetab --- core/views_other.py | 29 ++-- parsers/QMs.py | 17 ++- parsers/cavetab.py | 264 ------------------------------------ parsers/logbooks.py | 21 ++- templates/controlPanel.html | 21 --- urls.py | 1 - 6 files changed, 31 insertions(+), 322 deletions(-) delete mode 100644 parsers/cavetab.py diff --git a/core/views_other.py b/core/views_other.py index 47a071a..4da8e70 100644 --- a/core/views_other.py +++ b/core/views_other.py @@ -1,17 +1,20 @@ -from troggle.core.models import Cave, Expedition, Person, LogbookEntry, PersonExpedition, PersonTrip, QM -#from troggle.core.forms import UploadFileForm, DPhoto -from django.conf import settings -from django import forms -from django.template import loader, Context -from django.db.models import Q -from django.shortcuts import render -import databaseReset import re -from django.http import HttpResponse, HttpResponseRedirect + +from django import forms +from django.conf import settings from django.core.urlresolvers import reverse +from django.db.models import Q +from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import render +from django.template import Context, loader + +import databaseReset from troggle.core.models import * +from troggle.core.models import (QM, Cave, Expedition, LogbookEntry, Person, + PersonExpedition, PersonTrip) from troggle.helper import login_required_if_public + def showrequest(request): return HttpResponse(request.GET) @@ -71,18 +74,10 @@ def controlPanel(request): return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed}) -def downloadCavetab(request): - from export import tocavetab - response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename=CAVETAB2.CSV' - tocavetab.writeCaveTab(response) - return response - def downloadSurveys(request): from export import tosurveys response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename=Surveys.csv' - tosurveys.writeCaveTab(response) return response def downloadLogbook(request,year=None,extension=None,queryset=None): diff --git a/parsers/QMs.py b/parsers/QMs.py index a5ecd2f..578e422 100644 --- a/parsers/QMs.py +++ b/parsers/QMs.py @@ -1,11 +1,15 @@ # -*- coding: UTF-8 -*- import csv -from django.conf import settings -from troggle.core.models import QM, LogbookEntry, Cave +import os +import re from datetime import * + +from django.conf import settings + +from troggle.core.models import QM, Cave, LogbookEntry from utils import save_carefully -import re, os + def deleteQMs(): QM.objects.all().delete() @@ -17,19 +21,19 @@ def parseCaveQMs(cave,inputFile): try: steinBr=Cave.objects.get(official_name="Steinbrückenhöhle") except Cave.DoesNotExist: - print("Steinbruckenhoehle is not in the database. Please run parsers.cavetab first.") + print("Steinbruckenhoehle is not in the database. Please run parsers.") return elif cave=='hauch': try: hauchHl=Cave.objects.get(official_name="Hauchhöhle") except Cave.DoesNotExist: - print("Hauchhoele is not in the database. Please run parsers.cavetab first.") + print("Hauchhoele is not in the database. Please run parsers.") return elif cave =='kh': try: kh=Cave.objects.get(official_name="Kaninchenhöhle") except Cave.DoesNotExist: - print("KH is not in the database. Please run parsers.cavetab first.") + print("KH is not in the database. Please run parsers.") parse_KH_QMs(kh, inputFile=inputFile) return @@ -114,4 +118,3 @@ parseCaveQMs(cave='stein',inputFile=r"1623/204/qm.csv") parseCaveQMs(cave='hauch',inputFile=r"1623/234/qm.csv") parseCaveQMs(cave='kh', inputFile="1623/161/qmtodo.htm") #parseCaveQMs(cave='balkonhoehle',inputFile=r"1623/264/qm.csv") - diff --git a/parsers/cavetab.py b/parsers/cavetab.py deleted file mode 100644 index 8d257a7..0000000 --- a/parsers/cavetab.py +++ /dev/null @@ -1,264 +0,0 @@ -# -*- coding: utf-8 -*- -import troggle.core.models as models -from django.conf import settings -import csv, time, re, os, logging -from utils import save_carefully -from django.core.urlresolvers import reverse -import troggle.flatpages.models - -##format of CAVETAB2.CSV is -KatasterNumber = 0 -KatStatusCode = 1 -Entrances = 2 -UnofficialNumber = 3 -MultipleEntrances = 4 -AutogenFile = 5 -LinkFile = 6 -LinkEntrance = 7 -Name = 8 -UnofficialName = 9 -Comment = 10 -Area = 11 -Explorers = 12 -UndergroundDescription = 13 -Equipment = 14 -QMList = 15 -KatasterStatus = 16 -References = 17 -UndergroundCentreLine = 18 -UndergroundDrawnSurvey = 19 -SurvexFile = 20 -Length = 21 -Depth = 22 -Extent = 23 -Notes = 24 -EntranceName = 25 -TagPoint = 26 -OtherPoint = 27 -DescriptionOfOtherPoint = 28 -ExactEntrance = 29 -TypeOfFix = 30 -GPSpreSA = 31 -GPSpostSA = 32 -Northing = 33 -Easting = 34 -Altitude = 35 -Bearings = 36 -Map = 37 -Location = 38 -Approach = 39 -EntranceDescription = 40 -PhotoOfLocation = 41 -Marking = 42 -MarkingComment = 43 -Findability = 44 -FindabilityComment = 45 - -def LoadCaveTab(): - - cavetab = open(os.path.join(settings.EXPOWEB, "noinfo", "CAVETAB2.CSV"),'rU') - caveReader = csv.reader(cavetab) - next(caveReader) # Strip out column headers - - logging.info("Beginning to import caves from "+str(cavetab)+"\n"+"-"*60+"\n") - - for katArea in ['1623', '1626']: - if not models.Area.objects.filter(short_name = katArea): - newArea = models.Area(short_name = katArea) - newArea.save() - logging.info("Added area "+str(newArea.short_name)+"\n") - area1626 = models.Area.objects.filter(short_name = '1626')[0] - area1623 = models.Area.objects.filter(short_name = '1623')[0] - - counter=0 - for line in caveReader : - if line[Area] == 'nonexistent': - continue - entranceLetters=[] #Used in caves that have mulitlple entrances, which are not described on seperate lines - if line[MultipleEntrances] == 'yes' or line[MultipleEntrances]=='': #When true, this line contains an actual cave, otherwise it is an extra entrance. - args = {} - defaultArgs = {} - - def addToArgs(CSVname, modelName): - if line[CSVname]: - args[modelName] = line[CSVname] - - def addToDefaultArgs(CSVname, modelName): #This has to do with the non-destructive import. These arguments will be passed as the "default" dictionary in a get_or_create - if line[CSVname]: - defaultArgs[modelName] = line[CSVname] - - # The attributes added using "addToArgs" will be used to look up an existing cave. Those added using "addToDefaultArgs" will not. - addToArgs(KatasterNumber, "kataster_number") - addToDefaultArgs(KatStatusCode, "kataster_code") - addToArgs(UnofficialNumber, "unofficial_number") - addToArgs(Name, "official_name") - addToDefaultArgs(Comment, "notes") - addToDefaultArgs(Explorers, "explorers") - addToDefaultArgs(UndergroundDescription, "underground_description") - addToDefaultArgs(Equipment, "equipment") - addToDefaultArgs(KatasterStatus, "kataster_status") - addToDefaultArgs(References, "references") - addToDefaultArgs(UndergroundCentreLine, "underground_centre_line") - addToDefaultArgs(UndergroundDrawnSurvey, "survey") - addToDefaultArgs(Length, "length") - addToDefaultArgs(Depth, "depth") - addToDefaultArgs(Extent, "extent") - addToDefaultArgs(SurvexFile, "survex_file") - addToDefaultArgs(Notes, "notes") - addToDefaultArgs(AutogenFile, "url") - if line[Area] == "1626": - if line[KatasterNumber] != "": - slug = line[Area] + "-" + line[KatasterNumber] - else: - slug = line[Area] + "-" + line[UnofficialNumber] - else: - if line[KatasterNumber] != "": - slug = "1623" + "-" + line[KatasterNumber] - else: - slug = "1623" + "-" + line[UnofficialNumber] - args["filename"] = "%s.html" % slug - #The following adds the legacy_file_path. This is always in either Autogen file or Link file - for header in (AutogenFile,LinkFile): - if line[header]: - addToDefaultArgs(header,"description_file") - break - - #The following checks if this cave is non-public i.e. we don't have rights to display it online. - #Noinfo was the name of the old password protected directory, so if it has that then we will - #set the non_public field of the model instance to true. - defaultArgs["non_public"]=line[AutogenFile].startswith('noinfo') or line[LinkFile].startswith('noinfo') - newCave, created = save_carefully(models.Cave, lookupAttribs=args, nonLookupAttribs=defaultArgs) - cs = models.CaveSlug(slug = slug, cave = newCave, primary = True) - cs.save() - logging.info("Added cave "+str(newCave)+"\n") - - #If we created a new cave, add the area to it. This does mean that if a cave's identifying features have not changed, areas will not be updated from csv. - if created and line[Area]: - if line[Area] == "1626": - newCave.area.add(area1626) - else: - area = models.Area.objects.filter(short_name = line[Area]) - if area: - newArea = area[0] - else: - newArea = models.Area(short_name = line[Area], parent = area1623) - newArea.save() - newCave.area.add(newArea) - newCave.area.add(area1623) - elif created: - newCave.area.add(area1623) - - newCave.save() - - logging.info("Added area "+line[Area]+" to cave "+str(newCave)+"\n") - if created and line[UnofficialName]: - newUnofficialName = models.OtherCaveName(cave = newCave, name = line[UnofficialName]) - newUnofficialName.save() - - logging.info("Added unofficial name "+str(newUnofficialName)+" to cave "+str(newCave)+"\n") - - - if created and line[MultipleEntrances] == '' or \ - line[MultipleEntrances] == 'entrance' or \ - line[MultipleEntrances] == 'last entrance': - args = {} - def addToArgs(CSVname, modelName): - if line[CSVname]: - args[modelName] = line[CSVname] - def addToArgsViaDict(CSVname, modelName, dictionary): - if line[CSVname]: - args[modelName] = dictionary[line[CSVname]] - - if line[Entrances]: - entrance_letter = line[Entrances] - else: - entrance_letter = '' - - if not line[LinkFile]: - addToArgs(Map, 'map_description') - addToArgs(Location, 'location_description') - addToArgs(Approach, 'approach') - addToArgs(EntranceDescription, 'entrance_description') - if line[MultipleEntrances] == 'entrance' or \ - line[MultipleEntrances] == 'last entrance': - addToArgs(UndergroundDescription, 'underground_description') - addToArgs(AutogenFile, 'url') - addToArgs(EntranceName, 'name') - addToArgs(Explorers, 'explorers') - addToArgs(PhotoOfLocation, 'photo') - addToArgsViaDict(Marking, 'marking', {"Paint": "P", - "Paint (?)": "P?", - "Tag": "T", - "Tag (?)": "T?", - "Retagged": "R", - "Retag": "R", - "Spit": "S", - "Spit (?)": "S?", - "Unmarked": "U", - "": "?", - }) - - addToArgs(MarkingComment, 'marking_comment') - addToArgsViaDict(Findability, 'findability', {"Surveyed": "S", - "Lost": "L", - "Refindable": "R", - "": "?", - "?": "?", - }) - addToArgs(FindabilityComment, 'findability_description') - addToArgs(Easting, 'easting') - addToArgs(Northing, 'northing') - addToArgs(Altitude, 'alt') - addToArgs(DescriptionOfOtherPoint, 'other_description') - addToArgs(TagPoint, 'tag_station') - addToArgs(ExactEntrance, 'exact_station') - addToArgs(OtherPoint, 'other_station') - addToArgs(OtherPoint, 'other_description') - if line[GPSpreSA]: - addToArgs(GPSpreSA, 'other_station') - args['other_description'] = 'pre selective availability GPS' - if line[GPSpostSA]: - addToArgs(GPSpostSA, 'other_station') - args['other_description'] = 'post selective availability GPS' - addToArgs(Bearings, 'bearings') - args["filename"] = "%s.html" % (slug + entrance_letter) - - newEntrance = models.Entrance(**args) - newEntrance.save() - es = models.EntranceSlug(slug = slug + entrance_letter, entrance = newEntrance, primary = True) - es.save() - - - logging.info("Added entrance "+str(newEntrance)+"\n") - - - newCaveAndEntrance = models.CaveAndEntrance(cave = newCave, - entrance = newEntrance, - entrance_letter = entrance_letter) - newCaveAndEntrance.save() - - logging.info("Added CaveAndEntrance "+str(newCaveAndEntrance)+"\n") - if line[AutogenFile] != "": - f = flatpages.models.EntranceRedirect(originalURL = line[AutogenFile], entrance = newEntrance) - f.save() - - -# lookup function modelled on GetPersonExpeditionNameLookup -Gcavelookup = None -def GetCaveLookup(): - global Gcavelookup - if Gcavelookup: - return Gcavelookup - Gcavelookup = {"NONEPLACEHOLDER":None} - for cave in models.Cave.objects.all(): - Gcavelookup[cave.official_name.lower()] = cave - if cave.kataster_number: - Gcavelookup[cave.kataster_number] = cave - if cave.unofficial_number: - Gcavelookup[cave.unofficial_number] = cave - - Gcavelookup["tunnocks"] = Gcavelookup["258"] - Gcavelookup["hauchhole"] = Gcavelookup["234"] - return Gcavelookup - - diff --git a/parsers/logbooks.py b/parsers/logbooks.py index ded90e4..e2f0ba0 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -1,21 +1,18 @@ #.-*- coding: utf-8 -*- -from django.conf import settings -import troggle.core.models as models - -from parsers.people import GetPersonExpeditionNameLookup -from parsers.cavetab import GetCaveLookup - -from django.template.defaultfilters import slugify -from django.utils.timezone import get_current_timezone -from django.utils.timezone import make_aware - import csv -import re -import datetime, time +import datetime import os import pickle +import re +import time +from django.conf import settings +from django.template.defaultfilters import slugify +from django.utils.timezone import get_current_timezone, make_aware + +import troggle.core.models as models +from parsers.people import GetPersonExpeditionNameLookup from utils import save_carefully # diff --git a/templates/controlPanel.html b/templates/controlPanel.html index 41c9409..7b35545 100644 --- a/templates/controlPanel.html +++ b/templates/controlPanel.html @@ -41,10 +41,6 @@ people from folk.csv using parsers\people.py - - caves from cavetab2.csv using parsers\cavetab.py - - logbook entries using parsers\logbooks.py @@ -77,23 +73,6 @@ Export onto server Export and Download - - - caves to cavetab2.csv - - -
-

Overwrite the existing CAVETAB2.CSV file with one generated by Troggle.

- -
- - -
-

Download a CAVETAB2.CSV file which is dynamically generated by Troggle.

- -
- - diff --git a/urls.py b/urls.py index 36354cd..cf32a93 100644 --- a/urls.py +++ b/urls.py @@ -89,7 +89,6 @@ actualurlpatterns = patterns('', # Is all this lot out of date ? Maybe the logbooks work? url(r'^controlpanel/?$', views_other.controlPanel, name="controlpanel"), - url(r'^CAVETAB2\.CSV/?$', views_other.downloadCavetab, name="downloadcavetab"), url(r'^Surveys\.csv/?$', views_other.downloadSurveys, name="downloadsurveys"), url(r'^logbook(?P\d\d\d\d)\.(?P.*)/?$',views_other.downloadLogbook), url(r'^logbook/?$',views_other.downloadLogbook, name="downloadlogbook"),