diff --git a/core/admin.py b/core/admin.py index 0fe47f7..3bc14d4 100644 --- a/core/admin.py +++ b/core/admin.py @@ -1,10 +1,12 @@ -from troggle.core.models import * +import django.forms as forms from django.contrib import admin from django.forms import ModelForm -import django.forms as forms from django.http import HttpResponse from django.core import serializers + from troggle.core.views_other import downloadLogbook +from troggle.core.models import * +from troggle.core.models_caves import * #from troggle.reversion.admin import VersionAdmin #django-reversion version control diff --git a/core/forms.py b/core/forms.py index b8d2e07..e8e620c 100644 --- a/core/forms.py +++ b/core/forms.py @@ -1,12 +1,16 @@ -from django.forms import ModelForm -from .models import Cave, Person, PersonExpedition, LogbookEntry, QM, Expedition, Entrance, CaveAndEntrance -import django.forms as forms -from django.forms.models import modelformset_factory -from django.contrib.admin.widgets import AdminDateWidget import string from datetime import date + +import django.forms as forms +from django.forms import ModelForm +from django.forms.models import modelformset_factory +from django.contrib.admin.widgets import AdminDateWidget + from tinymce.widgets import TinyMCE +from troggle.core.models import Person, PersonExpedition, LogbookEntry, Expedition +from troggle.core.models_caves import Cave, QM, Entrance, CaveAndEntrance + class CaveForm(ModelForm): underground_description = forms.CharField(required = False, widget=forms.Textarea()) explorers = forms.CharField(required = False, widget=forms.Textarea()) diff --git a/core/management/commands/reset_db.py b/core/management/commands/reset_db.py index 2aea94a..03d143e 100644 --- a/core/management/commands/reset_db.py +++ b/core/management/commands/reset_db.py @@ -7,7 +7,7 @@ from django.core.urlresolvers import reverse from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import User -from troggle.core.models import Cave, Entrance +from troggle.core.models_caves import Cave, Entrance import troggle.flatpages.models import settings diff --git a/core/models.py b/core/models.py index 42219e7..142cfa6 100644 --- a/core/models.py +++ b/core/models.py @@ -3,29 +3,25 @@ import os import datetime import logging import re -import subprocess +from subprocess import call -from urllib.request import * -from urllib.parse import * -from urllib.error import * +from urllib.parse import urljoin from decimal import Decimal, getcontext getcontext().prec=2 #use 2 significant figures for decimal calculations import settings -from django.forms import ModelForm from django.db import models from django.contrib import admin -from django.core.files.storage import FileSystemStorage from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType -from django.db.models import Min, Max +#from django.db.models import Min, Max from django.conf import settings from django.core.urlresolvers import reverse from django.template import Context, loader -from troggle.core.models_survex import * - +import troggle.core.models_survex +import troggle.core.models_caves as models_caves def get_related_by_wikilinks(wiki_text): found=re.findall(settings.QM_PATTERN,wiki_text) @@ -33,7 +29,7 @@ def get_related_by_wikilinks(wiki_text): for wikilink in found: qmdict={'urlroot':settings.URL_ROOT,'cave':wikilink[2],'year':wikilink[1],'number':wikilink[3]} try: - cave_slugs = CaveSlug.objects.filter(cave__kataster_number = qmdict['cave']) + cave_slugs = models_caves.CaveSlug.objects.filter(cave__kataster_number = qmdict['cave']) qm=QM.objects.get(found_by__cave_slug__in = cave_slugs, found_by__date__year = qmdict['year'], number = qmdict['number']) @@ -73,10 +69,20 @@ class TroggleImageModel(models.Model): def get_admin_url(self): return urllib.parse.urljoin(settings.URL_ROOT, "/admin/core/" + self.object_name().lower() + "/" + str(self.pk)) - class Meta: abstract = True +class DataIssue(TroggleModel): + date = models.DateTimeField(auto_now_add=True, blank=True) + parser = models.CharField(max_length=50, blank=True, null=True) + message = models.CharField(max_length=400, blank=True, null=True) + + class Meta: + ordering = ['date'] + + def __str__(self): + return "%s - %s" % (self.parser, self.message) + # # single Expedition, usually seen by year # @@ -231,6 +237,7 @@ class PersonExpedition(TroggleModel): res = self.persontrip_set.all().aggregate(day_max=Max("expeditionday__date")) return res["day_max"] + class LogbookEntry(TroggleModel): """Single parsed entry from Logbook """ @@ -258,7 +265,7 @@ class LogbookEntry(TroggleModel): def __getattribute__(self, item): if item == "cave": #Allow a logbookentries cave to be directly accessed despite not having a proper foreignkey - return CaveSlug.objects.get(slug = self.cave_slug).cave + return models_caves.CaveSlug.objects.get(slug = self.cave_slug).cave # parse error in python3.8 # https://stackoverflow.com/questions/41343263/provide-classcell-example-for-python-3-6-metaclass #https://github.com/django/django/pull/7653 @@ -269,7 +276,7 @@ class LogbookEntry(TroggleModel): def __init__(self, *args, **kwargs): if "cave" in list(kwargs.keys()): if kwargs["cave"] is not None: - kwargs["cave_slug"] = CaveSlug.objects.get(cave=kwargs["cave"], primary=True).slug + kwargs["cave_slug"] = models_caves.CaveSlug.objects.get(cave=kwargs["cave"], primary=True).slug kwargs.pop("cave") # parse error in python3.8 return TroggleModel.__init__(self, *args, **kwargs) # seems OK in 3.5 & 3.8! failure later elsewhere with 3.8 @@ -309,6 +316,7 @@ class LogbookEntry(TroggleModel): def DayIndex(self): return list(self.expeditionday.logbookentry_set.all()).index(self) + # # Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry) # @@ -346,507 +354,3 @@ class PersonTrip(TroggleModel): def __str__(self): return "%s (%s)" % (self.personexpedition, self.logbook_entry.date) - -########################################## -# move following classes into models_cave -########################################## - -class Area(TroggleModel): - short_name = models.CharField(max_length=100) - name = models.CharField(max_length=200, blank=True, null=True) - description = models.TextField(blank=True,null=True) - parent = models.ForeignKey('Area', blank=True, null=True) - def __str__(self): - if self.parent: - return str(self.parent) + " - " + str(self.short_name) - else: - return str(self.short_name) - def kat_area(self): - if self.short_name in ["1623", "1626"]: - return self.short_name - elif self.parent: - return self.parent.kat_area() - -class CaveAndEntrance(models.Model): - cave = models.ForeignKey('Cave') - entrance = models.ForeignKey('Entrance') - entrance_letter = models.CharField(max_length=20,blank=True,null=True) - def __str__(self): - return str(self.cave) + str(self.entrance_letter) - -class CaveSlug(models.Model): - cave = models.ForeignKey('Cave') - slug = models.SlugField(max_length=50, unique = True) - primary = models.BooleanField(default=False) - -class Cave(TroggleModel): - # too much here perhaps, - official_name = models.CharField(max_length=160) - area = models.ManyToManyField(Area, blank=True, null=True) - kataster_code = models.CharField(max_length=20,blank=True,null=True) - kataster_number = models.CharField(max_length=10,blank=True, null=True) - unofficial_number = models.CharField(max_length=60,blank=True, null=True) - entrances = models.ManyToManyField('Entrance', through='CaveAndEntrance') - explorers = models.TextField(blank=True,null=True) - underground_description = models.TextField(blank=True,null=True) - equipment = models.TextField(blank=True,null=True) - references = models.TextField(blank=True,null=True) - survey = models.TextField(blank=True,null=True) - kataster_status = models.TextField(blank=True,null=True) - underground_centre_line = models.TextField(blank=True,null=True) - notes = models.TextField(blank=True,null=True) - length = models.CharField(max_length=100,blank=True,null=True) - depth = models.CharField(max_length=100,blank=True,null=True) - extent = models.CharField(max_length=100,blank=True,null=True) - survex_file = models.CharField(max_length=100,blank=True,null=True) - description_file = models.CharField(max_length=200,blank=True,null=True) - url = models.CharField(max_length=200,blank=True,null=True) - filename = models.CharField(max_length=200) - - #class Meta: - # unique_together = (("area", "kataster_number"), ("area", "unofficial_number")) - # FIXME Kataster Areas and CUCC defined sub areas need seperating - - #href = models.CharField(max_length=100) - - class Meta: - ordering = ('kataster_code', 'unofficial_number') - - def hassurvey(self): - if not self.underground_centre_line: - return "No" - if (self.survey.find(" -1 or self.survey.find(" -1 or self.survey.find(" -1 or self.survey.find(" -1): - return "Yes" - return "Missing" - - def hassurveydata(self): - if not self.underground_centre_line: - return "No" - if self.survex_file: - return "Yes" - return "Missing" - - def slug(self): - primarySlugs = self.caveslug_set.filter(primary = True) - if primarySlugs: - return primarySlugs[0].slug - else: - slugs = self.caveslug_set.filter() - if slugs: - return slugs[0].slug - - def ours(self): - return bool(re.search(r'CUCC', self.explorers)) - - def reference(self): - if self.kataster_number: - return "%s-%s" % (self.kat_area(), self.kataster_number) - else: - return "%s-%s" % (self.kat_area(), self.unofficial_number) - - def get_absolute_url(self): - if self.kataster_number: - href = self.kataster_number - elif self.unofficial_number: - href = self.unofficial_number - else: - href = self.official_name.lower() - #return settings.URL_ROOT + '/cave/' + href + '/' - return urllib.parse.urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) - - def __str__(self, sep = ": "): - return str("slug:"+self.slug()) - - def get_QMs(self): - return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all()) - - def new_QM_number(self, year=datetime.date.today().year): - """Given a cave and the current year, returns the next QM number.""" - try: - res=QM.objects.filter(found_by__date__year=year, found_by__cave=self).order_by('-number')[0] - except IndexError: - return 1 - return res.number+1 - - def kat_area(self): - for a in self.area.all(): - if a.kat_area(): - return a.kat_area() - - def entrances(self): - return CaveAndEntrance.objects.filter(cave=self) - - def singleentrance(self): - return len(CaveAndEntrance.objects.filter(cave=self)) == 1 - - def entrancelist(self): - rs = [] - res = "" - for e in CaveAndEntrance.objects.filter(cave=self): - rs.append(e.entrance_letter) - rs.sort() - prevR = None - n = 0 - for r in rs: - if prevR: - if chr(ord(prevR) + 1 ) == r: - prevR = r - n += 1 - else: - if n == 0: - res += ", " + prevR - else: - res += "–" + prevR - else: - prevR = r - n = 0 - res += r - if n == 0: - res += ", " + prevR - else: - res += "–" + prevR - return res - - def writeDataFile(self): - try: - f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "wb") - except: - subprocess.call(settings.FIX_PERMISSIONS) - f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "wb") - t = loader.get_template('dataformat/cave.xml') - c = Context({'cave': self}) - u = t.render(c) - u8 = u.encode("utf-8") - f.write(u8) - f.close() - - def getArea(self): - areas = self.area.all() - lowestareas = list(areas) - for area in areas: - if area.parent in areas: - try: - lowestareas.remove(area.parent) - except: - pass - return lowestareas[0] - -def getCaveByReference(reference): - areaname, code = reference.split("-", 1) - #print(areaname, code) - area = Area.objects.get(short_name = areaname) - #print(area) - foundCaves = list(Cave.objects.filter(area = area, kataster_number = code).all()) + list(Cave.objects.filter(area = area, unofficial_number = code).all()) - print((list(foundCaves))) - if len(foundCaves) == 1: - return foundCaves[0] - else: - return False - -class OtherCaveName(TroggleModel): - name = models.CharField(max_length=160) - cave = models.ForeignKey(Cave) - def __str__(self): - return str(self.name) - -class EntranceSlug(models.Model): - entrance = models.ForeignKey('Entrance') - slug = models.SlugField(max_length=50, unique = True) - primary = models.BooleanField(default=False) - -class Entrance(TroggleModel): - name = models.CharField(max_length=100, blank=True,null=True) - entrance_description = models.TextField(blank=True,null=True) - explorers = models.TextField(blank=True,null=True) - map_description = models.TextField(blank=True,null=True) - location_description = models.TextField(blank=True,null=True) - approach = models.TextField(blank=True,null=True) - underground_description = models.TextField(blank=True,null=True) - photo = models.TextField(blank=True,null=True) - MARKING_CHOICES = ( - ('P', 'Paint'), - ('P?', 'Paint (?)'), - ('T', 'Tag'), - ('T?', 'Tag (?)'), - ('R', 'Needs Retag'), - ('S', 'Spit'), - ('S?', 'Spit (?)'), - ('U', 'Unmarked'), - ('?', 'Unknown')) - marking = models.CharField(max_length=2, choices=MARKING_CHOICES) - marking_comment = models.TextField(blank=True,null=True) - FINDABLE_CHOICES = ( - ('?', 'To be confirmed ...'), - ('S', 'Coordinates'), - ('L', 'Lost'), - ('R', 'Refindable')) - findability = models.CharField(max_length=1, choices=FINDABLE_CHOICES, blank=True, null=True) - findability_description = models.TextField(blank=True,null=True) - alt = models.TextField(blank=True, null=True) - northing = models.TextField(blank=True, null=True) - easting = models.TextField(blank=True, null=True) - tag_station = models.TextField(blank=True, null=True) - exact_station = models.TextField(blank=True, null=True) - other_station = models.TextField(blank=True, null=True) - other_description = models.TextField(blank=True,null=True) - bearings = models.TextField(blank=True,null=True) - url = models.CharField(max_length=200,blank=True,null=True) - filename = models.CharField(max_length=200) - cached_primary_slug = models.CharField(max_length=200,blank=True,null=True) - - def __str__(self): - return str(self.slug()) - - def exact_location(self): - return SurvexStation.objects.lookup(self.exact_station) - def other_location(self): - return SurvexStation.objects.lookup(self.other_station) - - - def find_location(self): - r = {'': 'To be entered ', - '?': 'To be confirmed:', - 'S': '', - 'L': 'Lost:', - 'R': 'Refindable:'}[self.findability] - if self.tag_station: - try: - s = SurvexStation.objects.lookup(self.tag_station) - return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z) - except: - return r + "%s Tag Station not in dataset" % self.tag_station - if self.exact_station: - try: - s = SurvexStation.objects.lookup(self.exact_station) - return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z) - except: - return r + "%s Exact Station not in dataset" % self.tag_station - if self.other_station: - try: - s = SurvexStation.objects.lookup(self.other_station) - return r + "%0.0fE %0.0fN %0.0fAlt %s" % (s.x, s.y, s.z, self.other_description) - except: - return r + "%s Other Station not in dataset" % self.tag_station - if self.FINDABLE_CHOICES == "S": - r += "ERROR, Entrance has been surveyed but has no survex point" - if self.bearings: - return r + self.bearings - return r - - def best_station(self): - if self.tag_station: - return self.tag_station - if self.exact_station: - return self.exact_station - if self.other_station: - return self.other_station - - def has_photo(self): - if self.photo: - if (self.photo.find(" -1 or self.photo.find(" -1 or self.photo.find(" -1 or self.photo.find(" -1): - return "Yes" - else: - return "Missing" - else: - return "No" - - def marking_val(self): - for m in self.MARKING_CHOICES: - if m[0] == self.marking: - return m[1] - def findability_val(self): - for f in self.FINDABLE_CHOICES: - if f[0] == self.findability: - return f[1] - - def tag(self): - return SurvexStation.objects.lookup(self.tag_station) - - def needs_surface_work(self): - return self.findability != "S" or not self.has_photo or self.marking != "T" - - def get_absolute_url(self): - - ancestor_titles='/'.join([subcave.title for subcave in self.get_ancestors()]) - if ancestor_titles: - res = '/'.join((self.get_root().cave.get_absolute_url(), ancestor_titles, self.title)) - - else: - res = '/'.join((self.get_root().cave.get_absolute_url(), self.title)) - - return res - - def slug(self): - if not self.cached_primary_slug: - primarySlugs = self.entranceslug_set.filter(primary = True) - if primarySlugs: - self.cached_primary_slug = primarySlugs[0].slug - self.save() - else: - slugs = self.entranceslug_set.filter() - if slugs: - self.cached_primary_slug = slugs[0].slug - self.save() - return self.cached_primary_slug - - def writeDataFile(self): - try: - f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w") - except: - subprocess.call(settings.FIX_PERMISSIONS) - f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w") - t = loader.get_template('dataformat/entrance.xml') - c = Context({'entrance': self}) - u = t.render(c) - u8 = u.encode("utf-8") - f.write(u8) - f.close() - -class CaveDescription(TroggleModel): - short_name = models.CharField(max_length=50, unique = True) - long_name = models.CharField(max_length=200, blank=True, null=True) - description = models.TextField(blank=True,null=True) - linked_subcaves = models.ManyToManyField("NewSubCave", blank=True,null=True) - linked_entrances = models.ManyToManyField("Entrance", blank=True,null=True) - linked_qms = models.ManyToManyField("QM", blank=True,null=True) - - def __str__(self): - if self.long_name: - return str(self.long_name) - else: - return str(self.short_name) - - def get_absolute_url(self): - return urllib.parse.urljoin(settings.URL_ROOT, reverse('cavedescription', args=(self.short_name,))) - - def save(self): - """ - Overridden save method which stores wikilinks in text as links in database. - """ - TroggleModel.save() - #super(CaveDescription, self).save() # fails in python 3.8, OK in python 3.5 - qm_list=get_related_by_wikilinks(self.description) - for qm in qm_list: - self.linked_qms.add(qm) - TroggleModel.save() - #super(CaveDescription, self).save() # fails in python 3.8, OK in python 3.5 - -class NewSubCave(TroggleModel): - name = models.CharField(max_length=200, unique = True) - def __str__(self): - return str(self.name) - -class QM(TroggleModel): - #based on qm.csv in trunk/expoweb/1623/204 which has the fields: - #"Number","Grade","Area","Description","Page reference","Nearest station","Completion description","Comment" - found_by = models.ForeignKey(LogbookEntry, related_name='QMs_found',blank=True, null=True ) - ticked_off_by = models.ForeignKey(LogbookEntry, related_name='QMs_ticked_off',null=True,blank=True) - #cave = models.ForeignKey(Cave) - #expedition = models.ForeignKey(Expedition) - - number = models.IntegerField(help_text="this is the sequential number in the year", ) - GRADE_CHOICES=( - ('A', 'A: Large obvious lead'), - ('B', 'B: Average lead'), - ('C', 'C: Tight unpromising lead'), - ('D', 'D: Dig'), - ('X', 'X: Unclimbable aven') - ) - grade = models.CharField(max_length=1, choices=GRADE_CHOICES) - location_description = models.TextField(blank=True) - nearest_station_description = models.CharField(max_length=400,null=True,blank=True) - nearest_station_name = models.CharField(max_length=200,blank=True,null=True) - nearest_station = models.ForeignKey(SurvexStation,null=True,blank=True) - area = models.CharField(max_length=100,blank=True,null=True) - completion_description = models.TextField(blank=True,null=True) - comment=models.TextField(blank=True,null=True) - - def __str__(self): - return "%s %s" % (self.code(), self.grade) - - def code(self): - return "%s-%s-%s" % (str(self.found_by.cave)[6:], self.found_by.date.year, self.number) - - def get_absolute_url(self): - #return settings.URL_ROOT + '/cave/' + self.found_by.cave.kataster_number + '/' + str(self.found_by.date.year) + '-' + '%02d' %self.number - return urllib.parse.urljoin(settings.URL_ROOT, reverse('qm',kwargs={'cave_id':self.found_by.cave.kataster_number,'year':self.found_by.date.year,'qm_id':self.number,'grade':self.grade})) - - def get_next_by_id(self): - return QM.objects.get(id=self.id+1) - - def get_previous_by_id(self): - return QM.objects.get(id=self.id-1) - - def wiki_link(self): - return "%s%s%s" % ('[[QM:',self.code(),']]') - -scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL) -def get_scan_path(instance, filename): - year=instance.survey.expedition.year - #print("WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number, instance.survey.wallet_letter) - number=str(instance.survey.wallet_number) - if str(instance.survey.wallet_letter) != "None": - number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01 - return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg') - -class ScannedImage(TroggleImageModel): - file = models.ImageField(storage=scansFileStorage, upload_to=get_scan_path) - scanned_by = models.ForeignKey(Person,blank=True, null=True) - scanned_on = models.DateField(null=True) - survey = models.ForeignKey('Survey') - contents = models.CharField(max_length=20,choices=(('notes','notes'),('plan','plan_sketch'),('elevation','elevation_sketch'))) - number_in_wallet = models.IntegerField(null=True) - lon_utm = models.FloatField(blank=True,null=True) - lat_utm = models.FloatField(blank=True,null=True) - - #content_type = models.ForeignKey(ContentType) - #object_id = models.PositiveIntegerField() - #location = generic.GenericForeignKey('content_type', 'object_id') - - #This is an ugly hack to deal with the #s in our survey scan paths. The correct thing is to write a custom file storage backend which calls urlencode on the name for making file.url but not file.path. - def correctURL(self): - return string.replace(self.file.url,r'#',r'%23') - - def __str__(self): - return get_scan_path(self,'') - -class Survey(TroggleModel): - expedition = models.ForeignKey('Expedition') #REDUNDANT (logbook_entry) - wallet_number = models.IntegerField(blank=True,null=True) - wallet_letter = models.CharField(max_length=1,blank=True,null=True) - comments = models.TextField(blank=True,null=True) - location = models.CharField(max_length=400,blank=True,null=True) #REDUNDANT - subcave = models.ForeignKey('NewSubCave', blank=True, null=True) - #notes_scan = models.ForeignKey('ScannedImage',related_name='notes_scan',blank=True, null=True) #Replaced by contents field of ScannedImage model - survex_block = models.OneToOneField('SurvexBlock',blank=True, null=True) - logbook_entry = models.ForeignKey('LogbookEntry') - centreline_printed_on = models.DateField(blank=True, null=True) - centreline_printed_by = models.ForeignKey('Person',related_name='centreline_printed_by',blank=True,null=True) - #sketch_scan = models.ForeignKey(ScannedImage,blank=True, null=True) #Replaced by contents field of ScannedImage model - tunnel_file = models.FileField(upload_to='surveyXMLfiles',blank=True, null=True) - tunnel_main_sketch = models.ForeignKey('Survey',blank=True,null=True) - integrated_into_main_sketch_on = models.DateField(blank=True,null=True) - integrated_into_main_sketch_by = models.ForeignKey('Person' ,related_name='integrated_into_main_sketch_by', blank=True,null=True) - rendered_image = models.ImageField(upload_to='renderedSurveys',blank=True,null=True) - def __str__(self): - return self.expedition.year+"#"+"%02d" % int(self.wallet_number) - - def notes(self): - return self.scannedimage_set.filter(contents='notes') - - def plans(self): - return self.scannedimage_set.filter(contents='plan') - - def elevations(self): - return self.scannedimage_set.filter(contents='elevation') - -class DataIssue(TroggleModel): - date = models.DateTimeField(auto_now_add=True, blank=True) - parser = models.CharField(max_length=50, blank=True, null=True) - message = models.CharField(max_length=400, blank=True, null=True) - - class Meta: - ordering = ['date'] - - def __str__(self): - return "%s - %s" % (self.parser, self.message) \ No newline at end of file diff --git a/core/models_caves.py b/core/models_caves.py new file mode 100644 index 0000000..4246f6d --- /dev/null +++ b/core/models_caves.py @@ -0,0 +1,513 @@ +import string +import os +import datetime +import logging +import re +from subprocess import call + +from urllib.parse import urljoin + +import settings + +from django.db import models +from django.core.files.storage import FileSystemStorage +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType +from django.db.models import Min, Max +from django.conf import settings +from django.core.urlresolvers import reverse +from django.template import Context, loader + +from troggle.core.models import * +from troggle.core.models_survex import * + +class Area(TroggleModel): + short_name = models.CharField(max_length=100) + name = models.CharField(max_length=200, blank=True, null=True) + description = models.TextField(blank=True,null=True) + parent = models.ForeignKey('Area', blank=True, null=True) + def __str__(self): + if self.parent: + return str(self.parent) + " - " + str(self.short_name) + else: + return str(self.short_name) + def kat_area(self): + if self.short_name in ["1623", "1626"]: + return self.short_name + elif self.parent: + return self.parent.kat_area() + +class CaveAndEntrance(models.Model): + cave = models.ForeignKey('Cave') + entrance = models.ForeignKey('Entrance') + entrance_letter = models.CharField(max_length=20,blank=True,null=True) + def __str__(self): + return str(self.cave) + str(self.entrance_letter) + +class CaveSlug(models.Model): + cave = models.ForeignKey('Cave') + slug = models.SlugField(max_length=50, unique = True) + primary = models.BooleanField(default=False) + +class Cave(TroggleModel): + # too much here perhaps, + official_name = models.CharField(max_length=160) + area = models.ManyToManyField(Area, blank=True, null=True) + kataster_code = models.CharField(max_length=20,blank=True,null=True) + kataster_number = models.CharField(max_length=10,blank=True, null=True) + unofficial_number = models.CharField(max_length=60,blank=True, null=True) + entrances = models.ManyToManyField('Entrance', through='CaveAndEntrance') + explorers = models.TextField(blank=True,null=True) + underground_description = models.TextField(blank=True,null=True) + equipment = models.TextField(blank=True,null=True) + references = models.TextField(blank=True,null=True) + survey = models.TextField(blank=True,null=True) + kataster_status = models.TextField(blank=True,null=True) + underground_centre_line = models.TextField(blank=True,null=True) + notes = models.TextField(blank=True,null=True) + length = models.CharField(max_length=100,blank=True,null=True) + depth = models.CharField(max_length=100,blank=True,null=True) + extent = models.CharField(max_length=100,blank=True,null=True) + survex_file = models.CharField(max_length=100,blank=True,null=True) + description_file = models.CharField(max_length=200,blank=True,null=True) + url = models.CharField(max_length=200,blank=True,null=True) + filename = models.CharField(max_length=200) + + #class Meta: + # unique_together = (("area", "kataster_number"), ("area", "unofficial_number")) + # FIXME Kataster Areas and CUCC defined sub areas need seperating + + #href = models.CharField(max_length=100) + + class Meta: + ordering = ('kataster_code', 'unofficial_number') + + def hassurvey(self): + if not self.underground_centre_line: + return "No" + if (self.survey.find(" -1 or self.survey.find(" -1 or self.survey.find(" -1 or self.survey.find(" -1): + return "Yes" + return "Missing" + + def hassurveydata(self): + if not self.underground_centre_line: + return "No" + if self.survex_file: + return "Yes" + return "Missing" + + def slug(self): + primarySlugs = self.caveslug_set.filter(primary = True) + if primarySlugs: + return primarySlugs[0].slug + else: + slugs = self.caveslug_set.filter() + if slugs: + return slugs[0].slug + + def ours(self): + return bool(re.search(r'CUCC', self.explorers)) + + def reference(self): + if self.kataster_number: + return "%s-%s" % (self.kat_area(), self.kataster_number) + else: + return "%s-%s" % (self.kat_area(), self.unofficial_number) + + def get_absolute_url(self): + if self.kataster_number: + href = self.kataster_number + elif self.unofficial_number: + href = self.unofficial_number + else: + href = self.official_name.lower() + #return settings.URL_ROOT + '/cave/' + href + '/' + return urllib.parse.urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,})) + + def __str__(self, sep = ": "): + return str("slug:"+self.slug()) + + def get_QMs(self): + return QM.objects.filter(found_by__cave_slug=self.caveslug_set.all()) + + def new_QM_number(self, year=datetime.date.today().year): + """Given a cave and the current year, returns the next QM number.""" + try: + res=QM.objects.filter(found_by__date__year=year, found_by__cave=self).order_by('-number')[0] + except IndexError: + return 1 + return res.number+1 + + def kat_area(self): + for a in self.area.all(): + if a.kat_area(): + return a.kat_area() + + def entrances(self): + return CaveAndEntrance.objects.filter(cave=self) + + def singleentrance(self): + return len(CaveAndEntrance.objects.filter(cave=self)) == 1 + + def entrancelist(self): + rs = [] + res = "" + for e in CaveAndEntrance.objects.filter(cave=self): + rs.append(e.entrance_letter) + rs.sort() + prevR = None + n = 0 + for r in rs: + if prevR: + if chr(ord(prevR) + 1 ) == r: + prevR = r + n += 1 + else: + if n == 0: + res += ", " + prevR + else: + res += "–" + prevR + else: + prevR = r + n = 0 + res += r + if n == 0: + res += ", " + prevR + else: + res += "–" + prevR + return res + + def writeDataFile(self): + try: + f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "wb") + except: + subprocess.call(settings.FIX_PERMISSIONS) + f = open(os.path.join(settings.CAVEDESCRIPTIONS, self.filename), "wb") + t = loader.get_template('dataformat/cave.xml') + c = Context({'cave': self}) + u = t.render(c) + u8 = u.encode("utf-8") + f.write(u8) + f.close() + + def getArea(self): + areas = self.area.all() + lowestareas = list(areas) + for area in areas: + if area.parent in areas: + try: + lowestareas.remove(area.parent) + except: + pass + return lowestareas[0] + +def getCaveByReference(reference): + areaname, code = reference.split("-", 1) + #print(areaname, code) + area = Area.objects.get(short_name = areaname) + #print(area) + foundCaves = list(Cave.objects.filter(area = area, kataster_number = code).all()) + list(Cave.objects.filter(area = area, unofficial_number = code).all()) + print((list(foundCaves))) + if len(foundCaves) == 1: + return foundCaves[0] + else: + return False + +class OtherCaveName(TroggleModel): + name = models.CharField(max_length=160) + cave = models.ForeignKey(Cave) + def __str__(self): + return str(self.name) + +class EntranceSlug(models.Model): + entrance = models.ForeignKey('Entrance') + slug = models.SlugField(max_length=50, unique = True) + primary = models.BooleanField(default=False) + +class Entrance(TroggleModel): + name = models.CharField(max_length=100, blank=True,null=True) + entrance_description = models.TextField(blank=True,null=True) + explorers = models.TextField(blank=True,null=True) + map_description = models.TextField(blank=True,null=True) + location_description = models.TextField(blank=True,null=True) + approach = models.TextField(blank=True,null=True) + underground_description = models.TextField(blank=True,null=True) + photo = models.TextField(blank=True,null=True) + MARKING_CHOICES = ( + ('P', 'Paint'), + ('P?', 'Paint (?)'), + ('T', 'Tag'), + ('T?', 'Tag (?)'), + ('R', 'Needs Retag'), + ('S', 'Spit'), + ('S?', 'Spit (?)'), + ('U', 'Unmarked'), + ('?', 'Unknown')) + marking = models.CharField(max_length=2, choices=MARKING_CHOICES) + marking_comment = models.TextField(blank=True,null=True) + FINDABLE_CHOICES = ( + ('?', 'To be confirmed ...'), + ('S', 'Coordinates'), + ('L', 'Lost'), + ('R', 'Refindable')) + findability = models.CharField(max_length=1, choices=FINDABLE_CHOICES, blank=True, null=True) + findability_description = models.TextField(blank=True,null=True) + alt = models.TextField(blank=True, null=True) + northing = models.TextField(blank=True, null=True) + easting = models.TextField(blank=True, null=True) + tag_station = models.TextField(blank=True, null=True) + exact_station = models.TextField(blank=True, null=True) + other_station = models.TextField(blank=True, null=True) + other_description = models.TextField(blank=True,null=True) + bearings = models.TextField(blank=True,null=True) + url = models.CharField(max_length=200,blank=True,null=True) + filename = models.CharField(max_length=200) + cached_primary_slug = models.CharField(max_length=200,blank=True,null=True) + + def __str__(self): + return str(self.slug()) + + def exact_location(self): + return SurvexStation.objects.lookup(self.exact_station) + def other_location(self): + return SurvexStation.objects.lookup(self.other_station) + + + def find_location(self): + r = {'': 'To be entered ', + '?': 'To be confirmed:', + 'S': '', + 'L': 'Lost:', + 'R': 'Refindable:'}[self.findability] + if self.tag_station: + try: + s = SurvexStation.objects.lookup(self.tag_station) + return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z) + except: + return r + "%s Tag Station not in dataset" % self.tag_station + if self.exact_station: + try: + s = SurvexStation.objects.lookup(self.exact_station) + return r + "%0.0fE %0.0fN %0.0fAlt" % (s.x, s.y, s.z) + except: + return r + "%s Exact Station not in dataset" % self.tag_station + if self.other_station: + try: + s = SurvexStation.objects.lookup(self.other_station) + return r + "%0.0fE %0.0fN %0.0fAlt %s" % (s.x, s.y, s.z, self.other_description) + except: + return r + "%s Other Station not in dataset" % self.tag_station + if self.FINDABLE_CHOICES == "S": + r += "ERROR, Entrance has been surveyed but has no survex point" + if self.bearings: + return r + self.bearings + return r + + def best_station(self): + if self.tag_station: + return self.tag_station + if self.exact_station: + return self.exact_station + if self.other_station: + return self.other_station + + def has_photo(self): + if self.photo: + if (self.photo.find(" -1 or self.photo.find(" -1 or self.photo.find(" -1 or self.photo.find(" -1): + return "Yes" + else: + return "Missing" + else: + return "No" + + def marking_val(self): + for m in self.MARKING_CHOICES: + if m[0] == self.marking: + return m[1] + def findability_val(self): + for f in self.FINDABLE_CHOICES: + if f[0] == self.findability: + return f[1] + + def tag(self): + return SurvexStation.objects.lookup(self.tag_station) + + def needs_surface_work(self): + return self.findability != "S" or not self.has_photo or self.marking != "T" + + def get_absolute_url(self): + + ancestor_titles='/'.join([subcave.title for subcave in self.get_ancestors()]) + if ancestor_titles: + res = '/'.join((self.get_root().cave.get_absolute_url(), ancestor_titles, self.title)) + + else: + res = '/'.join((self.get_root().cave.get_absolute_url(), self.title)) + + return res + + def slug(self): + if not self.cached_primary_slug: + primarySlugs = self.entranceslug_set.filter(primary = True) + if primarySlugs: + self.cached_primary_slug = primarySlugs[0].slug + self.save() + else: + slugs = self.entranceslug_set.filter() + if slugs: + self.cached_primary_slug = slugs[0].slug + self.save() + return self.cached_primary_slug + + def writeDataFile(self): + try: + f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w") + except: + subprocess.call(settings.FIX_PERMISSIONS) + f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w") + t = loader.get_template('dataformat/entrance.xml') + c = Context({'entrance': self}) + u = t.render(c) + u8 = u.encode("utf-8") + f.write(u8) + f.close() + +class CaveDescription(TroggleModel): + short_name = models.CharField(max_length=50, unique = True) + long_name = models.CharField(max_length=200, blank=True, null=True) + description = models.TextField(blank=True,null=True) + linked_subcaves = models.ManyToManyField("NewSubCave", blank=True,null=True) + linked_entrances = models.ManyToManyField("Entrance", blank=True,null=True) + linked_qms = models.ManyToManyField("QM", blank=True,null=True) + + def __str__(self): + if self.long_name: + return str(self.long_name) + else: + return str(self.short_name) + + def get_absolute_url(self): + return urllib.parse.urljoin(settings.URL_ROOT, reverse('cavedescription', args=(self.short_name,))) + + def save(self): + """ + Overridden save method which stores wikilinks in text as links in database. + """ + TroggleModel.save() + #super(CaveDescription, self).save() # fails in python 3.8, OK in python 3.5 + qm_list=get_related_by_wikilinks(self.description) + for qm in qm_list: + self.linked_qms.add(qm) + TroggleModel.save() + #super(CaveDescription, self).save() # fails in python 3.8, OK in python 3.5 + +class NewSubCave(TroggleModel): + name = models.CharField(max_length=200, unique = True) + def __str__(self): + return str(self.name) + +class QM(TroggleModel): + #based on qm.csv in trunk/expoweb/1623/204 which has the fields: + #"Number","Grade","Area","Description","Page reference","Nearest station","Completion description","Comment" + found_by = models.ForeignKey(LogbookEntry, related_name='QMs_found',blank=True, null=True ) + ticked_off_by = models.ForeignKey(LogbookEntry, related_name='QMs_ticked_off',null=True,blank=True) + #cave = models.ForeignKey(Cave) + #expedition = models.ForeignKey(Expedition) + + number = models.IntegerField(help_text="this is the sequential number in the year", ) + GRADE_CHOICES=( + ('A', 'A: Large obvious lead'), + ('B', 'B: Average lead'), + ('C', 'C: Tight unpromising lead'), + ('D', 'D: Dig'), + ('X', 'X: Unclimbable aven') + ) + grade = models.CharField(max_length=1, choices=GRADE_CHOICES) + location_description = models.TextField(blank=True) + nearest_station_description = models.CharField(max_length=400,null=True,blank=True) + nearest_station_name = models.CharField(max_length=200,blank=True,null=True) + nearest_station = models.ForeignKey(SurvexStation,null=True,blank=True) + area = models.CharField(max_length=100,blank=True,null=True) + completion_description = models.TextField(blank=True,null=True) + comment=models.TextField(blank=True,null=True) + + def __str__(self): + return "%s %s" % (self.code(), self.grade) + + def code(self): + return "%s-%s-%s" % (str(self.found_by.cave)[6:], self.found_by.date.year, self.number) + + def get_absolute_url(self): + #return settings.URL_ROOT + '/cave/' + self.found_by.cave.kataster_number + '/' + str(self.found_by.date.year) + '-' + '%02d' %self.number + return urllib.parse.urljoin(settings.URL_ROOT, reverse('qm',kwargs={'cave_id':self.found_by.cave.kataster_number,'year':self.found_by.date.year,'qm_id':self.number,'grade':self.grade})) + + def get_next_by_id(self): + return QM.objects.get(id=self.id+1) + + def get_previous_by_id(self): + return QM.objects.get(id=self.id-1) + + def wiki_link(self): + return "%s%s%s" % ('[[QM:',self.code(),']]') + +scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL) +def get_scan_path(instance, filename): + year=instance.survey.expedition.year + #print("WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number, instance.survey.wallet_letter) + number=str(instance.survey.wallet_number) + if str(instance.survey.wallet_letter) != "None": + number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01 + return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg') + +class ScannedImage(TroggleImageModel): + file = models.ImageField(storage=scansFileStorage, upload_to=get_scan_path) + scanned_by = models.ForeignKey(Person,blank=True, null=True) + scanned_on = models.DateField(null=True) + survey = models.ForeignKey('Survey') + contents = models.CharField(max_length=20,choices=(('notes','notes'),('plan','plan_sketch'),('elevation','elevation_sketch'))) + number_in_wallet = models.IntegerField(null=True) + lon_utm = models.FloatField(blank=True,null=True) + lat_utm = models.FloatField(blank=True,null=True) + + #content_type = models.ForeignKey(ContentType) + #object_id = models.PositiveIntegerField() + #location = generic.GenericForeignKey('content_type', 'object_id') + + #This is an ugly hack to deal with the #s in our survey scan paths. The correct thing is to write a custom file storage backend which calls urlencode on the name for making file.url but not file.path. + def correctURL(self): + return string.replace(self.file.url,r'#',r'%23') + + def __str__(self): + return get_scan_path(self,'') + +class Survey(TroggleModel): + expedition = models.ForeignKey('Expedition') #REDUNDANT (logbook_entry) + wallet_number = models.IntegerField(blank=True,null=True) + wallet_letter = models.CharField(max_length=1,blank=True,null=True) + comments = models.TextField(blank=True,null=True) + location = models.CharField(max_length=400,blank=True,null=True) #REDUNDANT + subcave = models.ForeignKey('NewSubCave', blank=True, null=True) + #notes_scan = models.ForeignKey('ScannedImage',related_name='notes_scan',blank=True, null=True) #Replaced by contents field of ScannedImage model + survex_block = models.OneToOneField('SurvexBlock',blank=True, null=True) + logbook_entry = models.ForeignKey('LogbookEntry') + centreline_printed_on = models.DateField(blank=True, null=True) + centreline_printed_by = models.ForeignKey('Person',related_name='centreline_printed_by',blank=True,null=True) + #sketch_scan = models.ForeignKey(ScannedImage,blank=True, null=True) #Replaced by contents field of ScannedImage model + tunnel_file = models.FileField(upload_to='surveyXMLfiles',blank=True, null=True) + tunnel_main_sketch = models.ForeignKey('Survey',blank=True,null=True) + integrated_into_main_sketch_on = models.DateField(blank=True,null=True) + integrated_into_main_sketch_by = models.ForeignKey('Person' ,related_name='integrated_into_main_sketch_by', blank=True,null=True) + rendered_image = models.ImageField(upload_to='renderedSurveys',blank=True,null=True) + def __str__(self): + return self.expedition.year+"#"+"%02d" % int(self.wallet_number) + + def notes(self): + return self.scannedimage_set.filter(contents='notes') + + def plans(self): + return self.scannedimage_set.filter(contents='plan') + + def elevations(self): + return self.scannedimage_set.filter(contents='elevation') + + diff --git a/core/view_surveys.py b/core/view_surveys.py index 65a4f30..845cab2 100644 --- a/core/view_surveys.py +++ b/core/view_surveys.py @@ -4,7 +4,7 @@ from django.shortcuts import render_to_response from django.http import HttpResponse, Http404 import os, stat import re -from troggle.core.models import SurvexScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile +from troggle.core.models_survex import SurvexScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile import parsers.surveys import urllib.request, urllib.parse, urllib.error diff --git a/core/views_caves.py b/core/views_caves.py index c306461..31f6707 100644 --- a/core/views_caves.py +++ b/core/views_caves.py @@ -21,7 +21,7 @@ from django.shortcuts import get_object_or_404, render import troggle.settings as settings import troggle.core.models as models -from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation +from troggle.core.models_caves import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm from troggle.helper import login_required_if_public diff --git a/core/views_logbooks.py b/core/views_logbooks.py index b29b72e..b02200f 100644 --- a/core/views_logbooks.py +++ b/core/views_logbooks.py @@ -1,23 +1,27 @@ -from django.shortcuts import render_to_response, render -from troggle.core.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry, SurvexBlock -import troggle.core.models as models -import troggle.settings as settings +import datetime +import os.path +import re + import django.db.models +from django.core.urlresolvers import reverse +from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import render, render_to_response +from django.template import Context, loader +from django.template.defaultfilters import slugify +from django.utils import timezone +from django.views.generic.list import ListView + +import troggle.core.models as models +import troggle.parsers.logbooks as logbookparsers +from troggle.core.forms import getTripForm # , get_name, PersonForm +from troggle.core.models import Expedition, LogbookEntry, Person, PersonExpedition, PersonTrip +from troggle.core.models_survex import SurvexBlock +from troggle.helper import login_required_if_public from troggle.parsers.logbooks import LoadLogbookForExpedition from troggle.parsers.people import GetPersonExpeditionNameLookup -from troggle.core.forms import getTripForm#, get_name, PersonForm -from django.core.urlresolvers import reverse -from django.http import HttpResponseRedirect, HttpResponse -from django.template import Context, loader -import os.path -import troggle.parsers.logbooks as logbookparsers -from django.template.defaultfilters import slugify -from troggle.helper import login_required_if_public -import datetime - -from django.views.generic.list import ListView -from django.utils import timezone +import troggle.settings as settings +from settings import * # Django uses Context, not RequestContext when you call render # to_response. We always want to use RequestContext, so that @@ -28,7 +32,6 @@ from django.utils import timezone # will make this unnecessary. # from troggle.alwaysUseRequestContext import render_response -import re @django.db.models.permalink #this allows the nice get_absolute_url syntax we are using @@ -162,7 +165,6 @@ def personForm(request,pk): form=PersonForm(instance=person) return render(request,'personform.html', {'form':form,}) -from settings import * def pathsreport(request): pathsdict={ "ADMIN_MEDIA_PREFIX" : ADMIN_MEDIA_PREFIX, diff --git a/core/views_other.py b/core/views_other.py index 4da8e70..28a6d94 100644 --- a/core/views_other.py +++ b/core/views_other.py @@ -9,9 +9,8 @@ from django.shortcuts import render from django.template import Context, loader import databaseReset -from troggle.core.models import * -from troggle.core.models import (QM, Cave, Expedition, LogbookEntry, Person, - PersonExpedition, PersonTrip) +from troggle.core.models import Expedition, LogbookEntry, Person, PersonExpedition, PersonTrip +from troggle.core.models_caves import QM, Cave from troggle.helper import login_required_if_public diff --git a/core/views_survex.py b/core/views_survex.py index 0435d52..cec4d6c 100644 --- a/core/views_survex.py +++ b/core/views_survex.py @@ -1,19 +1,22 @@ -from django import forms -from django.http import HttpResponseRedirect, HttpResponse -from django.shortcuts import render_to_response, render -from django.core.context_processors import csrf -from django.http import HttpResponse, Http404 import re import os import datetime import difflib -from troggle.core.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry, Cave -from troggle.core.models import SurvexBlock, SurvexPersonRole, SurvexFile, SurvexDirectory, SurvexTitle -from parsers.people import GetPersonExpeditionNameLookup +from django import forms +from django.http import HttpResponseRedirect, HttpResponse +from django.shortcuts import render_to_response, render +from django.core.context_processors import csrf +from django.http import HttpResponse, Http404 import troggle.settings as settings import parsers.survex +from troggle.core.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry +from troggle.core.models_survex import SurvexBlock, SurvexPersonRole, SurvexFile, SurvexDirectory, SurvexTitle +from troggle.core.models_caves import Cave +from troggle.parsers.people import GetPersonExpeditionNameLookup + + survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau (kataster group 1623) ; Cave: diff --git a/databaseReset.py b/databaseReset.py index 24afdaa..83d319f 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -15,7 +15,7 @@ from django.contrib.auth.models import User from django.http import HttpResponse from django.core.urlresolvers import reverse -from troggle.core.models import Cave, Entrance +from troggle.core.models_caves import Cave, Entrance import troggle.settings import troggle.flatpages.models import troggle.logbooksdump diff --git a/flatpages/models.py b/flatpages/models.py index 23739ca..50cf6aa 100644 --- a/flatpages/models.py +++ b/flatpages/models.py @@ -1,5 +1,5 @@ from django.db import models -from troggle.core.models import Cave, Entrance +from troggle.core.models_caves import Cave, Entrance class Redirect(models.Model): originalURL = models.CharField(max_length=200, unique=True) diff --git a/flatpages/views.py b/flatpages/views.py index 27f0cd0..1905461 100644 --- a/flatpages/views.py +++ b/flatpages/views.py @@ -11,7 +11,7 @@ from tinymce.widgets import TinyMCE from troggle.helper import login_required_if_public from troggle.flatpages.models import Redirect, EntranceRedirect -from troggle.core.models import Cave +from troggle.core.models_caves import Cave import troggle.core.views_caves import troggle.settings as settings diff --git a/logbooksdump.py b/logbooksdump.py index 29a0248..af7ac74 100644 --- a/logbooksdump.py +++ b/logbooksdump.py @@ -12,7 +12,7 @@ from django.contrib.auth.models import User from django.http import HttpResponse from django.core.urlresolvers import reverse -from troggle.core.models import Cave, Entrance +from troggle.core.models_caves import Cave, Entrance import troggle.flatpages.models # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/parsers/QMs.py b/parsers/QMs.py index 578e422..24ad7c9 100644 --- a/parsers/QMs.py +++ b/parsers/QMs.py @@ -7,7 +7,7 @@ from datetime import * from django.conf import settings -from troggle.core.models import QM, Cave, LogbookEntry +from troggle.core.models_caves import QM, Cave, LogbookEntry from utils import save_carefully diff --git a/parsers/caves.py b/parsers/caves.py index 745b119..514aa78 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -5,14 +5,15 @@ import re from django.conf import settings import troggle.core.models as models +import troggle.core.models_caves as models_caves def readcaves(): # Clear the cave data issues as we are reloading models.DataIssue.objects.filter(parser='caves').delete() - area_1623 = models.Area.objects.update_or_create(short_name = "1623", parent = None) - area_1626 = models.Area.objects.update_or_create(short_name = "1626", parent = None) + area_1623 = models_caves.Area.objects.update_or_create(short_name = "1623", parent = None) + area_1626 = models_caves.Area.objects.update_or_create(short_name = "1626", parent = None) print(" - Reading Entrances") #print "list of " for filename in next(os.walk(settings.ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files @@ -23,6 +24,7 @@ def readcaves(): if filename.endswith('.html'): readcave(filename) + def readentrance(filename): with open(os.path.join(settings.ENTRANCEDESCRIPTIONS, filename)) as f: contents = f.read() @@ -54,7 +56,7 @@ def readentrance(filename): bearings = getXML(entrancecontents, "bearings", maxItems = 1, context = context) url = getXML(entrancecontents, "url", maxItems = 1, context = context) if len(non_public) == 1 and len(slugs) >= 1 and len(name) >= 1 and len(entrance_description) == 1 and len(explorers) == 1 and len(map_description) == 1 and len(location_description) == 1 and len(approach) == 1 and len(underground_description) == 1 and len(marking) == 1 and len(marking_comment) == 1 and len(findability) == 1 and len(findability_description) == 1 and len(alt) == 1 and len(northing) == 1 and len(easting) == 1 and len(tag_station) == 1 and len(exact_station) == 1 and len(other_station) == 1 and len(other_description) == 1 and len(bearings) == 1 and len(url) == 1: - e, state = models.Entrance.objects.update_or_create(name = name[0], + e, state = models_caves.Entrance.objects.update_or_create(name = name[0], non_public = {"True": True, "False": False, "true": True, "false": False,}[non_public[0]], entrance_description = entrance_description[0], explorers = explorers[0], @@ -81,7 +83,7 @@ def readentrance(filename): primary = True for slug in slugs: #print slug, filename - cs = models.EntranceSlug.objects.update_or_create(entrance = e, + cs = models_caves.EntranceSlug.objects.update_or_create(entrance = e, slug = slug, primary = primary) primary = False @@ -118,7 +120,7 @@ def readcave(filename): url = getXML(cavecontents, "url", maxItems = 1, context = context) entrances = getXML(cavecontents, "entrance", context = context) if len(non_public) == 1 and len(slugs) >= 1 and len(official_name) == 1 and len(areas) >= 1 and len(kataster_code) == 1 and len(kataster_number) == 1 and len(unofficial_number) == 1 and len(explorers) == 1 and len(underground_description) == 1 and len(equipment) == 1 and len(references) == 1 and len(survey) == 1 and len(kataster_status) == 1 and len(underground_centre_line) == 1 and len(notes) == 1 and len(length) == 1 and len(depth) == 1 and len(extent) == 1 and len(survex_file) == 1 and len(description_file ) == 1 and len(url) == 1 and len(entrances) >= 1: - c, state = models.Cave.objects.update_or_create(non_public = {"True": True, "False": False, "true": True, "false": False,}[non_public[0]], + c, state = models_caves.Cave.objects.update_or_create(non_public = {"True": True, "False": False, "true": True, "false": False,}[non_public[0]], official_name = official_name[0], kataster_code = kataster_code[0], kataster_number = kataster_number[0], @@ -139,17 +141,17 @@ def readcave(filename): url = url[0], filename = filename) for area_slug in areas: - area = models.Area.objects.filter(short_name = area_slug) + area = models_caves.Area.objects.filter(short_name = area_slug) if area: newArea = area[0] else: - newArea = models.Area(short_name = area_slug, parent = models.Area.objects.get(short_name = "1623")) + newArea = models_caves.Area(short_name = area_slug, parent = models_caves.Area.objects.get(short_name = "1623")) newArea.save() c.area.add(newArea) primary = True for slug in slugs: try: - cs = models.CaveSlug.objects.update_or_create(cave = c, + cs = models_caves.CaveSlug.objects.update_or_create(cave = c, slug = slug, primary = primary) except: @@ -162,8 +164,8 @@ def readcave(filename): slug = getXML(entrance, "entranceslug", maxItems = 1, context = context)[0] letter = getXML(entrance, "letter", maxItems = 1, context = context)[0] try: - entrance = models.Entrance.objects.get(entranceslug__slug = slug) - ce = models.CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance) + entrance = models_caves.Entrance.objects.get(entranceslug__slug = slug) + ce = models_caves.CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance) except: message = " ! Entrance text (slug) %s missing %s" % (slug, context) models.DataIssue.objects.create(parser='caves', message=message) @@ -185,4 +187,4 @@ def getXML(text, itemname, minItems = 1, maxItems = None, printwarnings = True, "max": maxItems} + context models.DataIssue.objects.create(parser='caves', message=message) print(message) - return items + return items \ No newline at end of file diff --git a/parsers/logbooks.py b/parsers/logbooks.py index e2f0ba0..f0ae2fa 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -12,6 +12,7 @@ from django.template.defaultfilters import slugify from django.utils.timezone import get_current_timezone, make_aware import troggle.core.models as models +import troggle.core.models_caves as models_caves from parsers.people import GetPersonExpeditionNameLookup from utils import save_carefully @@ -49,23 +50,24 @@ def GetTripPersons(trippeople, expedition, logtime_underground): author = res[-1][0] return res, author -def GetTripCave(place): #need to be fuzzier about matching here. Already a very slow function... -# print "Getting cave for " , place +def GetTripCave(place): + #need to be fuzzier about matching here. Already a very slow function... + # print "Getting cave for " , place try: katastNumRes=[] - katastNumRes=list(models.Cave.objects.filter(kataster_number=int(place))) + katastNumRes=list(models_caves.Cave.objects.filter(kataster_number=int(place))) except ValueError: pass - officialNameRes=list(models.Cave.objects.filter(official_name=place)) + officialNameRes=list(models_caves.Cave.objects.filter(official_name=place)) tripCaveRes=officialNameRes+katastNumRes if len(tripCaveRes)==1: -# print "Place " , place , "entered as" , tripCaveRes[0] + # print "Place " , place , "entered as" , tripCaveRes[0] return tripCaveRes[0] - elif models.OtherCaveName.objects.filter(name=place): - tripCaveRes=models.OtherCaveName.objects.filter(name__icontains=place)[0].cave -# print "Place " , place , "entered as" , tripCaveRes + elif models_caves.OtherCaveName.objects.filter(name=place): + tripCaveRes=models_caves.OtherCaveName.objects.filter(name__icontains=place)[0].cave + # print "Place " , place , "entered as" , tripCaveRes return tripCaveRes elif len(tripCaveRes)>1: @@ -76,6 +78,25 @@ def GetTripCave(place): #need to be fuzzier about matching here. Already a very print(("No cave found for place " , place)) return +# lookup function modelled on GetPersonExpeditionNameLookup +Gcavelookup = None +def GetCaveLookup(): + global Gcavelookup + if Gcavelookup: + return Gcavelookup + Gcavelookup = {"NONEPLACEHOLDER":None} + for cave in models_caves.Cave.objects.all(): + Gcavelookup[cave.official_name.lower()] = cave + if cave.kataster_number: + Gcavelookup[cave.kataster_number] = cave + if cave.unofficial_number: + Gcavelookup[cave.unofficial_number] = cave + + Gcavelookup["tunnocks"] = Gcavelookup["258"] + Gcavelookup["hauchhole"] = Gcavelookup["234"] + return Gcavelookup + + logentries = [] # the entire logbook is a single object: a list of entries noncaveplaces = [ "Journey", "Loser Plateau" ] @@ -195,7 +216,6 @@ def Parseloghtmltxt(year, expedition, txt): if logbook_entry_count == 0: print(" - No trip entries found in logbook, check the syntax matches htmltxt format") - # main parser for 1991 - 2001. simpler because the data has been hacked so much to fit it def Parseloghtml01(year, expedition, txt): tripparas = re.findall(r"([\s\S]*?)(?= 0: # print(insp+from_section[0]) - from_station = models.SurvexStation.objects.filter(block=from_section[0], name=qm_from_station) + from_station = models_survex.SurvexStation.objects.filter(block=from_section[0], name=qm_from_station) # If we can find a from station then we have the nearest station and can import it if len(from_station) > 0: # print(insp+from_station[0]) - qm = models.QM.objects.create(number=qm_no, + qm = models_caves.QM.objects.create(number=qm_no, nearest_station=from_station[0], grade=qm_grade.upper(), location_description=qm_notes) @@ -261,7 +263,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): assert (int(yr)>1960 and int(yr)<2039), "Wallet year out of bounds: %s" % yr assert (int(wallet)<100), "Wallet number more than 100: %s" % wallet refscan = "%s#%s%s" % (yr, letterx, wallet) - survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan) + survexscansfolders = models_survex.SurvexScansFolder.objects.filter(walletname=refscan) if survexscansfolders: survexblock.survexscansfolder = survexscansfolders[0] survexblock.save() @@ -299,12 +301,12 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): if path_match: pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2)) # print(insp+pos_cave) - cave = models.getCaveByReference(pos_cave) + cave = models_caves.getCaveByReference(pos_cave) if cave: survexfile.cave = cave else: print((insp+' - No match in DB (i) for %s, so loading..' % includepath)) - includesurvexfile = models.SurvexFile(path=includepath) + includesurvexfile = models_survex.SurvexFile(path=includepath) includesurvexfile.save() includesurvexfile.SetDirectory() if includesurvexfile.exists(): @@ -322,7 +324,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): if path_match: pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2)) # print(insp+pos_cave) - cave = models.getCaveByReference(pos_cave) + cave = models_caves.getCaveByReference(pos_cave) if cave: survexfile.cave = cave else: @@ -331,7 +333,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): name = line.lower() print((insp+' - Begin found for: ' + name)) # print(insp+'Block cave: ' + str(survexfile.cave)) - survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0) + survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0) survexblockdown.save() survexblock.save() survexblock = survexblockdown @@ -376,7 +378,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower()) if (personexpedition, tm) not in teammembers: teammembers.append((personexpedition, tm)) - personrole = models.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm) + personrole = models_survex.SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm) personrole.expeditionday = survexblock.expeditionday if personexpedition: personrole.person=personexpedition.person @@ -384,7 +386,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): elif cmd == "title": #print(insp+' - Title found: ') - survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexfile.cave) + survextitle = models_survex.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexfile.cave) survextitle.save() pass @@ -431,14 +433,14 @@ def LoadAllSurvexBlocks(): print(' - Flushing All Survex Blocks...') - models.SurvexBlock.objects.all().delete() - models.SurvexFile.objects.all().delete() - models.SurvexDirectory.objects.all().delete() - models.SurvexEquate.objects.all().delete() - models.SurvexLeg.objects.all().delete() - models.SurvexTitle.objects.all().delete() - models.SurvexPersonRole.objects.all().delete() - models.SurvexStation.objects.all().delete() + models_survex.SurvexBlock.objects.all().delete() + models_survex.SurvexFile.objects.all().delete() + models_survex.SurvexDirectory.objects.all().delete() + models_survex.SurvexEquate.objects.all().delete() + models_survex.SurvexLeg.objects.all().delete() + models_survex.SurvexTitle.objects.all().delete() + models_survex.SurvexPersonRole.objects.all().delete() + models_survex.SurvexStation.objects.all().delete() print(" - Data flushed") # Clear the data issues as we are reloading @@ -450,12 +452,12 @@ def LoadAllSurvexBlocks(): # Redirect sys.stdout to the file sys.stdout = open('loadsurvexblks.log', 'w') - survexfile = models.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None) + survexfile = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None) survexfile.save() survexfile.SetDirectory() #Load all - survexblockroot = models.SurvexBlock(name="root", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0) + survexblockroot = models_survex.SurvexBlock(name="root", survexpath="", begin_char=0, cave=None, survexfile=survexfile, totalleglength=0.0) survexblockroot.save() fin = survexfile.OpenFile() textlines = [ ] @@ -545,14 +547,14 @@ def LoadPos(): for line in posfile.readlines(): r = poslineregex.match(line) if r: - x, y, z, id = r.groups() # easting, northing, altitude, survexstation + x, y, z, id = r.groups() if id in notfoundbefore: skip[id] = 1 else: for sid in mappoints: if id.endswith(sid): try: - ss = models.SurvexStation.objects.lookup(id) + ss = models_survex.SurvexStation.objects.lookup(id) ss.x = float(x) ss.y = float(y) ss.z = float(z) diff --git a/parsers/surveys.py b/parsers/surveys.py index 2f0ff90..80be33f 100644 --- a/parsers/surveys.py +++ b/parsers/surveys.py @@ -16,6 +16,8 @@ from functools import reduce import settings from troggle.core.models import * +from troggle.core.models_caves import * +from troggle.core.models_survex import * def get_or_create_placeholder(year): """ All surveys must be related to a logbookentry. We don't have a way to diff --git a/utils.py b/utils.py index 515b38d..22f50f6 100644 --- a/utils.py +++ b/utils.py @@ -4,7 +4,7 @@ import logging from django.conf import settings from django.shortcuts import render -from troggle.core.models import CaveDescription +from troggle.core.models_caves import CaveDescription def weighted_choice(lst): n = random.uniform(0,1)