Changes needed to stop the survex parser having to go through the data twice

Taken from the Django 1.10 upgrade branch

.gitignore

@@ -7,3 +7,10 @@ localsettings.py
 *~
 parsing_log.txt
 troggle
+troggle_log.txt
+.idea/*
+*.orig
+media/images/*
+.vscode/*
+.swp
+imagekit-off/

README.txt

@@ -1,6 +1,6 @@
 Troggle is an application for caving expedition data management, originally created for use on Cambridge University Caving Club (CUCC)expeditions and licensed under the GNU Lesser General Public License.
 
-Troggle has been forked into two projects. The original one is maintained by Aron Curtis and is used for Erebus caves. The CUCC variant uses files as the definitive data, not the database and lives at expo.sruvex.com/troggle.
+Troggle has been forked into two projects. The original one is maintained by Aron Curtis and is used for Erebus caves. The CUCC variant uses files as the definitive data, not the database and lives at expo.survex.com/troggle.
 
 Troggle setup
 ==========
@@ -18,12 +18,14 @@ If you want to use MySQL or Postgresql, download and install them. However, you
 
 Troggle itself
 -------------
-Choose a directory where you will keep troggle, and svn check out Troggle into it using the following command:
+Choose a directory where you will keep troggle, and git clone Troggle into it using the following command:
 
-svn co http://troggle.googlecode.com/svn/
+git clone git://expo.survex.com/troggle
+or more reliably
+git clone ssh://expo@expo.survex.com/home/expo/troggle
 
 
-If you want to work on the source code and be able to commit, you will need to use https instead of http, and your google account will need to be added to the troggle project members list. Contact aaron dot curtis at cantab dot net to get this set up.
+If you want to work on the source code and be able to commit, your account will need to be added to the troggle project members list. Contact wookey  at wookware dot org to get this set up.
 
 Next, you need to fill in your local settings. Copy either localsettingsubuntu.py or localsettingsserver.py to a new file called localsettings.py. Follow the instructions contained in the file to fill out your settings.
 
@@ -35,7 +37,7 @@ Run "python databaseReset.py reset" from the troggle directory.
 Once troggle is running, you can also log in and then go to "Import / export" data under "admin" on the menu.
 
 Adding a new year/expedition requires adding a column to the
-noinfo/folk.csv table - a year doesn't exist until that is done.
+folk/folk.csv table - a year doesn't exist until that is done.
 
 
 Running a Troggle server

core/admin.py

@@ -18,41 +18,50 @@ class TroggleModelAdmin(admin.ModelAdmin):
     class Media:
         js = ('jquery/jquery.min.js','js/QM_helper.js')
 
+
 class RoleInline(admin.TabularInline):
     model = SurvexPersonRole
     extra = 4
 
+
 class SurvexBlockAdmin(TroggleModelAdmin):
     inlines = (RoleInline,)
 
+
 class ScannedImageInline(admin.TabularInline):
     model = ScannedImage
     extra = 4
 
+
 class OtherCaveInline(admin.TabularInline):
     model = OtherCaveName
     extra = 1
 
+
 class SurveyAdmin(TroggleModelAdmin):
     inlines = (ScannedImageInline,)
     search_fields = ('expedition__year','wallet_number')    
 
+
 class QMsFoundInline(admin.TabularInline):
     model=QM
     fk_name='found_by'
     fields=('number','grade','location_description','comment')#need to add foreignkey to cave part
     extra=1
     
+
 class PhotoInline(admin.TabularInline):
     model = DPhoto
     exclude = ['is_mugshot' ]
     extra = 1
 
+
 class PersonTripInline(admin.TabularInline):
     model = PersonTrip
     raw_id_fields = ('personexpedition',)
     extra = 1
 
+
 #class LogbookEntryAdmin(VersionAdmin):
 class LogbookEntryAdmin(TroggleModelAdmin):
     prepopulated_fields = {'slug':("title",)} 
@@ -65,24 +74,25 @@ class LogbookEntryAdmin(TroggleModelAdmin):
         }
     actions=('export_logbook_entries_as_html','export_logbook_entries_as_txt')
     
-    def export_logbook_entries_as_html(modeladmin, request, queryset):
+    def export_logbook_entries_as_html(self, modeladmin, request, queryset):
         response=downloadLogbook(request=request, queryset=queryset, extension='html')
         return response
         
-    def export_logbook_entries_as_txt(modeladmin, request, queryset):
+    def export_logbook_entries_as_txt(self, modeladmin, request, queryset):
         response=downloadLogbook(request=request, queryset=queryset, extension='txt')
         return response
-    
+
-    
 
 class PersonExpeditionInline(admin.TabularInline):
     model = PersonExpedition
     extra = 1
 
+
 class PersonAdmin(TroggleModelAdmin):
     search_fields = ('first_name','last_name')
     inlines = (PersonExpeditionInline,)
 
+
 class QMAdmin(TroggleModelAdmin):
     search_fields = ('found_by__cave__kataster_number','number','found_by__date')
     list_display = ('__unicode__','grade','found_by','ticked_off_by')
@@ -91,17 +101,21 @@ class QMAdmin(TroggleModelAdmin):
     list_per_page = 20
     raw_id_fields=('found_by','ticked_off_by')
 
+
 class PersonExpeditionAdmin(TroggleModelAdmin):
     search_fields = ('person__first_name','expedition__year')
 
+
 class CaveAdmin(TroggleModelAdmin):
     search_fields = ('official_name','kataster_number','unofficial_number')
     inlines = (OtherCaveInline,)
     extra = 4
 
+
 class EntranceAdmin(TroggleModelAdmin):
     search_fields = ('caveandentrance__cave__kataster_number',)
 
+
 admin.site.register(DPhoto)
 admin.site.register(Cave, CaveAdmin)
 admin.site.register(Area)
@@ -125,17 +139,21 @@ admin.site.register(SurvexStation)
 admin.site.register(SurvexScansFolder)
 admin.site.register(SurvexScanSingle)
 
+admin.site.register(DataIssue)
+
 def export_as_json(modeladmin, request, queryset):
-    response = HttpResponse(mimetype="text/json")
+    response = HttpResponse(content_type="text/json")
     response['Content-Disposition'] = 'attachment; filename=troggle_output.json'
     serializers.serialize("json", queryset, stream=response)
     return response
 
+
 def export_as_xml(modeladmin, request, queryset):
-    response = HttpResponse(mimetype="text/xml")
+    response = HttpResponse(content_type="text/xml")
     response['Content-Disposition'] = 'attachment; filename=troggle_output.xml'
     serializers.serialize("xml", queryset, stream=response)
     return response
 
+
 #admin.site.add_action(export_as_xml)
 #admin.site.add_action(export_as_json)

core/fileAbstraction.py

@@ -26,7 +26,7 @@ def listdir(*path):
         else:
             c = ""
         c = c.replace("#", "%23")
-        print "FILE: ", settings.FILES + "listdir/" + c
+        print("FILE: ", settings.FILES + "listdir/" + c)
         return urllib.urlopen(settings.FILES + "listdir/" + c).read()
         
 def dirsAsList(*path):

core/forms.py

@@ -46,12 +46,12 @@ class EntranceForm(ModelForm):
     #underground_centre_line = forms.CharField(required = False, widget=TinyMCE(attrs={'cols': 80, 'rows': 10}))
     #notes = forms.CharField(required = False, widget=TinyMCE(attrs={'cols': 80, 'rows': 10}))
     #references = forms.CharField(required = False, widget=TinyMCE(attrs={'cols': 80, 'rows': 10}))
-    other_station = forms.CharField(required=False) # Trying to change this to a singl;e line entry
+    other_station = forms.CharField(required=False) # Trying to change this to a single line entry
-    tag_station = forms.CharField(required=False) # Trying to change this to a singl;e line entry
+    tag_station = forms.CharField(required=False) # Trying to change this to a single line entry
-    exact_station = forms.CharField(required=False) # Trying to change this to a singl;e line entry
+    exact_station = forms.CharField(required=False) # Trying to change this to a single line entry
-    northing = forms.CharField(required=False) # Trying to change this to a singl;e line entry
+    northing = forms.CharField(required=False) # Trying to change this to a single line entry
-    easting = forms.CharField(required=False) # Trying to change this to a singl;e line entry
+    easting = forms.CharField(required=False) # Trying to change this to a single line entry
-    alt = forms.CharField(required=False) # Trying to change this to a singl;e line entry
+    alt = forms.CharField(required=False) # Trying to change this to a single line entry
     class Meta:
         model = Entrance
         exclude = ("cached_primary_slug", "filename",)
@@ -123,7 +123,7 @@ def getTripForm(expedition):
         html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30})) 
 
         def clean(self):
-            print dir(self)
+            print(dir(self))
             if self.cleaned_data.get("caveOrLocation") == "cave" and not self.cleaned_data.get("cave"):
                 self._errors["cave"] = self.error_class(["This field is required"]) 
             if self.cleaned_data.get("caveOrLocation") == "location" and not self.cleaned_data.get("location"):
@@ -148,32 +148,32 @@ def get_name(pe):
             else:
                 return pe.person.first_name
 
-class UploadFileForm(forms.Form):
+#class UploadFileForm(forms.Form):
-    title = forms.CharField(max_length=50)
+#    title = forms.CharField(max_length=50)
-    file  = forms.FileField()
+#    file  = forms.FileField()
-    html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30}))
+#    html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30}))
-    lon_utm = forms.FloatField(required=False)
+#    lon_utm = forms.FloatField(required=False)
-    lat_utm = forms.FloatField(required=False)
+#    lat_utm = forms.FloatField(required=False)
-    slug = forms.CharField(max_length=50)
+#    slug = forms.CharField(max_length=50)
-    date = forms.DateField(required=False)
+#    date = forms.DateField(required=False)
 
-    caves = [cave.slug for cave in Cave.objects.all()]
+#    caves = [cave.slug for cave in Cave.objects.all()]
-    caves.sort()
+#    caves.sort()
-    caves = ["-----"] + caves
+#    caves = ["-----"] + caves
-    cave = forms.ChoiceField([(c, c) for c in caves], required=False)
+#    cave = forms.ChoiceField([(c, c) for c in caves], required=False)
 
-    entrance = forms.ChoiceField([("-----", "Please select a cave"), ], required=False) 
+#    entrance = forms.ChoiceField([("-----", "Please select a cave"), ], required=False) 
-    qm = forms.ChoiceField([("-----", "Please select a cave"), ], required=False) 
+#    qm = forms.ChoiceField([("-----", "Please select a cave"), ], required=False) 
 
-    expeditions = [e.year for e in Expedition.objects.all()]
+#    expeditions = [e.year for e in Expedition.objects.all()]
-    expeditions.sort()
+#    expeditions.sort()
-    expeditions = ["-----"] + expeditions
+#    expeditions = ["-----"] + expeditions
-    expedition = forms.ChoiceField([(e, e) for e in expeditions], required=False)  
+#    expedition = forms.ChoiceField([(e, e) for e in expeditions], required=False)  
 
-    logbookentry = forms.ChoiceField([("-----", "Please select an expedition"), ], required=False) 
+#    logbookentry = forms.ChoiceField([("-----", "Please select an expedition"), ], required=False) 
 
-    person = forms.ChoiceField([("-----", "Please select an expedition"), ], required=False) 
+#    person = forms.ChoiceField([("-----", "Please select an expedition"), ], required=False) 
     
-    survey_point = forms.CharField()
+#    survey_point = forms.CharField()
 
 

core/management/commands/reset_db.py

@@ -0,0 +1,183 @@
+from django.core.management.base import BaseCommand, CommandError
+from optparse import make_option
+from troggle.core.models import Cave
+import settings
+import os
+
+from django.db import connection
+from django.core import management
+from django.contrib.auth.models import User
+from django.core.urlresolvers import reverse
+from troggle.core.models import Cave, Entrance
+import troggle.flatpages.models
+
+databasename=settings.DATABASES['default']['NAME']
+expouser=settings.EXPOUSER
+expouserpass=settings.EXPOUSERPASS
+expouseremail=settings.EXPOUSER_EMAIL
+
+class Command(BaseCommand):
+    help = 'This is normal usage, clear database and reread everything'
+
+    option_list = BaseCommand.option_list + (
+        make_option('--reset',
+                    action='store_true',
+                    dest='reset',
+                    default=False,
+                    help='Reset the entier DB from files'),
+    )
+
+    def handle(self, *args, **options):
+        print(args)
+        print(options)
+        if "desc" in args:
+            self.resetdesc()
+        elif "scans" in args:
+            self.import_surveyscans()
+        elif "caves" in args:
+            self.reload_db()
+            self.make_dirs()
+            self.pageredirects()
+            self.import_caves()
+        elif "people" in args:
+            self.import_people()
+        elif "QMs" in args:
+            self.import_QMs()
+        elif "tunnel" in args:
+            self.import_tunnelfiles()
+        elif options['reset']:
+            self.reset(self)
+        elif "survex" in args:
+            self.import_survex()
+        elif "survexpos" in args:
+            import parsers.survex
+            parsers.survex.LoadPos()
+        elif "logbooks" in args:
+            self.import_logbooks()
+        elif "autologbooks" in args:
+            self.import_auto_logbooks()
+        elif "dumplogbooks" in args:
+            self.dumplogbooks()
+        elif "writeCaves" in args:
+            self.writeCaves()
+        elif options['foo']:
+            self.stdout.write(self.style.WARNING('Tesing....'))
+        else:
+            #self.stdout.write("%s not recognised" % args)
+            #self.usage(options)
+            self.stdout.write("poo")
+            #print(args)
+
+    def reload_db(obj):
+        if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
+            try:
+                os.remove(databasename)
+            except OSError:
+                pass
+        else:
+            cursor = connection.cursor()
+            cursor.execute("DROP DATABASE %s" % databasename)
+            cursor.execute("CREATE DATABASE %s" % databasename)
+            cursor.execute("ALTER DATABASE %s CHARACTER SET=utf8" % databasename)
+            cursor.execute("USE %s" % databasename)
+        management.call_command('migrate', interactive=False)
+        # management.call_command('syncdb', interactive=False)
+        user = User.objects.create_user(expouser, expouseremail, expouserpass)
+        user.is_staff = True
+        user.is_superuser = True
+        user.save()
+
+    def make_dirs(obj):
+        """Make directories that troggle requires"""
+        # should also deal with permissions here.
+        if not os.path.isdir(settings.PHOTOS_ROOT):
+            os.mkdir(settings.PHOTOS_ROOT)
+
+    def import_caves(obj):
+        import parsers.caves
+        print("Importing Caves")
+        parsers.caves.readcaves()
+
+    def import_people(obj):
+        import parsers.people
+        parsers.people.LoadPersonsExpos()
+
+    def import_logbooks(obj):
+        # The below line was causing errors I didn't understand (it said LOGFILE was a string), and I couldn't be bothered to figure
+        # what was going on so I just catch the error with a try. - AC 21 May
+        try:
+            settings.LOGFILE.write('\nBegun importing logbooks at ' + time.asctime() + '\n' + '-' * 60)
+        except:
+            pass
+
+        import parsers.logbooks
+        parsers.logbooks.LoadLogbooks()
+
+    def import_survex(obj):
+        import parsers.survex
+        parsers.survex.LoadAllSurvexBlocks()
+        parsers.survex.LoadPos()
+
+    def import_QMs(obj):
+        import parsers.QMs
+
+    def import_surveys(obj):
+        import parsers.surveys
+        parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
+
+    def import_surveyscans(obj):
+        import parsers.surveys
+        parsers.surveys.LoadListScans()
+
+    def import_tunnelfiles(obj):
+        import parsers.surveys
+        parsers.surveys.LoadTunnelFiles()
+
+    def reset(self, mgmt_obj):
+        """ Wipe the troggle database and import everything from legacy data
+        """
+        self.reload_db()
+        self.make_dirs()
+        self.pageredirects()
+        self.import_caves()
+        self.import_people()
+        self.import_surveyscans()
+        self.import_survex()
+        self.import_logbooks()
+        self.import_QMs()
+        try:
+            self.import_tunnelfiles()
+        except:
+            print("Tunnel files parser broken.")
+
+        self.import_surveys()
+
+    def pageredirects(obj):
+        for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
+            f = troggle.flatpages.models.Redirect(originalURL=oldURL, newURL=newURL)
+            f.save()
+
+    def writeCaves(obj):
+        for cave in Cave.objects.all():
+            cave.writeDataFile()
+        for entrance in Entrance.objects.all():
+            entrance.writeDataFile()
+
+    def troggle_usage(obj):
+        print("""Usage is 'manage.py reset_db <command>'
+                 where command is:
+                 reset - this is normal usage, clear database and reread everything
+                 desc
+                 caves - read in the caves
+                 logbooks - read in the logbooks
+                 autologbooks
+                 dumplogbooks
+                 people
+                 QMs      - read in the QM files
+                 resetend
+                 scans    - read in the scanned surveynotes
+                 survex   - read in the survex files
+                 survexpos
+                 tunnel   - read in the Tunnel files
+                 writeCaves
+                 """)

core/models.py

@@ -30,7 +30,7 @@ def get_related_by_wikilinks(wiki_text):
                               number = qmdict['number'])
             res.append(qm)         
         except QM.DoesNotExist:
-            print 'fail on '+str(wikilink)
+            print('fail on '+str(wikilink))
     
     return res
 
@@ -57,7 +57,7 @@ class TroggleModel(models.Model):
     class Meta:
         abstract = True
 
-class TroggleImageModel(ImageModel):
+class TroggleImageModel(models.Model):
     new_since_parsing = models.BooleanField(default=False, editable=False)
     
     def object_name(self):
@@ -104,44 +104,38 @@ class Expedition(TroggleModel):
     def day_max(self):
         res = self.expeditionday_set.all()
         return res and res[len(res) - 1] or None
-        
+
-        
 
 class ExpeditionDay(TroggleModel):
     expedition  = models.ForeignKey("Expedition")
     date        = models.DateField()
 
     class Meta:
-        ordering = ('date',)  
+        ordering = ('date',)
 
     def GetPersonTrip(self, personexpedition):
         personexpeditions = self.persontrip_set.filter(expeditionday=self)
         return personexpeditions and personexpeditions[0] or None
-    
+
-        
 #
 # single Person, can go on many years
 #
 class Person(TroggleModel):
     first_name  = models.CharField(max_length=100)
     last_name   = models.CharField(max_length=100)
+    fullname    = models.CharField(max_length=200)
     is_vfho     = models.BooleanField(help_text="VFHO is the Vereines für Höhlenkunde in Obersteier, a nearby Austrian caving club.", default=False)
     mug_shot    = models.CharField(max_length=100, blank=True,null=True)
     blurb = models.TextField(blank=True,null=True)
     
     #href        = models.CharField(max_length=200)
     orderref    = models.CharField(max_length=200)  # for alphabetic 
-    
-    #the below have been removed and made methods. I'm not sure what the b in bisnotable stands for. - AC 16 Feb
-    #notability  = models.FloatField()               # for listing the top 20 people
-    #bisnotable  = models.BooleanField(default=False)
     user	= models.OneToOneField(User, null=True, blank=True)
     def get_absolute_url(self):
         return urlparse.urljoin(settings.URL_ROOT,reverse('person',kwargs={'first_name':self.first_name,'last_name':self.last_name}))
 
     class Meta:
         verbose_name_plural = "People"
-    class Meta:
         ordering = ('orderref',)  # "Wookey" makes too complex for: ('last_name', 'first_name') 
     
     def __unicode__(self):
@@ -152,9 +146,15 @@ class Person(TroggleModel):
     
     def notability(self):
         notability = Decimal(0)
+        max_expo_val = 0
+
+        max_expo_year = Expedition.objects.all().aggregate(Max('year'))
+        max_expo_val = int(max_expo_year['year__max']) + 1
+
         for personexpedition in self.personexpedition_set.all():
              if not personexpedition.is_guest:
-                notability += Decimal(1) / (2012 - int(personexpedition.expedition.year))
+                print(personexpedition.expedition.year)
+                notability += Decimal(1) / (max_expo_val - int(personexpedition.expedition.year))
         return notability
 
     def bisnotable(self):
@@ -242,18 +242,22 @@ class PersonExpedition(TroggleModel):
 # Single parsed entry from Logbook
 #    
 class LogbookEntry(TroggleModel):
-    date    = models.DateField()#MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.
+
+    LOGBOOK_ENTRY_TYPES = (
+        ("wiki", "Wiki style logbook"),
+        ("html", "Html style logbook")
+    )
+
+    date    = models.DateField()#MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.ld()
     expeditionday = models.ForeignKey("ExpeditionDay", null=True)#MJG wants to KILL THIS (redundant information)
     expedition  = models.ForeignKey(Expedition,blank=True,null=True)  # yes this is double-
-    #author  = models.ForeignKey(PersonExpedition,blank=True,null=True)  # the person who writes it up doesn't have to have been on the trip.
+    title      = models.CharField(max_length=settings.MAX_LOGBOOK_ENTRY_TITLE_LENGTH)
-    # Re: the above- so this field should be "typist" or something, not "author". - AC 15 jun 09
+    cave_slug  = models.SlugField(max_length=50)
-    #MJG wants to KILL THIS, as it is typically redundant with PersonTrip.is_logbook_entry_author, in the rare it was not redundanty and of actually interest it could be added to the text.
+    place      = models.CharField(max_length=100,blank=True,null=True,help_text="Only use this if you haven't chosen a cave")
-    title     = models.CharField(max_length=settings.MAX_LOGBOOK_ENTRY_TITLE_LENGTH)
+    text       = models.TextField()
-    cave_slug = models.SlugField(max_length=50)
+    slug       = models.SlugField(max_length=50)
-    place     = models.CharField(max_length=100,blank=True,null=True,help_text="Only use this if you haven't chosen a cave")
+    filename   = models.CharField(max_length=200,null=True)
-    text      = models.TextField()
+    entry_type = models.CharField(default="wiki",null=True,choices=LOGBOOK_ENTRY_TYPES,max_length=50)
-    slug      = models.SlugField(max_length=50)
-    filename  = models.CharField(max_length=200,null=True)
 
     class Meta:
         verbose_name_plural = "Logbook Entries"
@@ -292,7 +296,7 @@ class LogbookEntry(TroggleModel):
         if self.cave:
             nextQMnumber=self.cave.new_QM_number(self.date.year)
         else:
-            return none
+            return None
         return nextQMnumber
 
     def new_QM_found_link(self):
@@ -302,6 +306,7 @@ class LogbookEntry(TroggleModel):
     def DayIndex(self):
         return list(self.expeditionday.logbookentry_set.all()).index(self)
 
+
 #
 # Single Person going on a trip, which may or may not be written up (accounts for different T/U for people in same logbook entry)
 #
@@ -398,6 +403,7 @@ class Cave(TroggleModel):
     url = models.CharField(max_length=200,blank=True,null=True)
     filename = models.CharField(max_length=200)
 
+
     #class Meta:
     #    unique_together = (("area", "kataster_number"), ("area", "unofficial_number"))
     # FIXME Kataster Areas and CUCC defined sub areas need seperating    
@@ -446,7 +452,7 @@ class Cave(TroggleModel):
         elif self.unofficial_number:
             href = self.unofficial_number
         else:
-            href = official_name.lower()
+            href = self.official_name.lower()
         #return settings.URL_ROOT + '/cave/' + href + '/'
         return urlparse.urljoin(settings.URL_ROOT, reverse('cave',kwargs={'cave_id':href,}))
 
@@ -529,13 +535,15 @@ class Cave(TroggleModel):
 
 def getCaveByReference(reference):
     areaname, code = reference.split("-", 1)
-    print  areaname, code
+    #print(areaname, code)
     area = Area.objects.get(short_name = areaname)
-    print area
+    #print(area)
-    foundCaves = list(Cave.objects.filter(area = area,  kataster_number = code).all()) + list(Cave.objects.filter(area = area,  unofficial_number = code).all()) 
+    foundCaves = list(Cave.objects.filter(area = area,  kataster_number = code).all()) + list(Cave.objects.filter(area = area,  unofficial_number = code).all())
-    print list(foundCaves)
+    print(list(foundCaves))
-    assert len(foundCaves) == 1
+    if len(foundCaves) == 1:
-    return foundCaves[0]
+        return foundCaves[0]
+    else:
+        return False
 
 class OtherCaveName(TroggleModel):
     name = models.CharField(max_length=160)
@@ -738,17 +746,17 @@ class QM(TroggleModel):
 
     number = models.IntegerField(help_text="this is the sequential number in the year", )
     GRADE_CHOICES=(
-	('A', 'A: Large obvious lead'),
+    ('A', 'A: Large obvious lead'),
-	('B', 'B: Average lead'),
+    ('B', 'B: Average lead'),
-	('C', 'C: Tight unpromising lead'),
+    ('C', 'C: Tight unpromising lead'),
-	('D', 'D: Dig'),
+    ('D', 'D: Dig'),
-	('X', 'X: Unclimbable aven')
+    ('X', 'X: Unclimbable aven')
     )
     grade = models.CharField(max_length=1, choices=GRADE_CHOICES)
     location_description = models.TextField(blank=True)
-    #should be a foreignkey to surveystation
     nearest_station_description = models.CharField(max_length=400,null=True,blank=True)
-    nearest_station = models.CharField(max_length=200,blank=True,null=True)
+    nearest_station_name = models.CharField(max_length=200,blank=True,null=True)
+    nearest_station = models.ForeignKey(SurvexStation,null=True,blank=True)
     area = models.CharField(max_length=100,blank=True,null=True)
     completion_description = models.TextField(blank=True,null=True)
     comment=models.TextField(blank=True,null=True)
@@ -801,9 +809,11 @@ class DPhoto(TroggleImageModel):
 scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL)
 def get_scan_path(instance, filename):
     year=instance.survey.expedition.year
-    #print "WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number
+    #print("WN: ", type(instance.survey.wallet_number), instance.survey.wallet_number, instance.survey.wallet_letter)
-    number=str(instance.survey.wallet_number) + str(instance.survey.wallet_letter) #two strings formatting because convention is 2009#01 or 2009#X01
+    number=str(instance.survey.wallet_number)
-    return os.path.join('./',year,year+r'#'+number,instance.contents+str(instance.number_in_wallet)+r'.jpg')
+    if str(instance.survey.wallet_letter) != "None":
+        number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01
+    return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg')
 
 class ScannedImage(TroggleImageModel): 
     file = models.ImageField(storage=scansFileStorage, upload_to=get_scan_path)
@@ -826,7 +836,7 @@ class ScannedImage(TroggleImageModel):
     #This is an ugly hack to deal with the #s in our survey scan paths. The correct thing is to write a custom file storage backend which calls urlencode on the name for making file.url but not file.path.
     def correctURL(self):
         return string.replace(self.file.url,r'#',r'%23')
-    
+
     def __unicode__(self):
         return get_scan_path(self,'')
 
@@ -859,3 +869,14 @@ class Survey(TroggleModel):
 
     def elevations(self):
         return self.scannedimage_set.filter(contents='elevation')
+
+class DataIssue(TroggleModel):
+    date = models.DateTimeField(auto_now_add=True, blank=True)
+    parser = models.CharField(max_length=50, blank=True, null=True)
+    message = models.CharField(max_length=400, blank=True, null=True)
+
+    class Meta:
+        ordering = ['date']
+
+    def __unicode__(self):
+        return u"%s - %s" % (self.parser, self.message)

core/models_survex.py

@@ -97,7 +97,7 @@ class SurvexBlockLookUpManager(models.Manager):
             blocknames = []
         else:
             blocknames = name.split(".")
-        block = SurvexBlock.objects.get(parent=None, survexfile__path="all")
+        block = SurvexBlock.objects.get(parent=None, survexfile__path=settings.SURVEX_TOPNAME)
         for blockname in blocknames:
             block = SurvexBlock.objects.get(parent=block, name__iexact=blockname)
         return block
@@ -225,4 +225,4 @@ class TunnelFile(models.Model):
     
     class Meta:
         ordering = ('tunnelpath',)
-    
+

core/templatetags/survex_markup.py

@@ -47,6 +47,6 @@ def survex_to_html(value, autoescape=None):
     if autoescape:
         value = conditional_escape(value)
     for regex, sub in regexes:
-        print sub
+        print(sub)
         value = regex.sub(sub, value)
     return mark_safe(value)

core/templatetags/wiki_markup.py

@@ -7,7 +7,6 @@ from troggle.core.models import QM, DPhoto, LogbookEntry, Cave
 import re, urlparse
 
 register = template.Library()
-                
 
 @register.filter()
 def plusone(n):
@@ -77,7 +76,7 @@ def wiki_to_html_short(value, autoescape=None):
         if number>1:
             return '<h'+num+'>'+matchobj.groups()[1]+'</h'+num+'>'
         else:
-            print 'morethanone'
+            print('morethanone')
             return matchobj.group()
     value = re.sub(r"(?m)^(=+)([^=]+)(=+)$",headerrepl,value)
     
@@ -143,13 +142,13 @@ def wiki_to_html_short(value, autoescape=None):
     value = re.sub(photoSrcPattern,photoSrcRepl, value, re.DOTALL)
 
     #make cave links
-    value = re.sub("\[\[\s*cave:([^\s]+)\s*\s*\]\]", r'<a href="%scave/\1/">\1</a>' % settings.URL_ROOT, value, re.DOTALL)
+    value = re.sub(r"\[\[\s*cave:([^\s]+)\s*\s*\]\]", r'<a href="%scave/\1/">\1</a>' % settings.URL_ROOT, value, re.DOTALL)
     #make people links
-    value = re.sub("\[\[\s*person:(.+)\|(.+)\]\]",r'<a href="%sperson/\1/">\2</a>' % settings.URL_ROOT, value, re.DOTALL)
+    value = re.sub(r"\[\[\s*person:(.+)\|(.+)\]\]",r'<a href="%sperson/\1/">\2</a>' % settings.URL_ROOT, value, re.DOTALL)
     #make subcave links
-    value = re.sub("\[\[\s*subcave:(.+)\|(.+)\]\]",r'<a href="%ssubcave/\1/">\2</a>' % settings.URL_ROOT, value, re.DOTALL)
+    value = re.sub(r"\[\[\s*subcave:(.+)\|(.+)\]\]",r'<a href="%ssubcave/\1/">\2</a>' % settings.URL_ROOT, value, re.DOTALL)
     #make cavedescription links
-    value = re.sub("\[\[\s*cavedescription:(.+)\|(.+)\]\]",r'<a href="%scavedescription/\1/">\2</a>' % settings.URL_ROOT, value, re.DOTALL)
+    value = re.sub(r"\[\[\s*cavedescription:(.+)\|(.+)\]\]",r'<a href="%scavedescription/\1/">\2</a>' % settings.URL_ROOT, value, re.DOTALL)
 
 
 

core/view_surveys.py

@@ -6,6 +6,7 @@ import os, stat
 import re
 from troggle.core.models import SurvexScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile
 import parsers.surveys
+import urllib
 
 # inline fileabstraction into here if it's not going to be useful anywhere else 
 # keep things simple and ignore exceptions everywhere for now
@@ -17,13 +18,13 @@ def getMimeType(extension):
                 "html": "text/html",
                 }[extension]
     except:
-        print "unknown file type"
+        print("unknown file type")
         return "text/plain"
 
 
 def listdir(request, path):
     #try:
-        return HttpResponse(fileAbstraction.listdir(path), mimetype = "text/plain")
+        return HttpResponse(fileAbstraction.listdir(path), content_type="text/plain")
     #except:
     #    raise Http404
 
@@ -33,7 +34,7 @@ def upload(request, path):
 def download(request, path):
     #try:
         
-        return HttpResponse(fileAbstraction.readFile(path), mimetype=getMimeType(path.split(".")[-1]))
+        return HttpResponse(fileAbstraction.readFile(path), content_type=getMimeType(path.split(".")[-1]))
     #except:
     #    raise Http404
 
@@ -46,6 +47,7 @@ extmimetypes = {".txt": "text/plain",
              ".html": "text/html",
              ".png": "image/png",
              ".jpg": "image/jpeg",
+             ".jpeg": "image/jpeg",
             }
             
 # dead
@@ -93,9 +95,9 @@ def jgtfile(request, f):
         fin = open(fp)
         ftext = fin.read()
         fin.close()
-        return HttpResponse(ftext, mimetype=mimetype)
+        return HttpResponse(ftext, content_type=mimetype)
 
-    return HttpResponse("unknown file::%s::" % f, mimetype = "text/plain")
+    return HttpResponse("unknown file::%s::" % f, content_type = "text/plain")
 
 
 def UniqueFile(fname):
@@ -165,13 +167,13 @@ def jgtuploadfile(request):
 
 def surveyscansfolder(request, path):
     #print [ s.walletname  for s in SurvexScansFolder.objects.all() ]
-    survexscansfolder = SurvexScansFolder.objects.get(walletname=path)
+    survexscansfolder = SurvexScansFolder.objects.get(walletname=urllib.unquote(path))
     return render_to_response('survexscansfolder.html', { 'survexscansfolder':survexscansfolder, 'settings': settings })
 
 def surveyscansingle(request, path, file):
-    survexscansfolder = SurvexScansFolder.objects.get(walletname=path)
+    survexscansfolder = SurvexScansFolder.objects.get(walletname=urllib.unquote(path))
     survexscansingle = SurvexScanSingle.objects.get(survexscansfolder=survexscansfolder, name=file)
-    return HttpResponse(content=open(survexscansingle.ffile), mimetype="image/png")
+    return HttpResponse(content=open(survexscansingle.ffile), content_type=getMimeType(path.split(".")[-1]))
     #return render_to_response('survexscansfolder.html', { 'survexscansfolder':survexscansfolder, 'settings': settings })
     
 def surveyscansfolders(request):
@@ -185,12 +187,12 @@ def tunneldata(request):
     
 
 def tunnelfile(request, path):
-    tunnelfile = TunnelFile.objects.get(tunnelpath=path)
+    tunnelfile = TunnelFile.objects.get(tunnelpath=urllib.unquote(path))
     tfile = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
-    return HttpResponse(content=open(tfile), mimetype="text/plain")
+    return HttpResponse(content=open(tfile), content_type="text/plain")
     
 def tunnelfileupload(request, path):
-    tunnelfile = TunnelFile.objects.get(tunnelpath=path)
+    tunnelfile = TunnelFile.objects.get(tunnelpath=urllib.unquote(path))
     tfile = os.path.join(settings.TUNNEL_DATA, tunnelfile.tunnelpath)
     
     project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"]
@@ -202,13 +204,13 @@ def tunnelfileupload(request, path):
     uploadedfile = request.FILES.values()[0]
         
     if uploadedfile.field_name != "sketch":
-        return HttpResponse(content="Error: non-sketch file uploaded", mimetype="text/plain")
+        return HttpResponse(content="Error: non-sketch file uploaded", content_type="text/plain")
     if uploadedfile.content_type != "text/plain":
-        return HttpResponse(content="Error: non-plain content type", mimetype="text/plain")
+        return HttpResponse(content="Error: non-plain content type", content_type="text/plain")
         
     # could use this to add new files
     if os.path.split(path)[1] != uploadedfile.name:  
-        return HttpResponse(content="Error: name disagrees", mimetype="text/plain")
+        return HttpResponse(content="Error: name disagrees", content_type="text/plain")
     
     orgsize = tunnelfile.filesize   # = os.stat(tfile)[stat.ST_SIZE]
         
@@ -226,7 +228,7 @@ def tunnelfileupload(request, path):
     
     uploadedfile.close()
     message = "File size %d overwritten with size %d" % (orgsize, tunnelfile.filesize)
-    return HttpResponse(content=message, mimetype="text/plain")
+    return HttpResponse(content=message, content_type="text/plain")
 
     
     

core/views_caves.py

@@ -10,15 +10,15 @@ from troggle.helper import login_required_if_public
 from django.forms.models import modelformset_factory
 from django import forms
 from django.core.urlresolvers import reverse
-from utils import render_with_context # see views_logbooks for explanation on this.
 from django.http import HttpResponse, HttpResponseRedirect
 from django.conf import settings
 import re, urlparse
-from django.shortcuts import get_object_or_404
+from django.shortcuts import get_object_or_404, render
 import settings
 
+
 from PIL import Image, ImageDraw, ImageFont
-import string, os, sys
+import string, os, sys, subprocess
 
 def getCave(cave_id):
     """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm."""
@@ -57,48 +57,68 @@ def caveindex(request):
     caves1626 = list(Cave.objects.filter(area__short_name = "1626"))
     caves1623.sort(caveCmp)
     caves1626.sort(caveCmp)
-    return render_with_context(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':notablecaves, 'cavepage': True})
+    return render(request,'caveindex.html', {'caves1623': caves1623, 'caves1626': caves1626, 'notablecaves':notablecaves, 'cavepage': True})
+
+def millenialcaves(request):
+    #RW messing around area
+    return HttpResponse("Test text", content_type="text/plain")
+    
+
+
+def cave3d(request, cave_id=''):
+    cave = getCave(cave_id)
+    survexfilename = settings.SURVEX_DATA + cave.survex_file
+    threedfilename = settings.THREEDCACHEDIR + '%s.3d' % cave_id
+    if True or os.path.getmtime(survexfilename) > os.path.getmtime(threedfilename):
+        subprocess.call(["cavern", "--output=%s" % threedfilename, survexfilename])
+    test_file = open(threedfilename, 'rb')
+    response = HttpResponse(content=test_file, content_type='application/3d')#mimetype is replaced by content_type for django 1.7
+    response['Content-Disposition'] = 'attachment; filename=%s.3d' % cave_id
+    # response['X-Sendfile'] = "%s.3d" % cave_id
+    # It's usually a good idea to set the 'Content-Length' header too.
+    # You can also set any other required headers: Cache-Control, etc.
+    return response
 
 def cave(request, cave_id='', offical_name=''):
     cave=getCave(cave_id)
     if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
-        return render_with_context(request,'nonpublic.html', {'instance': cave, 'cavepage': True})
+        return render(request,'nonpublic.html', {'instance': cave, 'cavepage': True, 'cave_id': cave_id})
     else:
-        return render_with_context(request,'cave.html', {'settings': settings, 'cave': cave, 'cavepage': True})
+        return render(request,'cave.html', {'settings': settings, 'cave': cave, 'cavepage': True, 'cave_id': cave_id})
 
 def caveEntrance(request, slug):
     cave = Cave.objects.get(caveslug__slug = slug)
     if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
-        return render_with_context(request,'nonpublic.html', {'instance': cave})
+        return render(request,'nonpublic.html', {'instance': cave})
     else:
-        return render_with_context(request,'cave_entrances.html', {'cave': cave})
+        return render(request,'cave_entrances.html', {'cave': cave})
 
 def caveDescription(request, slug):
     cave = Cave.objects.get(caveslug__slug = slug)
     if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
-        return render_with_context(request,'nonpublic.html', {'instance': cave})
+        return render(request,'nonpublic.html', {'instance': cave})
     else:
-        return render_with_context(request,'cave_uground_description.html', {'cave': cave})
+        return render(request,'cave_uground_description.html', {'cave': cave})
 
 def caveQMs(request, slug):
     cave = Cave.objects.get(caveslug__slug = slug)
     if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
-        return render_with_context(request,'nonpublic.html', {'instance': cave})
+        return render(request,'nonpublic.html', {'instance': cave})
     else:
-        return render_with_context(request,'cave_qms.html', {'cave': cave})
+        return render(request,'cave_qms.html', {'cave': cave})
 def caveLogbook(request, slug):
     cave = Cave.objects.get(caveslug__slug = slug)
     if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
-        return render_with_context(request,'nonpublic.html', {'instance': cave})
+        return render(request,'nonpublic.html', {'instance': cave})
     else:
-        return render_with_context(request,'cave_logbook.html', {'cave': cave})
+        return render(request,'cave_logbook.html', {'cave': cave})
 
 def caveSlug(request, slug):
     cave = Cave.objects.get(caveslug__slug = slug)
     if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated():
-        return render_with_context(request,'nonpublic.html', {'instance': cave, 'cave_editable': slug})
+        return render(request,'nonpublic.html', {'instance': cave, 'cave_editable': slug})
     else:
-        return render_with_context(request,'cave.html', {'cave': cave, 'cave_editable': slug})
+        return render(request,'cave.html', {'cave': cave, 'cave_editable': slug})
 
 @login_required_if_public
 def edit_cave(request, slug=None):
@@ -139,7 +159,7 @@ def edit_cave(request, slug=None):
         ceFormSet = CaveAndEntranceFormSet(queryset=cave.caveandentrance_set.all())
         versionControlForm = VersionControlCommentForm()
         
-    return render_with_context(request, 
+    return render(request, 
                                'editcave2.html', 
                                {'form': form,
                                 'caveAndEntranceFormSet': ceFormSet,
@@ -183,7 +203,7 @@ def editEntrance(request, caveslug, slug=None):
             entletter = EntranceLetterForm(request.POST)
         else:
             entletter = None
-    return render_with_context(request, 
+    return render(request, 
                                'editentrance.html', 
                                {'form': form,
                                 'versionControlForm': versionControlForm,
@@ -194,7 +214,7 @@ def qm(request,cave_id,qm_id,year,grade=None):
     year=int(year)
     try:
         qm=getCave(cave_id).get_QMs().get(number=qm_id,found_by__date__year=year)
-        return render_with_context(request,'qm.html',locals())
+        return render(request,'qm.html',locals())
 
     except QM.DoesNotExist:
         url=urlparse.urljoin(settings.URL_ROOT, r'/admin/core/qm/add/'+'?'+  r'number=' + qm_id)
@@ -203,20 +223,20 @@ def qm(request,cave_id,qm_id,year,grade=None):
         return HttpResponseRedirect(url)
     
 
-
+ 
 def ent(request, cave_id, ent_letter):
     cave = Cave.objects.filter(kataster_number = cave_id)[0]
     cave_and_ent = CaveAndEntrance.objects.filter(cave = cave).filter(entrance_letter = ent_letter)[0]
-    return render_with_context(request,'entrance.html', {'cave': cave,
+    return render(request,'entrance.html', {'cave': cave,
                                                 'entrance': cave_and_ent.entrance,
                                                 'letter': cave_and_ent.entrance_letter,})
 
 def entranceSlug(request, slug):
     entrance = Entrance.objects.get(entranceslug__slug = slug)
     if entrance.non_public and not request.user.is_authenticated():
-        return render_with_context(request,'nonpublic.html', {'instance': entrance})
+        return render(request,'nonpublic.html', {'instance': entrance})
     else:
-        return render_with_context(request,'entranceslug.html', {'entrance': entrance})
+        return render(request,'entranceslug.html', {'entrance': entrance})
 
 def survexblock(request, survexpath):
     survexpath = re.sub("/", ".", survexpath)
@@ -224,12 +244,12 @@ def survexblock(request, survexpath):
     survexblock = models.SurvexBlock.objects.get(survexpath=survexpath)
     #ftext = survexblock.filecontents()
     ftext = survexblock.text
-    return render_with_context(request,'survexblock.html', {'survexblock':survexblock, 'ftext':ftext, })
+    return render(request,'survexblock.html', {'survexblock':survexblock, 'ftext':ftext, })
 
 def surveyindex(request):
     surveys=Survey.objects.all()
     expeditions=Expedition.objects.order_by("-year")
-    return render_with_context(request,'survey.html',locals())
+    return render(request,'survey.html',locals())
 
 def survey(request,year,wallet_number):
     surveys=Survey.objects.all()
@@ -242,19 +262,19 @@ def survey(request,year,wallet_number):
             planSketches=current_survey.scannedimage_set.filter(contents='plan')
             elevationSketches=current_survey.scannedimage_set.filter(contents='elevation')
 
-    return render_with_context(request,'survey.html', locals())
+    return render(request,'survey.html', locals())
 
 def cave_description(request, cavedescription_name):
     cave_description = get_object_or_404(CaveDescription, short_name = cavedescription_name)
-    return render_with_context(request,'cave_description.html', locals())
+    return render(request,'cave_description.html', locals())
 
 def get_entrances(request, caveslug):
     cave = Cave.objects.get(caveslug__slug = caveslug)
-    return render_with_context(request,'options.html', {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]})
+    return render(request,'options.html', {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]})
 
 def get_qms(request, caveslug):
     cave = Cave.objects.get(caveslug__slug = caveslug)
-    return render_with_context(request,'options.html', {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]})
+    return render(request,'options.html', {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]})
 
 areanames = [
 	#('',     'Location unclear'),
@@ -292,7 +312,7 @@ def prospecting(request):
         caves = list(a.cave_set.all())
         caves.sort(caveCmp)
         areas.append((name, a, caves))
-    return render_with_context(request, 'prospecting.html', {"areas": areas})
+    return render(request, 'prospecting.html', {"areas": areas})
     
 # Parameters for big map and zoomed subarea maps:
 # big map first (zoom factor ignored)
@@ -320,8 +340,8 @@ for n in maps.keys():
     W = (R-L)/2
     H = (T-B)/2
     for i in range(2):
-	    for j in range(2):
+        for j in range(2):
-		    maps["%s%i%i" % (n, i, j)] = [L + i * W, T - j * H, L + (i + 1) * W, T - (j + 1) * H, S, name]
+            maps["%s%i%i" % (n, i, j)] = [L + i * W, T - j * H, L + (i + 1) * W, T - (j + 1) * H, S, name]
 # Keys in the order in which we want the maps output
 mapcodes = ["all", "grieß","40", "76", "204", "tc"] 
 # Field codes

core/views_logbooks.py

@@ -1,4 +1,4 @@
-from django.shortcuts import render_to_response
+from django.shortcuts import render_to_response, render
 from troggle.core.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry, SurvexBlock
 import troggle.core.models as models
 import troggle.settings as settings
@@ -9,16 +9,24 @@ from troggle.core.forms import getTripForm#, get_name, PersonForm
 from  django.core.urlresolvers import reverse
 from django.http import HttpResponseRedirect, HttpResponse
 from django.template import Context, loader
-from utils import render_with_context
 import os.path
 import troggle.parsers.logbooks as logbookparsers
 from django.template.defaultfilters import slugify
 from troggle.helper import login_required_if_public
 import datetime
 
+from django.views.generic.list import ListView
+from django.utils import timezone
 
-# Django uses Context, not RequestContext when you call render_to_response. We always want to use RequestContext, so that django adds the context from settings.TEMPLATE_CONTEXT_PROCESSORS. This way we automatically get necessary settings variables passed to each template. So we use a custom method, render_response instead of render_to_response. Hopefully future Django releases will make this unnecessary.
+
-#from troggle.alwaysUseRequestContext import render_response
+# Django uses Context, not RequestContext when you call render
+# to_response. We always want to use RequestContext, so that
+# django adds the context from settings.TEMPLATE_CONTEXT_PROCESSORS.
+# This way we automatically get necessary settings variables passed
+# to each template. So we use a custom method, render_response
+# instead of render_to_response. Hopefully future Django releases
+# will make this unnecessary.
+# from troggle.alwaysUseRequestContext import render_response
 
 import re
 
@@ -46,17 +54,17 @@ def personindex(request):
             if person.bisnotable():
                 notablepersons.append(person)
 
-    return render_with_context(request,'personindex.html', {'persons': persons, 'personss':personss, 'notablepersons':notablepersons, })
+    return render(request,'personindex.html', {'persons': persons, 'personss':personss, 'notablepersons':notablepersons})
 
 
 def expedition(request, expeditionname):
-    expedition = Expedition.objects.get(year=int(expeditionname))
+    this_expedition = Expedition.objects.get(year=int(expeditionname))
     expeditions = Expedition.objects.all()
     personexpeditiondays = [ ]
-    dateditems = list(expedition.logbookentry_set.all()) + list(expedition.survexblock_set.all())
+    dateditems = list(this_expedition.logbookentry_set.all()) + list(this_expedition.survexblock_set.all())
     dates = list(set([item.date for item in dateditems]))
     dates.sort()
-    for personexpedition in expedition.personexpedition_set.all():
+    for personexpedition in this_expedition.personexpedition_set.all():
         prow = [ ]
         for date in dates:
             pcell = { "persontrips": PersonTrip.objects.filter(personexpedition=personexpedition, 
@@ -66,32 +74,40 @@ def expedition(request, expeditionname):
             prow.append(pcell)
         personexpeditiondays.append({"personexpedition":personexpedition, "personrow":prow})
         
-    message = ""
     if "reload" in request.GET:
-        message = LoadLogbookForExpedition(expedition)
+        LoadLogbookForExpedition(this_expedition)
-    return render_with_context(request,'expedition.html', {'expedition': expedition, 'expeditions':expeditions, 'personexpeditiondays':personexpeditiondays, 'message':message, 'settings':settings, 'dateditems': dateditems })
+    return render(request,'expedition.html', {'expedition': this_expedition, 'expeditions':expeditions, 'personexpeditiondays':personexpeditiondays, 'settings':settings, 'dateditems': dateditems })
 
-    def get_absolute_url(self):
+def get_absolute_url(self):
-        return ('expedition', (expedition.year))
+    return ('expedition', (expedition.year))
+
+class ExpeditionListView(ListView):
+
+    model = Expedition
+
+    def get_context_data(self, **kwargs):
+        context = super(ExpeditionListView, self).get_context_data(**kwargs)
+        context['now'] = timezone.now()
+        return context
 
 
 def person(request, first_name='', last_name='', ):
-    person = Person.objects.get(first_name = first_name, last_name = last_name)
+    this_person = Person.objects.get(first_name = first_name, last_name = last_name)
     
-    #This is for removing the reference to the user's profile, in case they set it to the wrong person
+    # This is for removing the reference to the user's profile, in case they set it to the wrong person
     if request.method == 'GET':
         if request.GET.get('clear_profile')=='True':
-            person.user=None
+            this_person.user=None
-            person.save()
+            this_person.save()
             return HttpResponseRedirect(reverse('profiles_select_profile'))
     
-    return render_with_context(request,'person.html', {'person': person, })
+    return render(request,'person.html', {'person': this_person, })
 
 
 def GetPersonChronology(personexpedition):
     res = { }
     for persontrip in personexpedition.persontrip_set.all():
-        a = res.setdefault(persontrip.date, { })
+        a = res.setdefault(persontrip.logbook_entry.date, { })
         a.setdefault("persontrips", [ ]).append(persontrip)
 
     for personrole in personexpedition.survexpersonrole_set.all():
@@ -115,20 +131,20 @@ def GetPersonChronology(personexpedition):
 
 def personexpedition(request, first_name='',  last_name='', year=''):
     person = Person.objects.get(first_name = first_name, last_name = last_name)
-    expedition = Expedition.objects.get(year=year)
+    this_expedition = Expedition.objects.get(year=year)
-    personexpedition = person.personexpedition_set.get(expedition=expedition)
+    personexpedition = person.personexpedition_set.get(expedition=this_expedition)
     personchronology = GetPersonChronology(personexpedition)
-    return render_with_context(request,'personexpedition.html', {'personexpedition': personexpedition, 'personchronology':personchronology})
+    return render(request,'personexpedition.html', {'personexpedition': personexpedition, 'personchronology':personchronology})
 
 
 def logbookentry(request, date, slug):
-    logbookentry = LogbookEntry.objects.filter(date=date, slug=slug)
+    this_logbookentry = LogbookEntry.objects.filter(date=date, slug=slug)
 
-    if len(logbookentry)>1:
+    if len(this_logbookentry)>1:
-        return render_with_context(request, 'object_list.html',{'object_list':logbookentry})
+        return render(request, 'object_list.html',{'object_list':this_logbookentry})
     else:
-        logbookentry=logbookentry[0]
+        this_logbookentry=this_logbookentry[0]
-        return render_with_context(request, 'logbookentry.html', {'logbookentry': logbookentry})
+        return render(request, 'logbookentry.html', {'logbookentry': this_logbookentry})
 
 
 def logbookSearch(request, extra):
@@ -139,14 +155,14 @@ def logbookSearch(request, extra):
     entry_query = search.get_query(query_string, ['text','title',])
     found_entries = LogbookEntry.objects.filter(entry_query)
 
-    return render_with_context(request,'logbooksearch.html',
+    return render(request,'logbooksearch.html',
                           { 'query_string': query_string, 'found_entries': found_entries, })
                           #context_instance=RequestContext(request))
 
 def personForm(request,pk):
     person=Person.objects.get(pk=pk)
     form=PersonForm(instance=person)
-    return render_with_context(request,'personform.html', {'form':form,})
+    return render(request,'personform.html', {'form':form,})
 
 
 def experimental(request):
@@ -163,7 +179,7 @@ def experimental(request):
             
     survexlegs = models.SurvexLeg.objects.all()
     totalsurvexlength = sum([survexleg.tape  for survexleg in survexlegs])
-    return render_with_context(request, 'experimental.html', { "nsurvexlegs":len(survexlegs), "totalsurvexlength":totalsurvexlength, "legsbyexpo":legsbyexpo })
+    return render(request, 'experimental.html', { "nsurvexlegs":len(survexlegs), "totalsurvexlength":totalsurvexlength, "legsbyexpo":legsbyexpo })
 
 @login_required_if_public
 def newLogbookEntry(request, expeditionyear, pdate = None, pslug = None):
@@ -196,7 +212,7 @@ def newLogbookEntry(request, expeditionyear, pdate = None, pslug = None):
                                'expeditionyear': expeditionyear})
             f.write(template.render(context))
             f.close()
-            print logbookparsers.parseAutoLogBookEntry(filename)
+            print(logbookparsers.parseAutoLogBookEntry(filename))
             return HttpResponseRedirect(reverse('expedition', args=[expedition.year])) # Redirect after POST
     else:
         if pslug and pdate:
@@ -222,7 +238,7 @@ def newLogbookEntry(request, expeditionyear, pdate = None, pslug = None):
             tripForm = TripForm() # An unbound form
             personTripFormSet = PersonTripFormSet()
 
-    return render_with_context(request, 'newlogbookentry.html', {
+    return render(request, 'newlogbookentry.html', {
         'tripForm': tripForm,
         'personTripFormSet': personTripFormSet,
 
@@ -244,9 +260,8 @@ def delLogbookEntry(lbe):
 
 def get_people(request, expeditionslug):
     exp = Expedition.objects.get(year = expeditionslug)
-    return render_with_context(request,'options.html', {"items": [(pe.slug, pe.name) for pe in exp.personexpedition_set.all()]})
+    return render(request,'options.html', {"items": [(pe.slug, pe.name) for pe in exp.personexpedition_set.all()]})
 
 def get_logbook_entries(request, expeditionslug):
     exp = Expedition.objects.get(year = expeditionslug)
-    return render_with_context(request,'options.html', {"items": [(le.slug, "%s - %s" % (le.date, le.title)) for le in exp.logbookentry_set.all()]})
+    return render(request,'options.html', {"items": [(le.slug, "%s - %s" % (le.date, le.title)) for le in exp.logbookentry_set.all()]})
-    

core/views_other.py

@@ -1,14 +1,14 @@
 from troggle.core.models import Cave, Expedition, Person, LogbookEntry, PersonExpedition, PersonTrip, DPhoto, QM
-from troggle.core.forms import UploadFileForm
+#from troggle.core.forms import UploadFileForm
 from django.conf import settings
 from django import forms
 from django.template import loader, Context
 from django.db.models import Q
+from django.shortcuts import render
 import databaseReset
 import re
 from django.http import HttpResponse, HttpResponseRedirect
 from django.core.urlresolvers import reverse
-from utils import render_with_context
 from troggle.core.models import *
 from troggle.helper import login_required_if_public
 
@@ -21,18 +21,18 @@ def stats(request):
     statsDict['caveCount'] = int(Cave.objects.count())
     statsDict['personCount'] = int(Person.objects.count())
     statsDict['logbookEntryCount'] = int(LogbookEntry.objects.count())
-    return render_with_context(request,'statistics.html', statsDict)
+    return render(request,'statistics.html', statsDict)
 
 def frontpage(request):
     if request.user.is_authenticated():
-        return render_with_context(request,'tasks.html')
+        return render(request,'tasks.html')
 
     expeditions =  Expedition.objects.order_by("-year")
     logbookentry = LogbookEntry
     cave = Cave
     photo = DPhoto
     from django.contrib.admin.templatetags import log
-    return render_with_context(request,'frontpage.html', locals())
+    return render(request,'frontpage.html', locals())
 
 def todo(request):
     message = "no test message"  #reverse('personn', kwargs={"name":"hkjhjh"}) 
@@ -45,7 +45,7 @@ def todo(request):
 
     expeditions =  Expedition.objects.order_by("-year")
     totallogbookentries = LogbookEntry.objects.count()
-    return render_with_context(request,'index.html', {'expeditions':expeditions, 'all':'all', 'totallogbookentries':totallogbookentries, "message":message})
+    return render(request,'index.html', {'expeditions':expeditions, 'all':'all', 'totallogbookentries':totallogbookentries, "message":message})
 
 
 def controlPanel(request):
@@ -59,27 +59,27 @@ def controlPanel(request):
             databaseReset.make_dirs()
             for item in importlist:
                 if item in request.POST:
-                    print "running"+ " databaseReset."+item+"()"
+                    print("running"+ " databaseReset."+item+"()")
-                    exec "databaseReset."+item+"()"
+                    exec("databaseReset."+item+"()")
                     jobs_completed.append(item)
         else:
             if request.user.is_authenticated(): #The user is logged in, but is not a superuser.
-                return render_with_context(request,'controlPanel.html', {'caves':Cave.objects.all(),'error':'You must be a superuser to use that feature.'})
+                return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'error':'You must be a superuser to use that feature.'})
             else:
                 return HttpResponseRedirect(reverse('auth_login'))
 
-    return render_with_context(request,'controlPanel.html', {'caves':Cave.objects.all(),'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
+    return render(request,'controlPanel.html', {'caves':Cave.objects.all(),'expeditions':Expedition.objects.all(),'jobs_completed':jobs_completed})
 
 def downloadCavetab(request):
     from export import tocavetab
-    response = HttpResponse(mimetype='text/csv')
+    response = HttpResponse(content_type='text/csv')
     response['Content-Disposition'] = 'attachment; filename=CAVETAB2.CSV'
     tocavetab.writeCaveTab(response)
     return response
 
 def downloadSurveys(request):
     from export import tosurveys
-    response = HttpResponse(mimetype='text/csv')
+    response = HttpResponse(content_type='text/csv')
     response['Content-Disposition'] = 'attachment; filename=Surveys.csv'
     tosurveys.writeCaveTab(response)
     return response
@@ -87,27 +87,26 @@ def downloadSurveys(request):
 def downloadLogbook(request,year=None,extension=None,queryset=None):
     
     if year:
-        expedition=Expedition.objects.get(year=year)
+        current_expedition=Expedition.objects.get(year=year)
-        logbook_entries=LogbookEntry.objects.filter(expedition=expedition)
+        logbook_entries=LogbookEntry.objects.filter(expedition=current_expedition)
         filename='logbook'+year
     elif queryset:
         logbook_entries=queryset
         filename='logbook'
     else:
+        response = HttpResponse(content_type='text/plain')
         return response(r"Error: Logbook downloader doesn't know what year you want")
     
     if 'year' in request.GET:
         year=request.GET['year']
     if 'extension' in request.GET:
         extension=request.GET['extension']
-    
+
-    
-    
     if extension =='txt':
-        response = HttpResponse(mimetype='text/plain')
+        response = HttpResponse(content_type='text/plain')
         style='2008'
     elif extension == 'html':
-        response = HttpResponse(mimetype='text/html')
+        response = HttpResponse(content_type='text/html')
         style='2005'
         
     template='logbook'+style+'style.'+extension
@@ -124,11 +123,11 @@ def downloadQMs(request):
         try:
             cave=Cave.objects.get(kataster_number=request.GET['cave_id'])
         except Cave.DoesNotExist:
-            cave=Cave.objects.get(name=cave_id)
+            cave=Cave.objects.get(name=request.GET['cave_id'])
 
     from export import toqms
 
-    response = HttpResponse(mimetype='text/csv')
+    response = HttpResponse(content_type='text/csv')
     response['Content-Disposition'] = 'attachment; filename=qm.csv'
     toqms.writeQmTable(response,cave)
     return response
@@ -136,7 +135,7 @@ def downloadQMs(request):
 def ajax_test(request):
     post_text = request.POST['post_data']
     return HttpResponse("{'response_text': '"+post_text+" recieved.'}", 
-                                   mimetype="application/json")
+                                   content_type="application/json")
                                    
 def eyecandy(request):
     return 
@@ -144,9 +143,9 @@ def eyecandy(request):
 def ajax_QM_number(request):
     if request.method=='POST':
         cave=Cave.objects.get(id=request.POST['cave'])
-        print cave
+        print(cave)
         exp=Expedition.objects.get(pk=request.POST['year'])
-        print exp
+        print(exp)
         res=cave.new_QM_number(exp.year)
 
     return HttpResponse(res)
@@ -167,7 +166,7 @@ def logbook_entry_suggestions(request):
     #unwiki_QMs=re.findall(unwiki_QM_pattern,lbo.text)
     unwiki_QMs=[m.groupdict() for m in unwiki_QM_pattern.finditer(lbo.text)]
     
-    print unwiki_QMs
+    print(unwiki_QMs)
     for qm in unwiki_QMs:
         #try:
             if len(qm['year'])==2:
@@ -180,7 +179,7 @@ def logbook_entry_suggestions(request):
                 try:
                     lbo=LogbookEntry.objects.get(date__year=qm['year'],title__icontains="placeholder for QMs in")
                 except:
-                    print "failed to get placeholder for year "+str(qm['year'])
+                    print("failed to get placeholder for year "+str(qm['year']))
             
             temp_QM=QM(found_by=lbo,number=qm['number'],grade=qm['grade'])
             temp_QM.grade=qm['grade']
@@ -188,7 +187,7 @@ def logbook_entry_suggestions(request):
         #except:
             #print 'failed'
 
-    print unwiki_QMs
+    print(unwiki_QMs)
     
     
     #wikilink_QMs=re.findall(wikilink_QM_pattern,lbo.text)
@@ -199,10 +198,10 @@ def logbook_entry_suggestions(request):
     #for qm in wikilink_QMs:
         #Try to look up the QM. 
         
-    print 'got 208'
+    print('got 208')
     any_suggestions=True
-    print 'got 210'
+    print('got 210')
-    return render_with_context(request,'suggestions.html',
+    return render(request,'suggestions.html',
         {
         'unwiki_QMs':unwiki_QMs,
         'any_suggestions':any_suggestions
@@ -259,10 +258,10 @@ def newFile(request, pslug = None):
 #                                                            "TU": py.time_underground, 
 #                                                            "author": py.is_logbook_entry_author}
 #                                                           for py in previouslbe.persontrip_set.all()])
-        else:        
+#        else:        
-            fileform = UploadFileForm() # An unbound form
+#            fileform = UploadFileForm() # An unbound form
 
-    return render_with_context(request, 'editfile.html', {
+    return render(request, 'editfile.html', {
         'fileForm': fileform,
 
     })

core/views_survex.py

@@ -1,6 +1,7 @@
 from django import forms
 from django.http import HttpResponseRedirect, HttpResponse
-from django.shortcuts import render_to_response
+from django.shortcuts import render_to_response, render
+from django.core.context_processors import csrf
 from django.http import HttpResponse, Http404
 import re
 import os
@@ -77,16 +78,16 @@ class SvxForm(forms.Form):
     def DiffCode(self, rcode):
         code = self.GetDiscCode()
         difftext = difflib.unified_diff(code.splitlines(), rcode.splitlines())
-        difflist = [ diffline.strip()  for diffline in difftext  if not re.match("\s*$", diffline) ]
+        difflist = [ diffline.strip()  for diffline in difftext  if not re.match(r"\s*$", diffline) ]
         return difflist
 
     def SaveCode(self, rcode):
         fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
         if not os.path.isfile(fname):
             # only save if appears valid
-            if re.search("\[|\]", rcode):   
+            if re.search(r"\[|\]", rcode):   
                 return "Error: clean up all []s from the text"
-            mbeginend = re.search("(?s)\*begin\s+(\w+).*?\*end\s+(\w+)", rcode)
+            mbeginend = re.search(r"(?s)\*begin\s+(\w+).*?\*end\s+(\w+)", rcode)
             if not mbeginend:
                 return "Error: no begin/end block here"
             if mbeginend.group(1) != mbeginend.group(2):
@@ -98,7 +99,7 @@ class SvxForm(forms.Form):
         return "SAVED"
 
     def Process(self):
-        print "....\n\n\n....Processing\n\n\n"
+        print("....\n\n\n....Processing\n\n\n")
         cwd = os.getcwd()
         os.chdir(os.path.split(settings.SURVEX_DATA + self.data['filename'])[0])
         os.system(settings.CAVERN + " --log " + settings.SURVEX_DATA + self.data['filename'] + ".svx")
@@ -137,13 +138,13 @@ def svx(request, survex_file):
                 if not difflist:
                     message = "OUTPUT FROM PROCESSING"
                     logmessage = form.Process()
-                    print logmessage
+                    print(logmessage)
                 else:
                     message = "SAVE FILE FIRST"
                     form.data['code'] = rcode
             if "save" in rform.data:
                 if request.user.is_authenticated():
-                    #print "sssavvving"
+                    #print("sssavvving")
                     message = form.SaveCode(rcode)
                 else:
                     message = "You do not have authority to save this file"
@@ -163,7 +164,7 @@ def svx(request, survex_file):
         difflist.insert(0, message)
     
     #print [ form.data['code'] ]
-    svxincludes = re.findall('\*include\s+(\S+)(?i)', form.data['code'] or "")
+    svxincludes = re.findall(r'\*include\s+(\S+)(?i)', form.data['code'] or "")
     
     vmap = {'settings': settings,
             'has_3d': os.path.isfile(settings.SURVEX_DATA + survex_file + ".3d"),
@@ -172,13 +173,14 @@ def svx(request, survex_file):
             'difflist': difflist,
             'logmessage':logmessage,
             'form':form}
+    vmap.update(csrf(request))
     if outputtype == "ajax":
         return render_to_response('svxfiledifflistonly.html', vmap)
     return render_to_response('svxfile.html', vmap)
 
 def svxraw(request, survex_file):
     svx = open(os.path.join(settings.SURVEX_DATA, survex_file+".svx"), "rb")
-    return HttpResponse(svx, mimetype="text")
+    return HttpResponse(svx, content_type="text")
 
 
 # The cavern running function
@@ -193,20 +195,20 @@ def threed(request, survex_file):
     process(survex_file)
     try:
         threed = open(settings.SURVEX_DATA + survex_file + ".3d", "rb")
-        return HttpResponse(threed, mimetype="model/3d")
+        return HttpResponse(threed, content_type="model/3d")
     except:
         log = open(settings.SURVEX_DATA + survex_file + ".log", "rb")
-        return HttpResponse(log, mimetype="text")
+        return HttpResponse(log, content_type="text")
 
 def log(request, survex_file):
     process(survex_file)
     log = open(settings.SURVEX_DATA + survex_file + ".log", "rb")
-    return HttpResponse(log, mimetype="text")
+    return HttpResponse(log, content_type="text")
 
 def err(request, survex_file):
     process(survex_file)
     err = open(settings.SURVEX_DATA + survex_file + ".err", "rb")
-    return HttpResponse(err, mimetype="text")
+    return HttpResponse(err, content_type="text")
 
 
 
@@ -256,7 +258,7 @@ def identifycavedircontents(gcavedir):
 # direct local non-database browsing through the svx file repositories
 # perhaps should use the database and have a reload button for it
 def survexcaveslist(request):
-    cavesdir = os.path.join(settings.SURVEX_DATA, "caves")
+    cavesdir = os.path.join(settings.SURVEX_DATA, "caves-1623")
     #cavesdircontents = { }
     
     onefilecaves = [ ]
@@ -264,9 +266,11 @@ def survexcaveslist(request):
     subdircaves = [ ]
     
     # first sort the file list
-    fnumlist = [ (-int(re.match("\d*", f).group(0) or "0"), f)  for f in os.listdir(cavesdir) ]
+    fnumlist = [ (-int(re.match(r"\d*", f).group(0) or "0"), f)  for f in os.listdir(cavesdir) ]
     fnumlist.sort()
     
+    print(fnumlist)
+    
     # go through the list and identify the contents of each cave directory
     for num, cavedir in fnumlist:
         if cavedir in ["144", "40"]:
@@ -278,7 +282,7 @@ def survexcaveslist(request):
             survdirobj = [ ]
             
             for lsubsvx in subsvx:
-                survdirobj.append(("caves/"+cavedir+"/"+lsubsvx, lsubsvx))
+                survdirobj.append(("caves-1623/"+cavedir+"/"+lsubsvx, lsubsvx))
             
             # caves with subdirectories
             if subdirs:
@@ -288,7 +292,7 @@ def survexcaveslist(request):
                     assert not dsubdirs
                     lsurvdirobj = [ ]
                     for lsubsvx in dsubsvx:
-                        lsurvdirobj.append(("caves/"+cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))
+                        lsurvdirobj.append(("caves-1623/"+cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))
                     subsurvdirs.append((lsurvdirobj[0], lsurvdirobj[1:]))
                 subdircaves.append((cavedir, (survdirobj[0], survdirobj[1:]), subsurvdirs))
             
@@ -297,6 +301,8 @@ def survexcaveslist(request):
                 multifilecaves.append((survdirobj[0], survdirobj[1:]))
             # single file caves
             else:
+                #print("survdirobj = ")
+                #print(survdirobj)
                 onefilecaves.append(survdirobj[0])
     
     return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves })

databaseReset.py

@@ -2,12 +2,12 @@ import os
 import time
 import settings
 os.environ['PYTHONPATH'] = settings.PYTHON_PATH
-os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
 from django.core import management
 from django.db import connection
 from django.contrib.auth.models import User
 from django.http import HttpResponse
-from  django.core.urlresolvers import reverse
+from django.core.urlresolvers import reverse
 from troggle.core.models import Cave, Entrance
 import troggle.flatpages.models
 
@@ -22,7 +22,7 @@ def reload_db():
             os.remove(databasename)
         except OSError:
             pass
-    else: 
+    else:
         cursor = connection.cursor()
         cursor.execute("DROP DATABASE %s" % databasename)
         cursor.execute("CREATE DATABASE %s" % databasename)
@@ -42,7 +42,7 @@ def make_dirs():
 
 def import_caves():
     import parsers.caves
-    print "importing caves"
+    print("Importing Caves")
     parsers.caves.readcaves()
 
 def import_people():
@@ -81,6 +81,15 @@ def import_tunnelfiles():
     parsers.surveys.LoadTunnelFiles()
 
 
+def rebuild():
+    """ Wipe the troggle database and sets up structure but imports nothing
+    """
+    reload_db()
+    make_dirs()
+    pageredirects()
+
+
+
 def reset():
     """ Wipe the troggle database and import everything from legacy data
     """
@@ -90,13 +99,13 @@ def reset():
     import_caves()
     import_people()
     import_surveyscans()
-    import_survex()
     import_logbooks()
     import_QMs()
+    import_survex()
     try:
         import_tunnelfiles()
     except:
-        print "Tunnel files parser broken."
+        print("Tunnel files parser broken.")
     
     import_surveys()
 
@@ -104,18 +113,18 @@ def reset():
 def import_auto_logbooks():
     import parsers.logbooks
     import os
-    for pt in core.models.PersonTrip.objects.all():
+    for pt in troggle.core.models.PersonTrip.objects.all():
         pt.delete()
-    for lbe in core.models.LogbookEntry.objects.all():
+    for lbe in troggle.core.models.LogbookEntry.objects.all():
         lbe.delete()
-    for expedition in core.models.Expedition.objects.all():
+    for expedition in troggle.core.models.Expedition.objects.all():
-        directory = os.path.join(settings.EXPOWEB, 
+        directory = os.path.join(settings.EXPOWEB,
-                                 "years", 
+                                 "years",
-                                 expedition.year, 
+                                 expedition.year,
-                                 "autologbook")       
+                                 "autologbook")
         for root, dirs, filenames in os.walk(directory):
             for filename in filenames:
-                print os.path.join(root, filename)
+                print(os.path.join(root, filename))
                 parsers.logbooks.parseAutoLogBookEntry(os.path.join(root, filename))
 
 #Temporary function until definative source of data transfered.
@@ -127,10 +136,10 @@ def dumplogbooks():
                 return pe.nickname
             else:
                 return pe.person.first_name
-    for lbe in core.models.LogbookEntry.objects.all():
+    for lbe in troggle.core.models.LogbookEntry.objects.all():
             dateStr = lbe.date.strftime("%Y-%m-%d")
             directory = os.path.join(settings.EXPOWEB, 
-                                "years", 
+                                "years",
                                 lbe.expedition.year, 
                                 "autologbook")
             if not os.path.isdir(directory):
@@ -138,7 +147,7 @@ def dumplogbooks():
             filename = os.path.join(directory, 
                                     dateStr + "." + slugify(lbe.title)[:50] + ".html")
             if lbe.cave:
-                print lbe.cave.reference()
+                print(lbe.cave.reference())
                 trip = {"title": lbe.title, "html":lbe.text, "cave": lbe.cave.reference(), "caveOrLocation": "cave"}
             else:
                 trip = {"title": lbe.title, "html":lbe.text, "location":lbe.place, "caveOrLocation": "location"}
@@ -156,45 +165,39 @@ def dumplogbooks():
 
 def pageredirects():
     for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
-        f = flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
+        f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
         f.save()
 
-def writeCaves():
-    for cave in Cave.objects.all():
-        cave.writeDataFile()
-    for entrance in Entrance.objects.all():
-        entrance.writeDataFile()
-
 def usage():
-    print """Usage is 'python databaseReset.py <command>'
+    print("""Usage is 'python databaseReset.py <command>'
              where command is:
-             reset - this is normal usage, clear database and reread everything
+             rebuild  - this reloads database and set up directories & redirects only
-             desc
+             reset    - this is normal usage, clear database and reread everything from files - time-consuming
-             caves - read in the caves
+             desc     - NOT WORKING: function resetdesc() missing
-             logbooks - read in the logbooks
+             caves    - read in the caves 
-             autologbooks
+             logbooks - read in the logbooks, but read in people first
-             dumplogbooks
+             autologbooks - read in autologbooks
-             people
+             dumplogbooks - write out autologbooks (not working?)
+             people   - read in the people from folk.csv
              QMs      - read in the QM files
              resetend
              scans    - read in the scanned surveynotes
              survex   - read in the survex files
              survexpos
+             surveys
              tunnel   - read in the Tunnel files
-             writeCaves
+             """)
-             """ 
 
 if __name__ == "__main__":
-    import core.models
+    import troggle.core.models
     import sys
+    import django
+    django.setup()
     if "desc" in sys.argv:
         resetdesc()
     elif "scans" in sys.argv:
         import_surveyscans()
     elif "caves" in sys.argv:
-        reload_db()
-        make_dirs()
-        pageredirects()
         import_caves()
     elif "people" in sys.argv:
         import_people()
@@ -210,19 +213,19 @@ if __name__ == "__main__":
         try:
             import_tunnelfiles()
         except:
-            print "Tunnel files parser broken."
+            print("Tunnel files parser broken.")
         import_surveys()
         import_descriptions()
         parse_descriptions()
     elif "survex" in sys.argv:
-        management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
+        # management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
         import_survex()
     elif "survexpos" in sys.argv:
-        management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
+        # management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
         import parsers.survex
         parsers.survex.LoadPos()    
     elif "logbooks" in sys.argv:
-        management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
+        # management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
         import_logbooks()
     elif "autologbooks" in sys.argv:
         import_auto_logbooks()
@@ -230,12 +233,14 @@ if __name__ == "__main__":
         dumplogbooks()
     elif "writeCaves" in sys.argv:
         writeCaves()
+    elif "surveys" in sys.argv:
+        import_surveys()
     elif "help" in sys.argv:
         usage()
+    elif "reload_db" in sys.argv:
+        reload_db()
+    elif "rebuild" in sys.argv:
+        rebuild()    
     else:
-        print "%s not recognised" % sys.argv
+        print("%s not recognised" % sys.argv)
         usage()
-
-
-    
-    

debian/serversetup

@@ -0,0 +1,85 @@
+Instructions for setting up new expo debian server/VM
+For Debian Stretch, June 2019.
+
+adduser expo
+apt install openssh-server mosh tmux mc zile emacs-nox mc most ncdu
+apt install python-django apache2 mysql-server survex make rsync 
+apt install libjs-openlayers make 
+apt install git mercurial mercurial-server?
+
+for boe:
+apt install libcgi-session-perl libcrypt-passwdmd5-perl libfile-slurp-perl libgit-wrapper-perl libhtml-template-perl libhtml-template-pro-perl libmime-lite-perl libtext-password-pronounceable-perl libtime-parsedate-perl libuuid-tiny-perl libcrypt-cracklib-perl
+
+obsolete-packages:
+ bins (move to jigl?) (for photos)
+ python-django 1.7
+backports: survex therion
+not-packaged: caveview
+
+make these dirs available at top documentroot:
+cuccfiles
+expofiles
+loser  (link to repo)
+tunneldata (link to repo)
+troggle (link to repo)
+expoweb (link to repo)
+boc/boe
+
+
+config
+containing:
+
+setup apache configs for cucc and expo
+#disable default website
+a2dissite 000-default
+a2ensite cucc
+a2ensite expo
+a2enmod cgid
+
+
+Boe config:
+Alias /boe /home/expo/boe/boc/boc.pl
+<Directory /home/expo/boe/boc>
+        AddHandler cgi-script .pl
+        SetHandler cgi-script
+        Options +ExecCGI
+        Require all granted
+</Directory>
+And remember to set both program and data dir to be
+www-data:www-data
+(optionally make file group read/write by treasurer account)
+create empty repo by clicking create in boe interface
+then set names in 'settings'
+
+Set up mysql (as root)
+mysql -p
+CREATE DATABASE troggle;
+GRANT ALL PRIVILEGES ON troggle.* TO 'expo'@'localhost' IDENTIFIED BY 'somepassword';  
+
+install django:
+sudo apt install python-django python-django-registration python-django-imagekit python-django-tinymce fonts-freefont-ttf libapache2-mod-wsgi
+
+python-django-imagekit comes from https://salsa.debian.org/python-team/modules/python-django-imagekit
+python-django-tinymce comes from https://salsa.debian.org/python-team/modules/python-django-tinymce
+(both modified for stretch/python2). packages under /home/wookey/packages/
+
+need fonts-freefont-ttf (to have truetype freesans available for troggle via PIL)
+need libapache2-mod-wsgi for apache wsgi support.
+
+On stretch the django 1.10 is no use so get rid of that:
+apt remove python3-django python-django python-django-common python-django-doc
+
+Then replace with django 1.7 (Needs to be built for stretch)
+apt install python-django python-django-common python-django-doc
+apt install python-django-registration python-django-imagekit python-django-tinymce
+
+then hold them to stop them being upgraded by unattended upgrades:
+echo "python-django hold" | sudo dpkg --set-selections
+echo "python-django-common hold" | sudo dpkg --set-selections
+echo "python-django-doc hold" | sudo dpkg --set-selections
+
+#troggle has to have a writable logfile otherwise the website explodes
+# 500 error on the server, and apache error log has non-rentrant errors
+create /var/log/troggle/troggle.log
+chown www-data:adm  /var/log/troggle/troggle.log
+chmod 660 /var/log/troggle/troggle.log

docker/Dockerfile

@@ -23,4 +23,6 @@ EXPOSE 8000
 
 WORKDIR /expo/troggle
 
-#CMD ["python","manage.py","runserver","0.0.0.0:8000"]
+#CMD ["python","manage.py","migrate"]
+
+ENTRYPOINT ["python","manage.py","runserver","0.0.0.0:8000"]

docker/README.md

@@ -0,0 +1,81 @@
+# Running troggle on Docker
+
+## Install
+First you need to install
+ - [docker-ce](https://docs.docker.com/install/)
+ - [docker-compose](https://docs.docker.com/compose/install/)
+
+If you don't want to type sudo for every docker command (you don't) you need to add your user to the docker group.
+
+To do this
+
+- Create the docker group.
+    ```bash
+    $ sudo groupadd docker
+    ```
+
+- Add your user to the docker group.
+    ```bash
+    $ sudo usermod -aG docker $USER
+    ```
+## Setup
+
+Checkout all 4 of the expo repos into one folder ([see here for repo cloning instructions](http://expo.survex.com/handbook/manual.html#quickstart)) eg.
+```
+$ ~/expo/loser
+        /troggle
+        /expoweb
+        /tunnel
+```
+In the troggle dir copy localsettingsdocker.py to localsettings.py
+
+In the folder you checked out all the repos into (expo in this example) create a folder called `expofiles` and in that a folder called `surveyscans` eg
+
+```bash
+cd ~/expo
+mkdir -p expofiles/surveyscans
+```
+
+## Starting the containers
+
+To start the containers run
+```bash
+$ docker-compose up
+```
+You will now have a working troggle but with no data. To import the data you need to access the container run
+```bash
+$ docker exec -it docker_troggle_1 /bin/bash
+```
+This will give you a shell inside the troggle container
+
+(You may get a warning like `bash: warning: setlocale: LC_ALL: cannot change locale (en_GB.UTF-8)` this can be ignored)
+
+To import the data into troggle now run
+```bash
+$ python databaseReset.py reset
+```
+and wait .... this takes a while.
+The MySQL database is stored in a docker volume so once run through once the database will remain full of expo data even if you restart the containers.
+
+## Using your new dev setup
+Even whilst the import is running you can browse to [http://localhost:8000]
+
+Any chnages made to files in your checkouts will be automatically loaded in the container and served. Somtimes changes to the python files will require the system to reload so refresh a couple of times before declaring you have broken something.
+
+If you edit files from within the container they will have their ownership changed to root and may become un editable to your user (you will have to become root to fix this) so don't do this!
+
+## Stopping the containers
+To stop the running containers press Ctrl-c 
+
+## Killing it all
+If you get it in some state you want to start again run
+```bash
+$ docker-compose down
+```
+to destroy the containers
+and
+```bash
+$ docker volume ls
+$ docker volume rm docker_expo-mysqldb
+```
+ to remove the database volume

docker/compose/mysql.env

@@ -0,0 +1,5 @@
+MYSQL_ROOT_PASSWORD=expo123
+MYSQL_DATABASE=troggle
+MYSQL_USER=troggleuser
+MYSQL_PASSWORD=expo123
+

docker/docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3'
+services:
+  troggle:
+    restart: always
+    build: .
+    ports:
+     - "8000:8000"
+    volumes:
+     - ../..:/expo
+    links:
+     - expo-mysql
+  expo-mysql:
+    restart: always
+    image: "mariadb"
+    env_file:
+      - compose/mysql.env
+    volumes:
+      - expo-mysqldb:/var/lib/mysql
+
+volumes:
+  expo-mysqldb:

docker/mysql.env

@@ -0,0 +1,5 @@
+MYSQL_ROOT_PASSWORD=expo123
+MYSQL_DATABASE=troggle
+MYSQL_USER=troggleuser
+MYSQL_PASSWORD=expo123
+

docker/requirements.txt

@@ -1,7 +0,0 @@
-Django==1.7.11
-django-registration==2.1.2
-mysql
-imagekit
-Image
-django-tinymce==2.7.0
-smartencoding

docker/requirements.txt

@@ -0,0 +1 @@
+requirements.txt.dj-1.7.11

docker/requirements.txt.dj-1.7.11

@@ -0,0 +1,9 @@
+Django==1.7.11
+django-registration==2.1.2
+mysql
+#imagekit
+django-imagekit
+Image
+django-tinymce==2.7.0
+smartencoding
+unidecode

docker/requirements.txt.dj-1.8.19

@@ -0,0 +1,7 @@
+Django==1.8.19
+django-registration==2.1.2
+mysql
+django-imagekit
+Image
+django-tinymce==2.7.0
+smartencoding

flatpages/views.py

@@ -1,6 +1,6 @@
 import troggle.settings as settings
 from troggle.helper import login_required_if_public
-from utils import render_with_context
+from django.shortcuts import render
 
 from django.http import HttpResponse, HttpResponseRedirect, Http404
 from  django.core.urlresolvers import reverse
@@ -35,10 +35,10 @@ def flatpage(request, path):
 
 
     if path.startswith("noinfo") and settings.PUBLIC_SITE and not request.user.is_authenticated():
-        print "flat path noinfo", path
+        print("flat path noinfo", path)
         return HttpResponseRedirect(reverse("auth_login") + '?next=%s' % request.path)
 
-    if path.endswith("/") or path == "":        
+    if path.endswith("/") or path == "":
         try:
             o = open(os.path.normpath(settings.EXPOWEB + path + "index.html"), "rb")
             path = path + "index.html"
@@ -47,13 +47,13 @@ def flatpage(request, path):
                 o = open(os.path.normpath(settings.EXPOWEB + path + "index.htm"), "rb")
                 path = path + "index.htm"
             except IOError:
-                return render_with_context(request, 'pagenotfound.html', {'path': path})
+                return render(request, 'pagenotfound.html', {'path': path})
     else:        
         try:
             filetobeopened = os.path.normpath(settings.EXPOWEB + path)
             o = open(filetobeopened, "rb")
         except IOError:
-            return render_with_context(request, 'pagenotfound.html', {'path': path})
+            return render(request, 'pagenotfound.html', {'path': path})
     if path.endswith(".htm") or path.endswith(".html"):
         html = o.read()
         
@@ -67,13 +67,15 @@ def flatpage(request, path):
             title, = m.groups()
         else:
             title = ""
-        linksmatch = re.match('(.*)<ul id="links">', body, re.DOTALL + re.IGNORECASE)
+        has_menu = False
-        if linksmatch:
+        menumatch = re.match('(.*)<div id="menu">', body, re.DOTALL + re.IGNORECASE)
-            body, = linksmatch.groups()
+        if menumatch:
+            has_menu = True
+            #body, = menumatch.groups()
         if re.search(r"iso-8859-1", html):
             body = unicode(body, "iso-8859-1")
             body.strip
-        return render_with_context(request, 'flatpage.html', {'editable': True, 'path': path, 'title': title, 'body': body, 'homepage': (path == "index.htm")})
+        return render(request, 'flatpage.html', {'editable': True, 'path': path, 'title': title, 'body': body, 'homepage': (path == "index.htm"), 'has_menu': has_menu})
     else:
         return HttpResponse(o.read(), content_type=getmimetype(path))
 
@@ -114,7 +116,7 @@ def editflatpage(request, path):
         if m:
             filefound = True
             preheader, headerargs, head, postheader, bodyargs, body, postbody = m.groups()
-            linksmatch = re.match('(.*)(<ul\s+id="links">.*)', body, re.DOTALL + re.IGNORECASE)
+            linksmatch = re.match(r'(.*)(<ul\s+id="links">.*)', body, re.DOTALL + re.IGNORECASE)
             if linksmatch:
                 body, links = linksmatch.groups()
             if re.search(r"iso-8859-1", html):
@@ -158,9 +160,9 @@ def editflatpage(request, path):
             flatpageForm = FlatPageForm({"html": body, "title": title})
         else:
             flatpageForm = FlatPageForm()
-    return render_with_context(request, 'editflatpage.html', {'path': path, 'form': flatpageForm, })
+    return render(request, 'editflatpage.html', {'path': path, 'form': flatpageForm, })
 
 class FlatPageForm(forms.Form):
     title = forms.CharField(widget=forms.TextInput(attrs={'size':'60'}))
 
-    html = forms.CharField(widget=forms.Textarea())
+    html = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 20}))

localsettingsdocker.py

@@ -0,0 +1,65 @@
+import sys
+# This is the local settings for use with the docker compose dev setup. It is imported automatically
+
+DATABASES = {
+    'default': { 
+        'ENGINE': 'django.db.backends.mysql', # 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
+        'NAME' : 'troggle',     # Or path to database file if using sqlite3.
+        'USER' : 'troggleuser',        # Not used with sqlite3.
+        'PASSWORD' : 'expo123',  # Not used with sqlite3.
+        'HOST' : 'expo-mysql',            # Set to empty string for localhost. Not used with sqlite3.
+        'PORT' : '',            # Set to empty string for default. Not used with sqlite3.
+    }
+}
+
+EXPOUSER = 'expo'
+EXPOUSERPASS = 'somepasshere'
+EXPOUSER_EMAIL = 'wookey@wookware.org'
+
+REPOS_ROOT_PATH = '/expo/'
+sys.path.append(REPOS_ROOT_PATH)
+sys.path.append(REPOS_ROOT_PATH + 'troggle')
+
+PUBLIC_SITE = False
+
+SURVEX_DATA = REPOS_ROOT_PATH + 'loser/'
+TUNNEL_DATA = REPOS_ROOT_PATH + 'tunneldata/'
+
+CAVERN = 'cavern'
+THREEDTOPOS = '3dtopos'
+EXPOWEB = REPOS_ROOT_PATH + 'expoweb/'
+SURVEYS = REPOS_ROOT_PATH
+SURVEY_SCANS = REPOS_ROOT_PATH + 'expofiles/'
+FILES = REPOS_ROOT_PATH + 'expofiles'
+
+CACHEDIR = REPOS_ROOT_PATH + 'expowebcache/'
+THREEDCACHEDIR = CACHEDIR + '3d/'
+THUMBNAILCACHE = CACHEDIR + 'thumbs'
+
+PYTHON_PATH = REPOS_ROOT_PATH + 'troggle/'
+
+URL_ROOT = 'http://127.0.0.1:8000/'
+DIR_ROOT = ''#this should end in / if a value is given
+EXPOWEB_URL = '/'
+SURVEYS_URL = '/survey_scans/'
+
+MEDIA_URL = URL_ROOT + DIR_ROOT + 'site_media/'
+MEDIA_ROOT = REPOS_ROOT_PATH + '/troggle/media/'
+MEDIA_ADMIN_DIR = '/usr/lib/python2.7/site-packages/django/contrib/admin/media/'
+
+STATIC_URL = "/static/"
+STATIC_ROOT = "/expo/static"
+
+JSLIB_URL = URL_ROOT + 'javascript/'
+
+TINY_MCE_MEDIA_ROOT = STATIC_ROOT + '/tiny_mce/'
+TINY_MCE_MEDIA_URL = STATIC_ROOT + '/tiny_mce/'
+
+TEMPLATE_DIRS = (
+    PYTHON_PATH + "templates",
+    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+LOGFILE = PYTHON_PATH + 'troggle_log.txt'

localsettingspotatohut.py

@@ -24,6 +24,7 @@ FIX_PERMISSIONS = ["sudo", "/usr/local/bin/fix_permissions"]
 
 SURVEX_DATA = REPOS_ROOT_PATH + 'loser/'
 TUNNEL_DATA = REPOS_ROOT_PATH + 'tunneldata/'
+THREEDCACHEDIR = REPOS_ROOT_PATH + 'expowebcache/3d/'
 
 CAVERN = 'cavern'
 THREEDTOPOS = '3dtopos'

localsettingsserver.py

@@ -26,6 +26,7 @@ PUBLIC_SITE = True
 
 SURVEX_DATA = REPOS_ROOT_PATH + 'loser/'
 TUNNEL_DATA = REPOS_ROOT_PATH + 'tunneldata/'
+THREEDCACHEDIR = REPOS_ROOT_PATH + 'expowebcache/3d/'
 
 CAVERN = 'cavern'
 THREEDTOPOS = '3dtopos'
@@ -51,8 +52,8 @@ MEDIA_ADMIN_DIR = '/usr/lib/python2.7/site-packages/django/contrib/admin/media/'
 
 JSLIB_URL = URL_ROOT + 'javascript/'
 
-TINY_MCE_MEDIA_ROOT = '/usr/share/tinymce/www/'
+TINY_MCE_MEDIA_ROOT = STATIC_ROOT + '/tiny_mce/'
-TINY_MCE_MEDIA_URL = URL_ROOT + DIR_ROOT + 'tinymce_media/'
+TINY_MCE_MEDIA_URL = STATIC_ROOT + '/tiny_mce/'
 
 TEMPLATE_DIRS = (
     PYTHON_PATH + "templates",

localsettingsubuntu.py

@@ -25,6 +25,7 @@ PUBLIC_SITE = False
 
 SURVEX_DATA = REPOS_ROOT_PATH + 'loser/'
 TUNNEL_DATA = REPOS_ROOT_PATH + 'tunneldata/'
+THREEDCACHEDIR = REPOS_ROOT_PATH + 'expowebcache/3d/'
 
 CAVERN = 'cavern'
 THREEDTOPOS = '3dtopos'

parsers/QMs.py

@@ -17,19 +17,19 @@ def parseCaveQMs(cave,inputFile):
         try:
             steinBr=Cave.objects.get(official_name="Steinbrückenhöhle")
         except Cave.DoesNotExist:
-            print "Steinbruckenhoehle is not in the database. Please run parsers.cavetab first."
+            print("Steinbruckenhoehle is not in the database. Please run parsers.cavetab first.")
             return
     elif cave=='hauch':
         try:
             hauchHl=Cave.objects.get(official_name="Hauchhöhle")
         except Cave.DoesNotExist:
-            print "Hauchhoele is not in the database. Please run parsers.cavetab first."
+            print("Hauchhoele is not in the database. Please run parsers.cavetab first.")
             return
     elif cave =='kh':
         try:
             kh=Cave.objects.get(official_name="Kaninchenhöhle")
         except Cave.DoesNotExist:
-            print "KH is not in the database. Please run parsers.cavetab first."
+            print("KH is not in the database. Please run parsers.cavetab first.")
         parse_KH_QMs(kh, inputFile=inputFile) 
         return
 
@@ -48,7 +48,7 @@ def parseCaveQMs(cave,inputFile):
             elif cave=='hauch':
                 placeholder, hadToCreate = LogbookEntry.objects.get_or_create(date__year=year, title="placeholder for QMs in 234", text="QMs temporarily attached to this should be re-attached to their actual trips", defaults={"date": date(year, 1, 1),"cave":hauchHl})            
             if hadToCreate:
-                print cave+" placeholder logbook entry for " + str(year) + " added to database"
+                print(cave + " placeholder logbook entry for " + str(year) + " added to database")
             QMnum=re.match(r".*?-\d*?-X?(?P<numb>\d*)",line[0]).group("numb")
             newQM = QM()
             newQM.found_by=placeholder
@@ -71,19 +71,18 @@ def parseCaveQMs(cave,inputFile):
                 if preexistingQM.new_since_parsing==False:  #if the pre-existing QM has not been modified, overwrite it
                     preexistingQM.delete()
                     newQM.save()
-                    print "overwriting " + str(preexistingQM) +"\r",
+                    print("overwriting " + str(preexistingQM) +"\r")
-                
                 else:  # otherwise, print that it was ignored
-                    print "preserving "+ str(preexistingQM) + ", which was edited in admin \r",
+                    print("preserving " + str(preexistingQM) + ", which was edited in admin \r")
                     
             except QM.DoesNotExist:         #if there is no pre-existing QM, save the new one
                 newQM.save() 
-                print "QM "+str(newQM) + ' added to database\r',
+                print("QM "+str(newQM) + ' added to database\r')
                 
         except KeyError: #check on this one
             continue
         except IndexError:
-            print "Index error in " + str(line)
+            print("Index error in " + str(line))
             continue
 
 def parse_KH_QMs(kh, inputFile):
@@ -104,7 +103,7 @@ def parse_KH_QMs(kh, inputFile):
                 }
             nonLookupArgs={
                 'grade':res['grade'],
-                'nearest_station':res['nearest_station'],
+                'nearest_station_name':res['nearest_station'],
                 'location_description':res['description']
                 }
  
@@ -115,3 +114,4 @@ parseCaveQMs(cave='stein',inputFile=r"1623/204/qm.csv")
 parseCaveQMs(cave='hauch',inputFile=r"1623/234/qm.csv")
 parseCaveQMs(cave='kh', inputFile="1623/161/qmtodo.htm")
 #parseCaveQMs(cave='balkonhoehle',inputFile=r"1623/264/qm.csv")
+

parsers/caves.py

@@ -6,18 +6,21 @@ import re
 
 
 def readcaves():
-  newArea = models.Area(short_name = "1623", parent = None)
+
-  newArea.save()
+  # Clear the cave data issues as we are reloading
-  newArea = models.Area(short_name = "1626", parent = None)
+  models.DataIssue.objects.filter(parser='caves').delete()
-  newArea.save()
+
-  print "Reading Entrances"
+  area_1623 = models.Area.objects.update_or_create(short_name = "1623", parent = None)
+  area_1626 = models.Area.objects.update_or_create(short_name = "1626", parent = None)
+  print(" - Reading Entrances")
   #print "list of <Slug> <Filename>"
   for filename in os.walk(settings.ENTRANCEDESCRIPTIONS).next()[2]: #Should be a better way of getting a list of files
     if filename.endswith('.html'):
-        readentrance(filename)
+      readentrance(filename)
-  print "Reading Caves"
+  print (" - Reading Caves")
   for filename in os.walk(settings.CAVEDESCRIPTIONS).next()[2]: #Should be a better way of getting a list of files
-    readcave(filename)
+    if filename.endswith('.html'):
+      readcave(filename)
 
 def readentrance(filename):
     with open(os.path.join(settings.ENTRANCEDESCRIPTIONS, filename)) as f:
@@ -50,7 +53,7 @@ def readentrance(filename):
         bearings = getXML(entrancecontents, "bearings", maxItems = 1, context = context)
         url = getXML(entrancecontents, "url", maxItems = 1, context = context)
         if len(non_public) == 1 and len(slugs) >= 1 and len(name) >= 1 and  len(entrance_description) == 1 and  len(explorers) == 1 and  len(map_description) == 1 and  len(location_description) == 1 and  len(approach) == 1 and  len(underground_description) == 1 and  len(marking) == 1 and  len(marking_comment) == 1 and  len(findability) == 1 and  len(findability_description) == 1 and  len(alt) == 1 and  len(northing) == 1 and  len(easting) == 1 and  len(tag_station) == 1 and  len(exact_station) == 1 and  len(other_station) == 1 and  len(other_description) == 1 and  len(bearings) == 1 and  len(url) == 1:
-            e = models.Entrance(name = name[0],
+            e, state = models.Entrance.objects.update_or_create(name = name[0],
                          non_public = {"True": True, "False": False, "true": True, "false": False,}[non_public[0]],
                          entrance_description = entrance_description[0],
                          explorers = explorers[0],
@@ -74,14 +77,12 @@ def readentrance(filename):
                          url = url[0],
                          filename = filename,
                          cached_primary_slug = slugs[0])
-            e.save()
             primary = True
             for slug in slugs:
                 #print slug, filename
-                cs = models.EntranceSlug(entrance = e,
+                cs = models.EntranceSlug.objects.update_or_create(entrance = e,
                                          slug = slug,
                                          primary = primary)
-                cs.save()
                 primary = False
 
 def readcave(filename):
@@ -116,7 +117,7 @@ def readcave(filename):
         url = getXML(cavecontents, "url", maxItems = 1, context = context)
         entrances = getXML(cavecontents, "entrance", context = context)
         if len(non_public) == 1 and len(slugs) >= 1 and len(official_name) == 1 and len(areas) >= 1 and len(kataster_code) == 1 and len(kataster_number) == 1 and len(unofficial_number) == 1 and len(explorers) == 1 and len(underground_description) == 1 and len(equipment) == 1 and len(references) == 1 and len(survey) == 1 and len(kataster_status) == 1 and len(underground_centre_line) == 1 and len(notes) == 1 and len(length) == 1 and len(depth) == 1 and len(extent) == 1 and len(survex_file) == 1 and len(description_file ) == 1 and len(url) == 1 and len(entrances) >= 1:
-            c = models.Cave(non_public = {"True": True, "False": False, "true": True, "false": False,}[non_public[0]],
+            c, state = models.Cave.objects.update_or_create(non_public = {"True": True, "False": False, "true": True, "false": False,}[non_public[0]],
                      official_name = official_name[0],
                      kataster_code = kataster_code[0],
                      kataster_number = kataster_number[0],
@@ -136,7 +137,6 @@ def readcave(filename):
                      description_file = description_file[0],
                      url = url[0],
                      filename = filename)
-            c.save()
             for area_slug in areas:
                 area = models.Area.objects.filter(short_name = area_slug)
                 if area:
@@ -148,12 +148,13 @@ def readcave(filename):
             primary = True
             for slug in slugs:
                 try:
-                    cs = models.CaveSlug(cave = c,
+                    cs = models.CaveSlug.objects.update_or_create(cave = c,
                               slug = slug,
                               primary = primary)
-                    cs.save()
                 except:
-                    print "Can't find text (slug): %s, skipping %s" % (slug, context)
+                    message = "Can't find text (slug): %s, skipping %s" % (slug, context)
+                    models.DataIssue.objects.create(parser='caves', message=message)
+                    print(message)
                     
                 primary = False
             for entrance in entrances:
@@ -161,20 +162,26 @@ def readcave(filename):
                 letter = getXML(entrance, "letter", maxItems = 1, context = context)[0]
                 try:
                     entrance = models.Entrance.objects.get(entranceslug__slug = slug)
-                    ce = models.CaveAndEntrance(cave = c, entrance_letter = letter, entrance = entrance)
+                    ce = models.CaveAndEntrance.objects.update_or_create(cave = c, entrance_letter = letter, entrance = entrance)
-                    ce.save()
                 except:
-                    print "Entrance text (slug) %s missing %s" % (slug, context)
+                    message = "Entrance text (slug) %s missing %s" % (slug, context)
+                    models.DataIssue.objects.create(parser='caves', message=message)
+                    print(message)
                 
 
 def getXML(text, itemname, minItems = 1, maxItems = None, printwarnings = True, context = ""):
     items = re.findall("<%(itemname)s>(.*?)</%(itemname)s>" % {"itemname": itemname}, text, re.S)
     if len(items) < minItems and printwarnings:
-        print "%(count)i %(itemname)s found, at least %(min)i expected" % {"count": len(items),
+        message = "%(count)i %(itemname)s found, at least %(min)i expected" % {"count": len(items),
                                                                            "itemname": itemname,
                                                                            "min": minItems} + context
+        models.DataIssue.objects.create(parser='caves', message=message)
+        print(message)
+        
     if maxItems is not None and len(items) > maxItems and printwarnings:
-        print "%(count)i %(itemname)s found, no more than %(max)i expected" % {"count": len(items),
+        message = "%(count)i %(itemname)s found, no more than %(max)i expected" % {"count": len(items),
                                                                                "itemname": itemname,
                                                                                "max": maxItems} + context
+        models.DataIssue.objects.create(parser='caves', message=message)
+        print(message)
     return items

parsers/logbooks.py

@@ -7,6 +7,8 @@ from parsers.people import GetPersonExpeditionNameLookup
 from parsers.cavetab import GetCaveLookup
 
 from django.template.defaultfilters import slugify
+from django.utils.timezone import get_current_timezone
+from django.utils.timezone import make_aware
 
 import csv
 import re
@@ -23,19 +25,23 @@ from utils import save_carefully
 #
 # the logbook loading section
 #
-def GetTripPersons(trippeople, expedition, logtime_underground):    
+def GetTripPersons(trippeople, expedition, logtime_underground):
     res = [ ]
     author = None
-    for tripperson in re.split(",|\+|&|&(?!\w+;)| and ", trippeople):
+    round_bracket_regex = re.compile(r"[\(\[].*?[\)\]]")
+    for tripperson in re.split(r",|\+|&|&(?!\w+;)| and ", trippeople):
         tripperson = tripperson.strip()
-        mul = re.match("<u>(.*?)</u>$(?i)", tripperson)
+        mul = re.match(r"<u>(.*?)</u>$(?i)", tripperson)
         if mul:
             tripperson = mul.group(1).strip()
         if tripperson and tripperson[0] != '*':
             #assert tripperson in personyearmap, "'%s' << %s\n\n %s" % (tripperson, trippeople, personyearmap)
+            tripperson = re.sub(round_bracket_regex, "", tripperson).strip()
             personyear = GetPersonExpeditionNameLookup(expedition).get(tripperson.lower())
             if not personyear:
-                print "NoMatchFor: '%s'" % tripperson    
+                print("   - No name match for: '%s'" % tripperson)
+                message = "No name match for: '%s' in year '%s'" % (tripperson, expedition.year)
+                models.DataIssue.objects.create(parser='logbooks', message=message)
             res.append((personyear, logtime_underground))
             if mul:
                 author = personyear
@@ -45,7 +51,7 @@ def GetTripPersons(trippeople, expedition, logtime_underground):
         author = res[-1][0]
     return res, author
 
-def GetTripCave(place):                     #need to be fuzzier about matching here. Already a very slow function...
+def GetTripCave(place):  #need to be fuzzier about matching here. Already a very slow function...
 #    print "Getting cave for " , place
     try:
         katastNumRes=[]
@@ -65,32 +71,34 @@ def GetTripCave(place):                     #need to be fuzzier about matching h
         return tripCaveRes
 
     elif len(tripCaveRes)>1:
-        print "Ambiguous place " + str(place) + " entered. Choose from " + str(tripCaveRes)
+        print("Ambiguous place " + str(place) + " entered. Choose from " + str(tripCaveRes))
         correctIndex=input("type list index of correct cave")
         return tripCaveRes[correctIndex]
     else:
-        print "No cave found for place " , place
+        print("No cave found for place " , place)
         return
 
 
 noncaveplaces = [ "Journey", "Loser Plateau" ]
-def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground):
+def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_underground, entry_type="wiki"):
     """ saves a logbook entry and related persontrips """
     trippersons, author = GetTripPersons(trippeople, expedition, logtime_underground)
     if not author:
-        print "skipping logentry", title
+        print("   - Skipping logentry: " + title + " - no author for entry")
+        message = "Skipping logentry: %s - no author for entry in year '%s'" % (title, expedition.year)
+        models.DataIssue.objects.create(parser='logbooks', message=message)
         return
-    
+
-#    tripCave = GetTripCave(place)
+    #tripCave = GetTripCave(place)
-    #
+
     lplace = place.lower()
     if lplace not in noncaveplaces:
         cave=GetCaveLookup().get(lplace)
 
     #Check for an existing copy of the current entry, and save
     expeditionday = expedition.get_expedition_day(date)
-    lookupAttribs={'date':date, 'title':title} 
+    lookupAttribs={'date':date, 'title':title}
-    nonLookupAttribs={'place':place, 'text':text, 'expedition':expedition, 'cave':cave, 'slug':slugify(title)[:50]}
+    nonLookupAttribs={'place':place, 'text':text, 'expedition':expedition, 'cave':cave, 'slug':slugify(title)[:50], 'entry_type':entry_type}
     lbo, created=save_carefully(models.LogbookEntry, lookupAttribs, nonLookupAttribs)
     
     for tripperson, time_underground in trippersons:
@@ -102,8 +110,8 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_
 
 def ParseDate(tripdate, year):
     """ Interprets dates in the expo logbooks and returns a correct datetime.date object  """
-    mdatestandard = re.match("(\d\d\d\d)-(\d\d)-(\d\d)", tripdate)
+    mdatestandard = re.match(r"(\d\d\d\d)-(\d\d)-(\d\d)", tripdate)
-    mdategoof = re.match("(\d\d?)/0?(\d)/(20|19)?(\d\d)", tripdate)
+    mdategoof = re.match(r"(\d\d?)/0?(\d)/(20|19)?(\d\d)", tripdate)
     if mdatestandard:
         assert mdatestandard.group(1) == year, (tripdate, year)
         year, month, day = int(mdatestandard.group(1)), int(mdatestandard.group(2)), int(mdatestandard.group(3))
@@ -115,9 +123,9 @@ def ParseDate(tripdate, year):
         assert False, tripdate
     return datetime.date(year, month, day)
 
-# 2007, 2008, 2006
+# 2006, 2008 - 2010
 def Parselogwikitxt(year, expedition, txt):
-    trippara = re.findall("===(.*?)===([\s\S]*?)(?====)", txt)
+    trippara = re.findall(r"===(.*?)===([\s\S]*?)(?====)", txt)
     for triphead, triptext in trippara:
         tripheadp = triphead.split("|")
         #print "ttt", tripheadp
@@ -126,7 +134,7 @@ def Parselogwikitxt(year, expedition, txt):
         tripsplace = tripplace.split(" - ")
         tripcave = tripsplace[0].strip()
 
-        tul = re.findall("T/?U:?\s*(\d+(?:\.\d*)?|unknown)\s*(hrs|hours)?", triptext)
+        tul = re.findall(r"T/?U:?\s*(\d+(?:\.\d*)?|unknown)\s*(hrs|hours)?", triptext)
         if tul:
             #assert len(tul) <= 1, (triphead, triptext)
             #assert tul[0][1] in ["hrs", "hours"], (triphead, triptext)
@@ -140,12 +148,16 @@ def Parselogwikitxt(year, expedition, txt):
         #print "\n", tripcave, "---   ppp", trippeople, len(triptext)
         EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
 
-# 2002, 2004, 2005
+# 2002, 2004, 2005, 2007, 2011 - 2018
 def Parseloghtmltxt(year, expedition, txt):
-    tripparas = re.findall("<hr\s*/>([\s\S]*?)(?=<hr)", txt)
+    #print(" - Starting log html parser")
+    tripparas = re.findall(r"<hr\s*/>([\s\S]*?)(?=<hr)", txt)
+    logbook_entry_count = 0
     for trippara in tripparas:
+        #print(" - HR detected - maybe a trip?")
+        logbook_entry_count += 1
         
-        s = re.match('''(?x)(?:\s*<div\sclass="tripdate"\sid=".*?">.*?</div>\s*<p>)?  # second date
+        s = re.match(r'''(?x)(?:\s*<div\sclass="tripdate"\sid=".*?">.*?</div>\s*<p>)?  # second date
                             \s*(?:<a\s+id="(.*?)"\s*/>\s*</a>)?
                             \s*<div\s+class="tripdate"\s*(?:id="(.*?)")?>(.*?)</div>(?:<p>)?
                             \s*<div\s+class="trippeople">\s*(.*?)</div>
@@ -155,38 +167,41 @@ def Parseloghtmltxt(year, expedition, txt):
                             \s*$
                      ''', trippara)
         if not s:
-            if not re.search("Rigging Guide", trippara):
+            if not re.search(r"Rigging Guide", trippara):
-                print "can't parse: ", trippara  # this is 2007 which needs editing
+                print("can't parse: ", trippara)  # this is 2007 which needs editing
             #assert s, trippara
             continue
-
         tripid, tripid1, tripdate, trippeople, triptitle, triptext, tu = s.groups()
         ldate = ParseDate(tripdate.strip(), year)
         #assert tripid[:-1] == "t" + tripdate, (tripid, tripdate)
-        trippeople = re.sub("Ol(?!l)", "Olly", trippeople)        
+        #trippeople = re.sub(r"Ol(?!l)", "Olly", trippeople)
-        trippeople = re.sub("Wook(?!e)", "Wookey", trippeople)        
+        #trippeople = re.sub(r"Wook(?!e)", "Wookey", trippeople)
         triptitles = triptitle.split(" - ")
         if len(triptitles) >= 2:
             tripcave = triptitles[0]
         else:
             tripcave = "UNKNOWN"
-        #print "\n", tripcave, "---   ppp", trippeople, len(triptext)
+        #print("\n", tripcave, "---   ppp", trippeople, len(triptext))
-        ltriptext = re.sub("</p>", "", triptext)
+        ltriptext = re.sub(r"</p>", "", triptext)
-        ltriptext = re.sub("\s*?\n\s*", " ", ltriptext)
+        ltriptext = re.sub(r"\s*?\n\s*", " ", ltriptext)
-        ltriptext = re.sub("<p>", "\n\n", ltriptext).strip()
+        ltriptext = re.sub(r"<p>", "</br></br>", ltriptext).strip()
-        EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
+        EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext,
+                          trippeople=trippeople, expedition=expedition, logtime_underground=0,
+                          entry_type="html")
+    if logbook_entry_count == 0:
+        print(" - No trip entrys found in logbook, check the syntax matches htmltxt format")
 
 
-# main parser for pre-2001.  simpler because the data has been hacked so much to fit it
+# main parser for 1991 - 2001.  simpler because the data has been hacked so much to fit it
 def Parseloghtml01(year, expedition, txt):
-    tripparas = re.findall("<hr[\s/]*>([\s\S]*?)(?=<hr)", txt)
+    tripparas = re.findall(r"<hr[\s/]*>([\s\S]*?)(?=<hr)", txt)
     for trippara in tripparas:
         s = re.match(u"(?s)\s*(?:<p>)?(.*?)</?p>(.*)$(?i)", trippara)
         assert s, trippara[:300]
         tripheader, triptext = s.group(1), s.group(2)
-        mtripid = re.search('<a id="(.*?)"', tripheader)
+        mtripid = re.search(r'<a id="(.*?)"', tripheader)
         tripid = mtripid and mtripid.group(1) or ""
-        tripheader = re.sub("</?(?:[ab]|span)[^>]*>", "", tripheader)
+        tripheader = re.sub(r"</?(?:[ab]|span)[^>]*>", "", tripheader)
 
         #print "   ", [tripheader]
         #continue
@@ -194,7 +209,7 @@ def Parseloghtml01(year, expedition, txt):
         tripdate, triptitle, trippeople = tripheader.split("|")
         ldate = ParseDate(tripdate.strip(), year)
     
-        mtu = re.search('<p[^>]*>(T/?U.*)', triptext)
+        mtu = re.search(r'<p[^>]*>(T/?U.*)', triptext)
         if mtu:
             tu = mtu.group(1)
             triptext = triptext[:mtu.start(0)] + triptext[mtu.end():]
@@ -206,38 +221,40 @@ def Parseloghtml01(year, expedition, txt):
 
         ltriptext = triptext
         
-        mtail = re.search('(?:<a href="[^"]*">[^<]*</a>|\s|/|-|&amp;|</?p>|\((?:same day|\d+)\))*$', ltriptext)
+        mtail = re.search(r'(?:<a href="[^"]*">[^<]*</a>|\s|/|-|&amp;|</?p>|\((?:same day|\d+)\))*$', ltriptext)
         if mtail:
             #print mtail.group(0)
             ltriptext = ltriptext[:mtail.start(0)]
-        ltriptext = re.sub("</p>", "", ltriptext)
+        ltriptext = re.sub(r"</p>", "", ltriptext)
-        ltriptext = re.sub("\s*?\n\s*", " ", ltriptext)
+        ltriptext = re.sub(r"\s*?\n\s*", " ", ltriptext)
-        ltriptext = re.sub("<p>|<br>", "\n\n", ltriptext).strip()
+        ltriptext = re.sub(r"<p>|<br>", "\n\n", ltriptext).strip()
         #ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!]", "NONASCII", ltriptext)
-        ltriptext = re.sub("</?u>", "_", ltriptext)
+        ltriptext = re.sub(r"</?u>", "_", ltriptext)
-        ltriptext = re.sub("</?i>", "''", ltriptext)
+        ltriptext = re.sub(r"</?i>", "''", ltriptext)
-        ltriptext = re.sub("</?b>", "'''", ltriptext)
+        ltriptext = re.sub(r"</?b>", "'''", ltriptext)
         
 
         #print ldate, trippeople.strip()
             # could includ the tripid (url link for cross referencing)
-        EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
+        EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext,
-
+                          trippeople=trippeople, expedition=expedition, logtime_underground=0,
+                          entry_type="html")
 
+# parser for 2003
 def Parseloghtml03(year, expedition, txt):
-    tripparas = re.findall("<hr\s*/>([\s\S]*?)(?=<hr)", txt)
+    tripparas = re.findall(r"<hr\s*/>([\s\S]*?)(?=<hr)", txt)
     for trippara in tripparas:
         s = re.match(u"(?s)\s*<p>(.*?)</p>(.*)$", trippara)
         assert s, trippara
         tripheader, triptext = s.group(1), s.group(2)
-        tripheader = re.sub("&nbsp;", " ", tripheader)
+        tripheader = re.sub(r"&nbsp;", " ", tripheader)
-        tripheader = re.sub("\s+", " ", tripheader).strip()
+        tripheader = re.sub(r"\s+", " ", tripheader).strip()
         sheader = tripheader.split(" -- ")
         tu = ""
         if re.match("T/U|Time underwater", sheader[-1]):
             tu = sheader.pop()
         if len(sheader) != 3:
-            print "header not three pieces", sheader
+            print("header not three pieces", sheader)
         tripdate, triptitle, trippeople = sheader
         ldate = ParseDate(tripdate.strip(), year)
         triptitles = triptitle.split(" , ")
@@ -246,37 +263,14 @@ def Parseloghtml03(year, expedition, txt):
         else:
             tripcave = "UNKNOWN"
         #print tripcave, "---   ppp", triptitle, trippeople, len(triptext)
-        ltriptext = re.sub("</p>", "", triptext)
+        ltriptext = re.sub(r"</p>", "", triptext)
-        ltriptext = re.sub("\s*?\n\s*", " ", ltriptext)
+        ltriptext = re.sub(r"\s*?\n\s*", " ", ltriptext)
-        ltriptext = re.sub("<p>", "\n\n", ltriptext).strip()
+        ltriptext = re.sub(r"<p>", "\n\n", ltriptext).strip()
-        ltriptext = re.sub("[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext)
+        ltriptext = re.sub(r"[^\s0-9a-zA-Z\-.,:;'!&()\[\]<>?=+*%]", "_NONASCII_", ltriptext)
-        EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle, text = ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
+        EnterLogIntoDbase(date = ldate, place = tripcave, title = triptitle,
+                          text = ltriptext, trippeople=trippeople, expedition=expedition,
+                          logtime_underground=0, entry_type="html")
 
-yearlinks = [ 
-#                ("2013", "2013/logbook.html", Parseloghtmltxt), 
-                ("2012", "2012/logbook.html", Parseloghtmltxt), 
-                ("2011", "2011/logbook.html", Parseloghtmltxt), 
-                ("2010", "2010/logbook.html", Parselogwikitxt), 
-                ("2009", "2009/2009logbook.txt", Parselogwikitxt), 
-                ("2008", "2008/2008logbook.txt", Parselogwikitxt), 
-                ("2007", "2007/logbook.html", Parseloghtmltxt), 
-                ("2006", "2006/logbook/logbook_06.txt", Parselogwikitxt), 
-                ("2005", "2005/logbook.html", Parseloghtmltxt), 
-                ("2004", "2004/logbook.html", Parseloghtmltxt), 
-                ("2003", "2003/logbook.html", Parseloghtml03), 
-                ("2002", "2002/logbook.html", Parseloghtmltxt), 
-                ("2001", "2001/log.htm", Parseloghtml01), 
-                ("2000", "2000/log.htm", Parseloghtml01), 
-                ("1999", "1999/log.htm", Parseloghtml01), 
-                ("1998", "1998/log.htm", Parseloghtml01), 
-                ("1997", "1997/log.htm", Parseloghtml01), 
-                ("1996", "1996/log.htm", Parseloghtml01),
-                ("1995", "1995/log.htm", Parseloghtml01), 
-                ("1994", "1994/log.htm", Parseloghtml01), 
-                ("1993", "1993/log.htm", Parseloghtml01), 		
-                ("1992", "1992/log.htm", Parseloghtml01), 		
-                ("1991", "1991/log.htm", Parseloghtml01), 		
-            ]
 
 def SetDatesFromLogbookEntries(expedition):
     """
@@ -295,54 +289,67 @@ def SetDatesFromLogbookEntries(expedition):
             persontrip.persontrip_next = None
             lprevpersontrip = persontrip
             persontrip.save()
-            
+
-        
+
-        
 def LoadLogbookForExpedition(expedition):
     """ Parses all logbook entries for one expedition """
         
-    expowebbase = os.path.join(settings.EXPOWEB, "years")  
+    expowebbase = os.path.join(settings.EXPOWEB, "years")
-    year = str(expedition.year)
+    yearlinks = settings.LOGBOOK_PARSER_SETTINGS
-    for lyear, lloc, parsefunc in yearlinks:
+
-        if lyear == year:
+    logbook_parseable = False
-            break
+
-    fin = open(os.path.join(expowebbase, lloc))
+    if expedition.year in yearlinks:
-    print "opennning", lloc
+        year_settings = yearlinks[expedition.year]
-    txt = fin.read().decode("latin1")
+        file_in = open(os.path.join(expowebbase, year_settings[0]))
-    fin.close()
+        txt = file_in.read().decode("latin1")
-    parsefunc(year, expedition, txt)
+        file_in.close()
-    SetDatesFromLogbookEntries(expedition)
+        parsefunc = year_settings[1]
-    return "TOLOAD: " + year + "  " + str(expedition.personexpedition_set.all()[1].logbookentry_set.count()) + "  " + str(models.PersonTrip.objects.filter(personexpedition__expedition=expedition).count())
+        logbook_parseable = True
+        print(" - Parsing logbook: " + year_settings[0] + "\n - Using parser: " + year_settings[1])
+    else:
+        try:
+            file_in = open(os.path.join(expowebbase, expedition.year, settings.DEFAULT_LOGBOOK_FILE))
+            txt = file_in.read().decode("latin1")
+            file_in.close()
+            logbook_parseable = True
+            print("No set parser found using default")
+            parsefunc = settings.DEFAULT_LOGBOOK_PARSER
+        except (IOError):
+            logbook_parseable = False
+            print("Couldn't open default logbook file and nothing in settings for expo " + expedition.year)
+
+    if logbook_parseable:
+        parser = globals()[parsefunc]
+        parser(expedition.year, expedition, txt)
+        SetDatesFromLogbookEntries(expedition)
+
+    #return "TOLOAD: " + year + "  " + str(expedition.personexpedition_set.all()[1].logbookentry_set.count()) + "  " + str(models.PersonTrip.objects.filter(personexpedition__expedition=expedition).count())
 
 
 def LoadLogbooks():
-    """ This is the master function for parsing all logbooks into the Troggle database. Requires yearlinks, which is a list of tuples for each expedition with expedition year, logbook path, and parsing function. """
+    """ This is the master function for parsing all logbooks into the Troggle database. """
-    
-    #Deletion has been moved to a seperate function to enable the non-destructive importing
-    #models.LogbookEntry.objects.all().delete()
-    expowebbase = os.path.join(settings.EXPOWEB, "years")  
-    #yearlinks = [ ("2001", "2001/log.htm", Parseloghtml01), ] #overwrite
-    #yearlinks = [ ("1996", "1996/log.htm", Parseloghtml01),] # overwrite
 
-    for year, lloc, parsefunc in yearlinks:
+    # Clear the logbook data issues as we are reloading
-        # This will not work until the corresponding year exists in the database. 
+    models.DataIssue.objects.filter(parser='logbooks').delete()
-        # In 2012 this needed noscript/folk.csv to be updated first.
+    # Fetch all expos
-        expedition = models.Expedition.objects.filter(year = year)[0]
+    expos = models.Expedition.objects.all()
-        fin = open(os.path.join(expowebbase, lloc))
+    for expo in expos:
-        txt = fin.read().decode("latin1")
+        print("\nLoading Logbook for: " + expo.year)
-        fin.close()
+        
-        parsefunc(year, expedition, txt)
+        # Load logbook for expo
-        SetDatesFromLogbookEntries(expedition)
+        LoadLogbookForExpedition(expo)
 
-dateRegex = re.compile('<span\s+class="date">(\d\d\d\d)-(\d\d)-(\d\d)</span>', re.S)
+
-expeditionYearRegex = re.compile('<span\s+class="expeditionyear">(.*?)</span>', re.S)
+dateRegex = re.compile(r'<span\s+class="date">(\d\d\d\d)-(\d\d)-(\d\d)</span>', re.S)
-titleRegex = re.compile('<H1>(.*?)</H1>', re.S)
+expeditionYearRegex = re.compile(r'<span\s+class="expeditionyear">(.*?)</span>', re.S)
-reportRegex = re.compile('<div\s+class="report">(.*)</div>\s*</body>', re.S)
+titleRegex = re.compile(r'<H1>(.*?)</H1>', re.S)
-personRegex = re.compile('<div\s+class="person">(.*?)</div>', re.S)
+reportRegex = re.compile(r'<div\s+class="report">(.*)</div>\s*</body>', re.S)
-nameAuthorRegex = re.compile('<span\s+class="name(,author|)">(.*?)</span>', re.S)
+personRegex = re.compile(r'<div\s+class="person">(.*?)</div>', re.S)
-TURegex = re.compile('<span\s+class="TU">([0-9]*\.?[0-9]+)</span>', re.S)
+nameAuthorRegex = re.compile(r'<span\s+class="name(,author|)">(.*?)</span>', re.S)
-locationRegex = re.compile('<span\s+class="location">(.*?)</span>', re.S)
+TURegex = re.compile(r'<span\s+class="TU">([0-9]*\.?[0-9]+)</span>', re.S)
-caveRegex = re.compile('<span\s+class="cave">(.*?)</span>', re.S)
+locationRegex = re.compile(r'<span\s+class="location">(.*?)</span>', re.S)
+caveRegex = re.compile(r'<span\s+class="cave">(.*?)</span>', re.S)
 
 def parseAutoLogBookEntry(filename):
     errors = []
@@ -435,4 +442,4 @@ def parseAutoLogBookEntry(filename):
                           time_underground = TU, 
                           logbook_entry = logbookEntry, 
                           is_logbook_entry_author = author).save()
-    print logbookEntry
+    print(logbookEntry)

parsers/people.py

@@ -4,6 +4,8 @@ from django.conf import settings
 import troggle.core.models as models
 import csv, re, datetime, os, shutil
 from utils import save_carefully
+from HTMLParser import HTMLParser
+from unidecode import unidecode
 
 def saveMugShot(mugShotPath, mugShotFilename, person):
     if mugShotFilename.startswith(r'i/'): #if filename in cell has the directory attached (I think they all do), remove it
@@ -44,13 +46,13 @@ def parseMugShotAndBlurb(personline, header, person):
 
 def LoadPersonsExpos():
     
-    persontab = open(os.path.join(settings.EXPOWEB, "noinfo", "folk.csv"))
+    persontab = open(os.path.join(settings.EXPOWEB, "folk", "folk.csv"))
     personreader = csv.reader(persontab)
     headers = personreader.next()
     header = dict(zip(headers, range(len(headers))))
     
     # make expeditions
-    print "Loading expeditions"
+    print("Loading expeditions")
     years = headers[5:]
     
     for year in years:
@@ -59,22 +61,35 @@ def LoadPersonsExpos():
         
         save_carefully(models.Expedition, lookupAttribs, nonLookupAttribs)
 
-    
     # make persons
-    print "Loading personexpeditions"
+    print("Loading personexpeditions")
-    #expoers2008 = """Edvin Deadman,Kathryn Hopkins,Djuke Veldhuis,Becka Lawson,Julian Todd,Natalie Uomini,Aaron Curtis,Tony Rooke,Ollie Stevens,Frank Tully,Martin Jahnke,Mark Shinwell,Jess Stirrups,Nial Peters,Serena Povia,Olly Madge,Steve Jones,Pete Harley,Eeva Makiranta,Keith Curtis""".split(",")
-    #expomissing = set(expoers2008)
 
     for personline in personreader:
         name = personline[header["Name"]]
-        name = re.sub("<.*?>", "", name)
+        name = re.sub(r"<.*?>", "", name)
-        mname = re.match("(\w+)(?:\s((?:van |ten )?\w+))?(?:\s\(([^)]*)\))?", name)
+
-        nickname = mname.group(3) or ""
+        firstname = ""
-	
+        nickname = ""
-        lookupAttribs={'first_name':mname.group(1), 'last_name':(mname.group(2) or "")}
+
-        nonLookupAttribs={'is_vfho':personline[header["VfHO member"]],}
+        rawlastname = personline[header["Lastname"]].strip()
+        matchlastname = re.match(r"^([\w&;\s]+)(?:\(([^)]*)\))?", rawlastname)
+        lastname = matchlastname.group(1).strip()
+
+        splitnick = re.match(r"^([\w&;\s]+)(?:\(([^)]*)\))?", name)
+        fullname = splitnick.group(1)
+
+        nickname = splitnick.group(2) or ""
+
+        fullname = fullname.strip()
+        names = fullname.split(' ')
+        firstname = names[0]
+        if len(names) == 1:
+            lastname = ""
+
+        lookupAttribs={'first_name':firstname, 'last_name':(lastname or "")}
+        nonLookupAttribs={'is_vfho':personline[header["VfHO member"]], 'fullname':fullname}
         person, created = save_carefully(models.Person, lookupAttribs, nonLookupAttribs)
-	
+
         parseMugShotAndBlurb(personline=personline, header=header, person=person)
     
         # make person expedition from table
@@ -88,6 +103,8 @@ def LoadPersonsExpos():
 
     # this fills in those people for whom 2008 was their first expo
     #print "Loading personexpeditions 2008"
+    #expoers2008 = """Edvin Deadman,Kathryn Hopkins,Djuke Veldhuis,Becka Lawson,Julian Todd,Natalie Uomini,Aaron Curtis,Tony Rooke,Ollie Stevens,Frank Tully,Martin Jahnke,Mark Shinwell,Jess Stirrups,Nial Peters,Serena Povia,Olly Madge,Steve Jones,Pete Harley,Eeva Makiranta,Keith Curtis""".split(",")
+    #expomissing = set(expoers2008)
     #for name in expomissing:
         # firstname, lastname = name.split()
         # is_guest = name in ["Eeva Makiranta", "Keith Curtis"]
@@ -103,18 +120,6 @@ def LoadPersonsExpos():
         # personexpedition = models.PersonExpedition(person=person, expedition=expedition, nickname="", is_guest=is_guest)
         # personexpedition.save()
 
-    #Notability is now a method of person. Makes no sense to store it in the database; it would need to be recalculated every time something changes. - AC 16 Feb 09
-    # could rank according to surveying as well
-    #print "Setting person notability"
-    #for person in models.Person.objects.all():
-        #person.notability = 0.0
-        #for personexpedition in person.personexpedition_set.all():
-            #if not personexpedition.is_guest:
-                #person.notability += 1.0 / (2012 - int(personexpedition.expedition.year))
-        #person.bisnotable = person.notability > 0.3 # I don't know how to filter by this
-        #person.save()
-        
-        
 # used in other referencing parser functions
 # expedition name lookup cached for speed (it's a very big list)
 Gpersonexpeditionnamelookup = { }
@@ -127,20 +132,33 @@ def GetPersonExpeditionNameLookup(expedition):
     res = { }
     duplicates = set()
     
-    print "Calculating GetPersonExpeditionNameLookup for", expedition.year
+    print("Calculating GetPersonExpeditionNameLookup for " + expedition.year)
     personexpeditions = models.PersonExpedition.objects.filter(expedition=expedition)
+    htmlparser = HTMLParser()
     for personexpedition in personexpeditions:
         possnames = [ ]
-        f = personexpedition.person.first_name.lower()
+        f = unidecode(htmlparser.unescape(personexpedition.person.first_name.lower()))
-        l = personexpedition.person.last_name.lower()
+        l = unidecode(htmlparser.unescape(personexpedition.person.last_name.lower()))
+        full = unidecode(htmlparser.unescape(personexpedition.person.fullname.lower()))
         if l:
             possnames.append(f + " " + l)
             possnames.append(f + " " + l[0])
             possnames.append(f + l[0])
             possnames.append(f[0] + " " + l)
         possnames.append(f)
-        if personexpedition.nickname:
+        if full not in possnames:
+            possnames.append(full)
+        if personexpedition.nickname not in possnames:
             possnames.append(personexpedition.nickname.lower())
+            if l:
+                # This allows for nickname to be used for short name eg Phil
+                # adding Phil Sargent to the list
+                if str(personexpedition.nickname.lower() + " " + l) not in possnames:
+                    possnames.append(personexpedition.nickname.lower() + " " + l)
+                if str(personexpedition.nickname.lower() + " " + l[0]) not in possnames:
+                    possnames.append(personexpedition.nickname.lower() + " " + l[0])
+                if str(personexpedition.nickname.lower() + l[0]) not in possnames:
+                    possnames.append(personexpedition.nickname.lower() + l[0])
         
         for possname in possnames:
             if possname in res:

parsers/survex.py

@@ -5,38 +5,43 @@ import troggle.settings as settings
 from subprocess import call, Popen, PIPE
 
 from troggle.parsers.people import GetPersonExpeditionNameLookup
+from django.utils.timezone import get_current_timezone
+from django.utils.timezone import make_aware
+
 import re
 import os
+from datetime import datetime
 
+line_leg_regex = re.compile(r"[\d\-+.]+$")
 
-
+def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
-def LoadSurvexLineLeg(survexblock, stardata, sline, comment):
+    # The try catches here need replacing as they are relativly expensive
     ls = sline.lower().split()
     ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]])
     ssto = survexblock.MakeSurvexStation(ls[stardata["to"]])
-    
+
     survexleg = models.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto)
     if stardata["type"] == "normal":
         try:
             survexleg.tape = float(ls[stardata["tape"]])
-        except ValueError: 
+        except ValueError:
-            print "Tape misread in", survexblock.survexfile.path
+            print("Tape misread in", survexblock.survexfile.path)
-            print "Stardata:", stardata
+            print("Stardata:", stardata)
-            print "Line:", ls
+            print("Line:", ls)
             survexleg.tape = 1000
         try:
             lclino = ls[stardata["clino"]]
         except:
-            print "Clino misread in", survexblock.survexfile.path
+            print("Clino misread in", survexblock.survexfile.path)
-            print "Stardata:", stardata
+            print("Stardata:", stardata)
-            print "Line:", ls
+            print("Line:", ls)
             lclino = error
         try:
             lcompass = ls[stardata["compass"]]
         except:
-            print "Compass misread in", survexblock.survexfile.path
+            print("Compass misread in", survexblock.survexfile.path)
-            print "Stardata:", stardata
+            print("Stardata:", stardata)
-            print "Line:", ls
+            print("Line:", ls)
             lcompass = error
         if lclino == "up":
             survexleg.compass = 0.0
@@ -48,28 +53,32 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment):
             try:
                 survexleg.compass = float(lcompass)
             except ValueError:
-                print "Compass misread in", survexblock.survexfile.path
+                print("Compass misread in", survexblock.survexfile.path)
-                print "Stardata:", stardata
+                print("Stardata:", stardata)
-                print "Line:", ls
+                print("Line:", ls)
                 survexleg.compass = 1000
             survexleg.clino = -90.0
         else:
-            assert re.match("[\d\-+.]+$", lcompass), ls
+            assert line_leg_regex.match(lcompass), ls
-            assert re.match("[\d\-+.]+$", lclino) and lclino != "-", ls
+            assert line_leg_regex.match(lclino) and lclino != "-", ls
             survexleg.compass = float(lcompass)
             survexleg.clino = float(lclino)
-        
+
+        if cave:
+            survexleg.cave = cave
+
         # only save proper legs
         survexleg.save()
-    
+
     itape = stardata.get("tape")
     if itape:
         try:
             survexblock.totalleglength += float(ls[itape])
         except ValueError:
-            print "Length not added"
+            print("Length not added")
         survexblock.save()
-        
+
+
 def LoadSurvexEquate(survexblock, sline):
     #print sline #
     stations = sline.split()
@@ -77,98 +86,215 @@ def LoadSurvexEquate(survexblock, sline):
     for station in stations:
         survexblock.MakeSurvexStation(station)
 
+
 def LoadSurvexLinePassage(survexblock, stardata, sline, comment):
     pass
-    
 
-stardatadefault = { "type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4 }
+stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4}
-stardataparamconvert = { "length":"tape", "bearing":"compass", "gradient":"clino" }
+stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"}
+
+regex_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$")
+regex_ref     = re.compile(r'.*?ref.*?(\d+)\s*#\s*(\d+)')
+regex_star    = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$')
+regex_team    = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)")
+regex_team_member        = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)")
+regex_qm      = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')
 
 def RecursiveLoad(survexblock, survexfile, fin, textlines):
     iblankbegins = 0
     text = [ ]
     stardata = stardatadefault
     teammembers = [ ]
- 
+
-# uncomment to print out all files during parsing 
+    # uncomment to print out all files during parsing 
-#    print "Reading file:", survexblock.survexfile.path
+    print("  - Reading file: " + survexblock.survexfile.path)
-    while True:
+    stamp = datetime.now()
-        svxline = fin.readline().decode("latin1")
+    lineno = 0
-        if not svxline:
+
-            return
+    # Try to find the cave in the DB if not use the string as before
-        textlines.append(svxline)
+    path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", survexblock.survexfile.path)
-        
+    if path_match:
+        pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
+        # print('Match')
+        # print(pos_cave)
+        cave = models.getCaveByReference(pos_cave)
+        if cave:
+            survexfile.cave = cave
+    svxlines = ''
+    svxlines = fin.read().splitlines()
+    # print('Cave - preloop ' + str(survexfile.cave))
+    # print(survexblock)
+    for svxline in svxlines:
+
+        # print(survexblock)
+
+        # print(svxline)
+        # if not svxline:
+        #     print('   - Not survex')
+        #     return
+        # textlines.append(svxline)
+
+        lineno += 1
+
+        # print('   - Line: %d' % lineno)
+
         # break the line at the comment
-        sline, comment = re.match("([^;]*?)\s*(?:;\s*(.*))?\n?$", svxline.strip()).groups()
+        sline, comment = regex_comment.match(svxline.strip()).groups()
-        
         # detect ref line pointing to the scans directory
-        mref = comment and re.match('.*?ref.*?(\d+)\s*#\s*(\d+)', comment)
+        mref = comment and regex_ref.match(comment)
         if mref:
             refscan = "%s#%s" % (mref.group(1), mref.group(2))
             survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan)
             if survexscansfolders:
                 survexblock.survexscansfolder = survexscansfolders[0]
                 #survexblock.refscandir = "%s/%s%%23%s" % (mref.group(1), mref.group(1), mref.group(2))
-                survexblock.save()   
+                survexblock.save()
             continue
-        
+
+        # This whole section should be moved if we can have *QM become a proper survex command
+        # Spec of QM in SVX files, currently commented out need to add to survex
+        # needs to match regex_qm
+        # ;Serial number   grade(A/B/C/D/X)  nearest-station  resolution-station description
+        # ;QM1	a	hobnob_hallway_2.42	hobnob-hallway_3.42	junction of keyhole passage
+        # ;QM1	a	hobnob_hallway_2.42	-	junction of keyhole passage
+        qmline = comment and regex_qm.match(comment)
+        if qmline:
+            print(qmline.groups())
+            #(u'1', u'B', u'miraclemaze', u'1.17', u'-', None, u'\tcontinuation of rift')
+            qm_no = qmline.group(1)
+            qm_grade = qmline.group(2)
+            qm_from_section = qmline.group(3)
+            qm_from_station = qmline.group(4)
+            qm_resolve_section = qmline.group(6)
+            qm_resolve_station = qmline.group(7)
+            qm_notes = qmline.group(8)
+
+            print('Cave - %s' % survexfile.cave)
+            print('QM no %d' % int(qm_no))
+            print('QM grade %s' % qm_grade)
+            print('QM section %s' % qm_from_section)
+            print('QM station %s' % qm_from_station)
+            print('QM res section %s' % qm_resolve_section)
+            print('QM res station %s' % qm_resolve_station)
+            print('QM notes %s' % qm_notes)
+
+            # If the QM isn't resolved (has a resolving station) thn load it
+            if not qm_resolve_section or qm_resolve_section is not '-' or qm_resolve_section is not 'None':
+                from_section = models.SurvexBlock.objects.filter(name=qm_from_section)
+                # If we can find a section (survex note chunck, named)
+                if len(from_section) > 0:
+                    print(from_section[0])
+                    from_station = models.SurvexStation.objects.filter(block=from_section[0], name=qm_from_station)
+                    # If we can find a from station then we have the nearest station and can import it
+                    if len(from_station) > 0:
+                        print(from_station[0])
+                        qm = models.QM.objects.create(number=qm_no,
+                                                      nearest_station=from_station[0],
+                                                      grade=qm_grade.upper(),
+                                                      location_description=qm_notes)
+            else:
+                print('QM found but resolved')
+
+        #print('Cave -sline ' + str(cave))
         if not sline:
             continue
-        
+
         # detect the star command
-        mstar = re.match('\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$', sline)
+        mstar = regex_star.match(sline)
         if not mstar:
             if "from" in stardata:
-                LoadSurvexLineLeg(survexblock, stardata, sline, comment)
+                # print('Cave ' + str(survexfile.cave))
+                # print(survexblock)
+                LoadSurvexLineLeg(survexblock, stardata, sline, comment, survexfile.cave)
+                # print('   - From: ')
+                #print(stardata)
+                pass
             elif stardata["type"] == "passage":
                 LoadSurvexLinePassage(survexblock, stardata, sline, comment)
+                # print('   - Passage: ')
             #Missing "station" in stardata.
             continue
-        
+
         # detect the star command
         cmd, line = mstar.groups()
         cmd = cmd.lower()
         if re.match("include$(?i)", cmd):
-            includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub("\.svx$", "", line))
+            includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line))
-            includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave)
+            print('   - Include file found including - ' + includepath)
+            # Try to find the cave in the DB if not use the string as before
+            path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
+            if path_match:
+                pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
+                # print(pos_cave)
+                cave = models.getCaveByReference(pos_cave)
+                if cave:
+                    survexfile.cave = cave
+            else:
+                print('No match for %s' % includepath)
+            includesurvexfile = models.SurvexFile(path=includepath)
             includesurvexfile.save()
             includesurvexfile.SetDirectory()
             if includesurvexfile.exists():
+                survexblock.save()
                 fininclude = includesurvexfile.OpenFile()
                 RecursiveLoad(survexblock, includesurvexfile, fininclude, textlines)
-        
+
         elif re.match("begin$(?i)", cmd):
-            if line: 
+            if line:
+                newsvxpath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line))
+                # Try to find the cave in the DB if not use the string as before
+                path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", newsvxpath)
+                if path_match:
+                    pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
+                    print(pos_cave)
+                    cave = models.getCaveByReference(pos_cave)
+                    if cave:
+                        survexfile.cave = cave
+                else:
+                    print('No match for %s' % newsvxpath)
+
                 name = line.lower()
-                survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile, totalleglength=0.0)
+                print('   - Begin found for: ' + name)
+                # print('Block cave: ' + str(survexfile.cave))
+                survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0)
                 survexblockdown.save()
+                survexblock.save()
+                survexblock = survexblockdown
+                # print(survexblockdown)
                 textlinesdown = [ ]
                 RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown)
             else:
                 iblankbegins += 1
-        
+
         elif re.match("end$(?i)", cmd):
             if iblankbegins:
                 iblankbegins -= 1
             else:
                 survexblock.text = "".join(textlines)
                 survexblock.save()
+                # print('   - End found: ')
+                endstamp = datetime.now()
+                timetaken = endstamp - stamp
+                # print('   - Time to process: ' + str(timetaken))
                 return
-        
+
         elif re.match("date$(?i)", cmd):
             if len(line) == 10:
-                survexblock.date = re.sub("\.", "-", line)
+                #print('   - Date found: ' + line)
+                survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone())
                 expeditions = models.Expedition.objects.filter(year=line[:4])
                 if expeditions:
                     assert len(expeditions) == 1
                     survexblock.expedition = expeditions[0]
                     survexblock.expeditionday = survexblock.expedition.get_expedition_day(survexblock.date)
                     survexblock.save()
-                    
+
         elif re.match("team$(?i)", cmd):
-            mteammember = re.match("(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line)
+            pass
+            # print('   - Team found: ')
+            mteammember = regex_team.match(line)
             if mteammember:
-                for tm in re.split(" and | / |, | & | \+ |^both$|^none$(?i)", mteammember.group(2)):
+                for tm in regex_team_member.split(mteammember.group(2)):
                     if tm:
                         personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
                         if (personexpedition, tm) not in teammembers:
@@ -178,18 +304,23 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
                             if personexpedition:
                                 personrole.person=personexpedition.person
                             personrole.save()
-                            
+
         elif cmd == "title":
-            survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave)
+            #print('   - Title found: ')
+            survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexfile.cave)
             survextitle.save()
-            
+            pass
+
         elif cmd == "require":
             # should we check survex version available for processing?
             pass
 
         elif cmd == "data":
+            #print('   - Data found: ')
             ls = line.lower().split()
             stardata = { "type":ls[0] }
+            #print('    - Star data: ', stardata)
+            #print(ls)
             for i in range(0, len(ls)):
                 stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1
             if ls[0] in ["normal", "cartesian", "nosurvey"]:
@@ -198,41 +329,27 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
                 stardata = stardatadefault
             else:
                 assert ls[0] == "passage", line
-        
+
         elif cmd == "equate":
+            #print('   - Equate found: ')
             LoadSurvexEquate(survexblock, line)
 
         elif cmd == "fix":
+            #print('   - Fix found: ')
             survexblock.MakeSurvexStation(line.split()[0])
 
         else:
-            if not cmd in [ "sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument", "calibrate", "set", "infer", "alias", "ref" ]:
+            #print('   - Stuff')
-                print ("Unrecognised command in line:", cmd, line, survexblock)
+            if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument",
-        
+                           "calibrate", "set", "infer", "alias", "ref", "cs", "declination", "case"]:
-        
+                print("Unrecognised command in line:", cmd, line, survexblock, survexblock.survexfile.path)
-
+        endstamp = datetime.now()
-def ReloadSurvexCave(survex_cave):
+        timetaken = endstamp - stamp
-    cave = models.Cave.objects.get(kataster_number=survex_cave)
+        # print('   - Time to process: ' + str(timetaken))
-    cave.survexblock_set.all().delete()
-    cave.survexfile_set.all().delete()
-    cave.survexdirectory_set.all().delete()
-    
-    survexfile = models.SurvexFile(path="caves/" + survex_cave + "/" + survex_cave, cave=cave)
-    survexfile.save()
-    survexfile.SetDirectory()
-    
-    survexblockroot = models.SurvexBlock(name="root", survexpath="caves", begin_char=0, cave=cave, survexfile=survexfile, totalleglength=0.0)
-    survexblockroot.save()
-    fin = survexfile.OpenFile()
-    textlines = [ ]
-    RecursiveLoad(survexblockroot, survexfile, fin, textlines)
-    survexblockroot.text = "".join(textlines)
-    survexblockroot.save()
-
 
 def LoadAllSurvexBlocks():
 
-    print 'Loading All Survex Blocks...'
+    print('Loading All Survex Blocks...')
 
     models.SurvexBlock.objects.all().delete()
     models.SurvexFile.objects.all().delete()
@@ -243,7 +360,9 @@ def LoadAllSurvexBlocks():
     models.SurvexPersonRole.objects.all().delete()
     models.SurvexStation.objects.all().delete()
 
-    survexfile = models.SurvexFile(path="all", cave=None)
+    print(" - Data flushed")
+
+    survexfile = models.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
     survexfile.save()
     survexfile.SetDirectory()
 
@@ -252,31 +371,26 @@ def LoadAllSurvexBlocks():
     survexblockroot.save()
     fin = survexfile.OpenFile()
     textlines = [ ]
+    # The real work starts here
     RecursiveLoad(survexblockroot, survexfile, fin, textlines)
+    fin.close()
     survexblockroot.text = "".join(textlines)
     survexblockroot.save()
 
 
-    #Load each cave, 
+poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
-    #FIXME this should be dealt with load all above
+
-    caves = models.Cave.objects.all()
+
-    for cave in caves:
-        if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves", cave.kataster_number)):
-            if cave.kataster_number not in ['40']:
-                print "loading", cave
-                ReloadSurvexCave(cave.kataster_number)
-    
-poslineregex = re.compile("^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$")
 def LoadPos():
 
-    print 'Loading Pos....'
+    print('Loading Pos....')
 
-    call([settings.CAVERN, "--output=%s/all.3d" % settings.SURVEX_DATA, "%s/all.svx" % settings.SURVEX_DATA])
+    call([settings.CAVERN, "--output=%s%s.3d" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME), "%s%s.svx" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME)])
-    call([settings.THREEDTOPOS, '%sall.3d' % settings.SURVEX_DATA], cwd = settings.SURVEX_DATA)
+    call([settings.THREEDTOPOS, '%s%s.3d' % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME)], cwd = settings.SURVEX_DATA)
-    posfile = open("%sall.pos" % settings.SURVEX_DATA)
+    posfile = open("%s%s.pos" % (settings.SURVEX_DATA, settings.SURVEX_TOPNAME))
-    posfile.readline()#Drop header
+    posfile.readline() #Drop header
     for line in posfile.readlines():
-        r = poslineregex.match(line)                
+        r = poslineregex.match(line)
         if r:
             x, y, z, name = r.groups()
             try:
@@ -286,4 +400,4 @@ def LoadPos():
                 ss.z = float(z)
                 ss.save()
             except:
-                print "%s not parsed in survex" % name
+                print("%s not parsed in survex" % name)

parsers/surveys.py

@@ -39,7 +39,7 @@ def readSurveysFromCSV():
 
     # test if the expeditions have been added yet
     if Expedition.objects.count()==0:
-        print "There are no expeditions in the database. Please run the logbook parser."
+        print("There are no expeditions in the database. Please run the logbook parser.")
         sys.exit()
 
     
@@ -56,7 +56,7 @@ def readSurveysFromCSV():
     for survey in surveyreader:
         #I hate this, but some surveys have a letter eg 2000#34a. The next line deals with that.
         walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']]) 
-    #    print walletNumberLetter.groups()
+    #    print(walletNumberLetter.groups())
         year=survey[header['Year']]
 
         
@@ -89,63 +89,73 @@ def listdir(*directories):
 # add survey scans
 def parseSurveyScans(expedition, logfile=None):
 #    yearFileList = listdir(expedition.year)
-    yearPath=os.path.join(settings.SURVEY_SCANS, "surveyscans", expedition.year)
+    try:
-    yearFileList=os.listdir(yearPath)
+        yearPath=os.path.join(settings.SURVEY_SCANS, "surveyscans", expedition.year)
-    print yearFileList
+        yearFileList=os.listdir(yearPath)
-    for surveyFolder in yearFileList:
+        print(yearFileList)
-        try:
+        for surveyFolder in yearFileList:
-            surveyNumber=re.match(r'\d\d\d\d#0*(\d+)',surveyFolder).groups()
-#            scanList = listdir(expedition.year, surveyFolder)
-            scanList=os.listdir(os.path.join(yearPath,surveyFolder))
-        except AttributeError:
-            print surveyFolder + " ignored",
-            continue
-
-        for scan in scanList:
             try:
-                scanChopped=re.match(r'(?i).*(notes|elev|plan|elevation|extend)(\d*)\.(png|jpg|jpeg)',scan).groups()
+                surveyNumber=re.match(r'\d\d\d\d#(X?)0*(\d+)',surveyFolder).groups()
-                scanType,scanNumber,scanFormat=scanChopped
+                #scanList = listdir(expedition.year, surveyFolder)
+                scanList=os.listdir(os.path.join(yearPath,surveyFolder))
             except AttributeError:
-                print scan + " ignored \r",
+                print("Folder: " + surveyFolder + " ignored\r")
                 continue
-            if scanType == 'elev' or scanType == 'extend':
-                scanType = 'elevation'
 
-            if scanNumber=='':
+            for scan in scanList:
-                scanNumber=1
+                try:
+                    scanChopped=re.match(r'(?i).*(notes|elev|plan|elevation|extend)(\d*)\.(png|jpg|jpeg)',scan).groups()
+                    scanType,scanNumber,scanFormat=scanChopped
+                except AttributeError:
+                    print("File: " + scan + " ignored\r")
+                    continue
+                if scanType == 'elev' or scanType == 'extend':
+                    scanType = 'elevation'
 
-            if type(surveyNumber)==types.TupleType:
+                if scanNumber=='':
-                surveyNumber=surveyNumber[0]
+                    scanNumber=1
-            try:
+
-                placeholder=get_or_create_placeholder(year=int(expedition.year))
+                if type(surveyNumber)==types.TupleType:
-                survey=Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=expedition, defaults={'logbook_entry':placeholder})[0]
+                    surveyLetter=surveyNumber[0]
-            except Survey.MultipleObjectsReturned:
+                    surveyNumber=surveyNumber[1]
-                survey=Survey.objects.filter(wallet_number=surveyNumber, expedition=expedition)[0]
+                try:
-            file_=os.path.join(yearPath, surveyFolder, scan)
+                    placeholder=get_or_create_placeholder(year=int(expedition.year))
-            scanObj = ScannedImage(
+                    survey=Survey.objects.get_or_create(wallet_number=surveyNumber, wallet_letter=surveyLetter, expedition=expedition, defaults={'logbook_entry':placeholder})[0]
-                file=file_,
+                except Survey.MultipleObjectsReturned:
-                contents=scanType,
+                    survey=Survey.objects.filter(wallet_number=surveyNumber, wallet_letter=surveyLetter, expedition=expedition)[0]
-                number_in_wallet=scanNumber,
+                file_=os.path.join(yearPath, surveyFolder, scan)
-                survey=survey,
+                scanObj = ScannedImage(
-                new_since_parsing=False,
+                    file=file_,
-                )
+                    contents=scanType,
-            print "Added scanned image at " + str(scanObj)
+                    number_in_wallet=scanNumber,
-            #if scanFormat=="png":
+                    survey=survey,
-                #if isInterlacedPNG(os.path.join(settings.SURVEY_SCANS, "surveyscans", file_)):
+                    new_since_parsing=False,
-                #    print file_+ " is an interlaced PNG. No can do."
+                    )
-                #continue
+                print("Added scanned image at " + str(scanObj))
-            scanObj.save()
+                #if scanFormat=="png":
+                    #if isInterlacedPNG(os.path.join(settings.SURVEY_SCANS, "surveyscans", file_)):
+                    #    print file_+ " is an interlaced PNG. No can do."
+                    #continue
+                scanObj.save()
+    except (IOError, OSError):
+        yearPath=os.path.join(settings.SURVEY_SCANS, "surveyscans", expedition.year)
+        print("No folder found for " + expedition.year + " at:- " + yearPath)
 
 # dead
 def parseSurveys(logfile=None):
-    readSurveysFromCSV()
+    try:
+        readSurveysFromCSV()
+    except (IOError, OSError):
+        print("Survey CSV not found..")
+        pass
+    
     for expedition in Expedition.objects.filter(year__gte=2000):   #expos since 2000, because paths and filenames were nonstandard before then
         parseSurveyScans(expedition)
 
 # dead
 def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the thumbnail engine can't handle them (uses PIL)
     file=Image.open(filePath)
-    print filePath
+    print(filePath)
     if 'interlace' in file.info:
         return file.info['interlace']
     else:
@@ -164,9 +174,6 @@ def GetListDir(sdir):
                 ff = os.path.join(sdir, f)
                 res.append((f, ff, os.path.isdir(ff)))
     return res
-        
-        
-        
 
 
 def LoadListScansFile(survexscansfolder):
@@ -180,8 +187,8 @@ def LoadListScansFile(survexscansfolder):
             gld.append((fyf, ffyf, fisdiryf))
     
     for (fyf, ffyf, fisdiryf) in gld:
-        assert not fisdiryf, ffyf
+        #assert not fisdiryf, ffyf
-        if re.search("\.(?:png|jpg|jpeg)(?i)$", fyf):
+        if re.search(r"\.(?:png|jpg|jpeg)(?i)$", fyf):
             survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)
             survexscansingle.save()
 
@@ -190,7 +197,7 @@ def LoadListScansFile(survexscansfolder):
 # and builds up the models we can access later
 def LoadListScans():
 
-    print 'Loading Survey Scans...'
+    print('Loading Survey Scans...')
 
     SurvexScanSingle.objects.all().delete()
     SurvexScansFolder.objects.all().delete()
@@ -208,7 +215,7 @@ def LoadListScans():
             continue
         
         # do the year folders
-        if re.match("\d\d\d\d$", f):
+        if re.match(r"\d\d\d\d$", f):
             for fy, ffy, fisdiry in GetListDir(ff):
                 if fisdiry:
                     assert fisdiry, ffy
@@ -225,7 +232,7 @@ def LoadListScans():
 
 def FindTunnelScan(tunnelfile, path):
     scansfolder, scansfile = None, None
-    mscansdir = re.search("(\d\d\d\d#\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg))$", path)
+    mscansdir = re.search(r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg))$", path)
     if mscansdir:
         scansfolderl = SurvexScansFolder.objects.filter(walletname=mscansdir.group(1))
         if len(scansfolderl):
@@ -234,6 +241,7 @@ def FindTunnelScan(tunnelfile, path):
         if scansfolder:
             scansfilel = scansfolder.survexscansingle_set.filter(name=mscansdir.group(2))
             if len(scansfilel):
+                print(scansfilel, len(scansfilel))
                 assert len(scansfilel) == 1
                 scansfile = scansfilel[0]
             
@@ -242,9 +250,9 @@ def FindTunnelScan(tunnelfile, path):
         if scansfile:
             tunnelfile.survexscans.add(scansfile)
     
-    elif path and not re.search("\.(?:png|jpg)$(?i)", path):
+    elif path and not re.search(r"\.(?:png|jpg|jpeg)$(?i)", path):
         name = os.path.split(path)[1]
-        print "ttt", tunnelfile.tunnelpath, path, name
+        print("ttt", tunnelfile.tunnelpath, path, name)
         rtunnelfilel = TunnelFile.objects.filter(tunnelname=name)
         if len(rtunnelfilel):
             assert len(rtunnelfilel) == 1, ("two paths with name of", path, "need more discrimination coded")

settings.py

@@ -10,7 +10,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(__file__))
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG
 
-ALLOWED_HOSTS = []
+ALLOWED_HOSTS = [u'expo.survex.com']
 
 ADMINS = (
     # ('Your Name', 'your_email@domain.com'),
@@ -45,14 +45,52 @@ NOTABLECAVESHREFS = [ "161", "204", "258", "76", "107", "264" ]
 # Examples: "http://foo.com/media/", "/media/".
 ADMIN_MEDIA_PREFIX = '/troggle/media-admin/'
 PHOTOS_ROOT = os.path.join(EXPOWEB, 'photos')
-CAVEDESCRIPTIONS = os.path.join(EXPOWEB, "noinfo", "cave_data")
+CAVEDESCRIPTIONS = os.path.join(EXPOWEB, "cave_data")
-ENTRANCEDESCRIPTIONS = os.path.join(EXPOWEB, "noinfo", "entrance_data")
+ENTRANCEDESCRIPTIONS = os.path.join(EXPOWEB, "entrance_data")
 
 MEDIA_URL = urlparse.urljoin(URL_ROOT , '/site_media/')
 SURVEYS_URL = urlparse.urljoin(URL_ROOT , '/survey_scans/')
 PHOTOS_URL  = urlparse.urljoin(URL_ROOT , '/photos/')
 SVX_URL = urlparse.urljoin(URL_ROOT , '/survex/')
 
+# top-level survex file basename (without .svx)
+SURVEX_TOPNAME = "1623"
+
+DEFAULT_LOGBOOK_PARSER = "Parseloghtmltxt"
+DEFAULT_LOGBOOK_FILE = "logbook.html"
+
+LOGBOOK_PARSER_SETTINGS = {
+                "2019": ("2019/logbook.html", "Parseloghtmltxt"),
+                "2018": ("2018/logbook.html", "Parseloghtmltxt"),
+                "2017": ("2017/logbook.html", "Parseloghtmltxt"),
+                "2016": ("2016/logbook.html", "Parseloghtmltxt"),
+                "2015": ("2015/logbook.html", "Parseloghtmltxt"),
+                "2014": ("2014/logbook.html", "Parseloghtmltxt"), 
+                "2013": ("2013/logbook.html", "Parseloghtmltxt"), 
+                "2012": ("2012/logbook.html", "Parseloghtmltxt"), 
+                "2011": ("2011/logbook.html", "Parseloghtmltxt"), 
+                "2010": ("2010/logbook.html", "Parselogwikitxt"), 
+                "2009": ("2009/2009logbook.txt", "Parselogwikitxt"), 
+                "2008": ("2008/2008logbook.txt", "Parselogwikitxt"), 
+                "2007": ("2007/logbook.html", "Parseloghtmltxt"), 
+                "2006": ("2006/logbook/logbook_06.txt", "Parselogwikitxt"), 
+                "2005": ("2005/logbook.html", "Parseloghtmltxt"), 
+                "2004": ("2004/logbook.html", "Parseloghtmltxt"), 
+                "2003": ("2003/logbook.html", "Parseloghtml03"), 
+                "2002": ("2002/logbook.html", "Parseloghtmltxt"), 
+                "2001": ("2001/log.htm", "Parseloghtml01"), 
+                "2000": ("2000/log.htm", "Parseloghtml01"), 
+                "1999": ("1999/log.htm", "Parseloghtml01"), 
+                "1998": ("1998/log.htm", "Parseloghtml01"), 
+                "1997": ("1997/log.htm", "Parseloghtml01"), 
+                "1996": ("1996/log.htm", "Parseloghtml01"),
+                "1995": ("1995/log.htm", "Parseloghtml01"), 
+                "1994": ("1994/log.htm", "Parseloghtml01"), 
+                "1993": ("1993/log.htm", "Parseloghtml01"), 		
+                "1992": ("1992/log.htm", "Parseloghtml01"), 		
+                "1991": ("1991/log.htm", "Parseloghtml01"), 		
+            }
+
 APPEND_SLASH = False
 SMART_APPEND_SLASH = True
 
@@ -130,4 +168,6 @@ TINYMCE_COMPRESSOR = True
 
 MAX_LOGBOOK_ENTRY_TITLE_LENGTH = 200
 
+TEST_RUNNER = 'django.test.runner.DiscoverRunner'
+
 from localsettings import * #localsettings needs to take precedence. Call it to override any existing vars.

templates/base.html

@@ -1,12 +1,12 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<!DOCTYPE html>
 <html lang="en">
 <head>
-<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"/>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 <link rel="stylesheet" type="text/css" href="{{ settings.MEDIA_URL }}css/main3.css" title="eyeCandy"/>
 <link rel="alternate stylesheet" type="text/css" href="{{ settings.MEDIA_URL }}css/mainplain.css" title="plain"/>
 <link rel="stylesheet" type="text/css" href="{{ settings.MEDIA_URL }}css/dropdownNavStyle.css" />
 <title>{% block title %}Troggle{% endblock %}</title>
-<script src="{{ settings.JSLIB_URL }}jquery/jquery.min.js" type="text/javascript"></script>
+<!-- <script src="{{ settings.JSLIB_URL }}jquery/jquery.min.js" type="text/javascript"></script> -->
 <script src="{{ settings.MEDIA_URL }}js/jquery.quicksearch.js" type="text/javascript"></script>
 <script src="{{ settings.MEDIA_URL }}js/base.js" type="text/javascript"></script>
 <script src="{{ settings.MEDIA_URL }}js/jquery.dropdownPlain.js" type="text/javascript"></script>
@@ -16,7 +16,7 @@
 <body  onLoad="contentHeight();">
 
 <div id="header"> 
-  <h1>CUCC Expeditions to Austria: 1976 - 2016</h1>
+  <h1>CUCC Expeditions to Austria: 1976 - 2020</h1>
   <div id="editLinks"> {% block loginInfo %}
     <a href="{{settings.EXPOWEB_URL}}">Website home</a> |
     {% if user.username %}
@@ -40,9 +40,11 @@
     <a href="{% url "survexcavessingle" 204 %}">204</a> |
     <a href="{% url "survexcavessingle" 258 %}">258</a> |
     <a href="{% url "survexcavessingle" 264 %}">264</a> |
-    <a href="{% url "expedition" 2014 %}">Expo2014</a> |
-    <a href="{% url "expedition" 2015 %}">Expo2015</a> |
     <a href="{% url "expedition" 2016 %}">Expo2016</a> |
+    <a href="{% url "expedition" 2017 %}">Expo2017</a> |
+    <a href="{% url "expedition" 2018 %}">Expo2018</a> |
+    <a href="{% url "expedition" 2019 %}">Expo2019</a> |
+    <a href="{% url "expedition" 2020 %}">Expo2020</a> |
  
     <a href="/admin/">Django admin</a>
 </div>
@@ -81,7 +83,7 @@
 
 		<li><a href="#">External links</a>
                 <ul class="sub_menu">
-                    <li><a id="cuccLink" href="http://www.srcf.ucam.org/caving/wiki/Main_Page">CUCC website</a></li>
+                    <li><a id="cuccLink" href="https://camcaving.uk">CUCC website</a></li>
                     <li><a id="expoWebsiteLink" href="http://expo.survex.com">Expedition website</a></li>
                 </ul>
         </li>
@@ -90,7 +92,7 @@
         <li><a id="caversLink" href="{% url "personindex" %}">cavers</a></li>
         <li><a href="#">expeditions</a>
         		<ul class="sub_menu">
-                    <li><a id="expeditionsLink" href="{{ Expedition.objects.latest.get_absolute_url }}">newest</a></li>
+                    <li><a id="expeditionsLink" href="{{ expedition.objects.latest.get_absolute_url }}">newest</a></li>
                 	<li><a id="expeditionsLink" href="{% url "expeditions" %}">list all</a></li>
                 </ul>
         </li>

templates/cave.html

@@ -1,5 +1,438 @@
 {% extends "cavebase.html" %}
 
+{% block extraheaders %}
+{% if cave.survex_file %}
+<style>
+
+
+div.cv-panel {
+	position: absolute;
+	top: 0;
+	left: 0;
+	z-index: 100;
+	background-color: rgba(50,50,50,0.5);
+	color: yellowgreen;
+	border: 1px solid black;
+	border-radius: 5px;
+}
+
+div.cv-compass, div.cv-ahi {
+	position: absolute; 
+	bottom: 95px;
+	right: 5px;
+	margin: 0;
+	padding-top: 2px;
+	/*	border: 1px solid white; */
+	text-align: center;
+	width: 78px;
+	height: 19px;
+	z-index: 50;
+	background-color: rgba(50,50,50,0.5);
+	background-color: black;
+	color: white;
+}
+	
+div.cv-ahi {
+	right: 95px;
+}
+
+div.scale-legend {
+	position: absolute;
+	color: white;
+	background-color: black;
+	bottom: 30px;
+}
+
+div.linear-scale {
+	position: absolute;
+	color: white;
+	background-color: black;
+	right: 30px;
+	width: 40px;
+	padding: 2px 0;
+	text-align: right;
+	border: 1px solid black;
+	font-size: 14px;
+}
+
+div.linear-scale-caption {
+	position: absolute;
+	color: white;
+	background-color: black;
+	right: 5px;
+	width: 65px;
+	padding: 2px 0 5px 0;
+	text-align: left;
+	border: 1px solid black;
+	font-size: 14px;
+}
+
+#min-div {
+	border-bottom: 1px solid white;
+}
+
+#max-div {
+	border-top: 1px solid white;
+}
+
+#angle-legend {
+	position: absolute;
+	width: 80px;
+	right: 5px;
+	bottom: 180px;
+	color: white;
+	background-color: black;
+	font-size: 14px;
+	text-align: center;
+}
+
+#scene {
+	width: 100%;
+	height: 700px;
+	position: relative;
+}
+
+#progress-bar {
+	position: absolute;
+	top: 55%;
+	height: 20px;
+	border: 1px solid white;
+	z-index: 100;
+}
+
+#status-text {
+	position: absolute;
+	top: 50%;
+	height: 20px;
+	padding-left: 4px;
+	background-color: black;
+	color: white;
+	z-index: 100;
+}
+
+#frame div.page ul {
+	list-style-type: none;
+	margin: 8px 0 0 0;
+	padding: 0;
+	width: 200px;
+	height: 100%;
+	cursor: default;
+	font-size: 12px;
+	overflow-y: auto;
+	overflow-x: hidden;
+}
+
+#frame div.page li {
+	position: relative;
+	margin-left: 16px;
+	border-bottom: 1px solid #444444;
+}
+
+#frame div.page li.selected {
+	color: #1ab4e5;
+}
+
+#frame div.page li:hover {
+	color: yellow;
+}
+
+#frame div.page div#ui-path {
+	font-size: 12px;
+	border-top: 1px solid grey;
+	border-bottom: 1px solid grey;
+	margin-top: 8px;
+	padding: 2px 0 2px 12px;
+}
+
+#frame div.page div#ui-path span {
+	color: #1ab4e5;
+}
+
+#frame div.page div.slide {
+	position: absolute;
+	top: 64px;
+	left: 0px;
+	height: auto; 
+	margin-top:0;
+	bottom: 44px;
+	background-color: #222222;
+	transition: transform 0.25s ease-in;
+}
+
+#frame div.slide-out {
+	border-right: 1px grey solid;
+	transform: translateX(-100%);
+}
+
+#frame div.page div.descend-tree {
+	position: absolute;
+	top: 0px;
+	right: 0px;
+	margin: 0;
+	color: #1ab4e5;
+	z-index: 110;
+}
+
+#frame {
+	position: absolute;
+	top: 0px;
+	left: 0px;
+	width: 240px;
+	height: 100%;
+	background-color: transparent;
+	transform: translateX(-200px);
+	transition: transform 0.25s ease-in;
+}
+
+#frame.onscreen {
+	transform: none;
+	transition: transform 0.25s ease-out;
+}
+
+#frame a.download {
+	border: 1px solid green;
+	display: block;
+	width: 180px;
+	box-sizing: border-box;
+	margin-top: 6px;
+	margin-bottom: 4px;
+	margin-left: 8px;
+	border: none;
+	border-bottom: 4px solid #1ab4e5;
+	color: #dddddd;
+	background-color: black;
+	padding-bottom: 4px;
+	box-shadow: 1px 1px 8px 0px #888888;
+	outline: nonlass="cavedisplay"e;
+	text-decoration: none;
+	text-align: center;
+}
+
+#frame a.download:hover {
+	color: white;
+}
+
+#frame a.download:active {
+	color: #dddddd;
+	border-bottom: 4px solid #0c536a;
+	box-shadow: none;
+	box-shadow: inset 1px 1px 8px 0px #888888;
+}
+#frame .tab {
+	position: absolute;
+	right: 0px;lass="cavedisplay"
+	width: 40px;
+	height: 40px;
+	box-sizing: border-box;
+	background-color: #444444;
+	border-left: 1px solid black;
+	background-position: center;
+	border-top: 1px solid black;
+}
+
+#frame #close {
+	position: absolute;
+	right: 40px;
+	bottom: 0px;
+	width: 40px;
+	height: 40px;
+	box-sizing: border-box;
+	z-index: 150;
+	background-image: url(../images/ic_remove.png);
+	background-position: center;
+}
+
+#icon_settings {
+	background-image: url(../images/ic_settings.png);
+}
+
+#icon_terrain {
+	background-image: url(../images/ic_terrain.png);
+}
+
+#icon_explore {
+	background-image: url(../images/ic_explore.png);
+}
+
+#icon_info {
+	background-image: url(../images/ic_info.png);
+}
+
+#icon_route {
+	background-image: url(../images/ic_route.png);
+}
+
+#icon_help {
+	background-image: url(../images/ic_help.png);
+}
+
+#frame div.toptab {
+	background-color: #222222;
+	border-left: none;
+	border-right: 1px solid grey;
+	border-top: 1px solid grey;
+}
+
+#frame div.page {
+	position: absolute;
+	top: 0px;
+	bottom: 40px;
+	left: 0px;
+	width: 200px;
+	height: 100%;
+	color: white;
+	background-color: #222222;
+	padding: 0 4px;
+	box-sizing: border-box;
+	cursor: default;
+	padding-bottom: 40px;
+}
+
+#frame div.page div.header {
+	margin: 16px 0px 8px 0px;
+	font-weight: bold;
+	height: 16px;
+	box-sizing: border-box;
+	padding-left: 2px;
+}
+
+#frame div.page div.control {
+	margin: 2px 0 2px 0;
+	padding-top: 2px;
+}
+
+#frame div.page label {
+	display: block;
+	border-top: 1px solid grey;
+	padding: 2px 0 2px 8px;
+	font-size: 12px;
+}
+
+#frame div.page select {
+	display: block;
+	width: 180px;
+	box-sizing: border-box;
+	padding-top: 2px;
+	margin: 2px 0 4px 8px;
+}
+
+#frame div.page select:empty {
+	background-color: #888888;
+}
+
+#frame div.page button {
+	display: block;
+	width: 180px;
+	box-sizing: border-box;
+	margin-top: 4px;
+	margin-bottom: 4px;
+	margin-left: 8px;
+	border: none;
+	border-bottom: 4px solid #1ab4e5;
+	color: #dddddd;
+	background-color: black;
+	padding-bottom: 4px;
+	box-shadow: 1px 1px 8px 0px #888888;
+	outline: none;
+}
+
+#frame div.page button:hover {
+	color: white;
+}
+
+#frame div.page button:active {
+	color: #dddddd;
+	border-bottom: 4px solid #0c536a;
+	box-shadow: none;
+	box-shadow: inset 1px 1px 8px 0px #888888;
+}
+
+#frame div.page input[type="text"] {
+	display: block;
+	width: 180px;
+	box-sizing: border-box;
+	margin-top: 2px;
+	margin-left: 8px;
+}
+
+#frame div.page input[type="checkbox"] {
+	position: absolute;
+	right: 0px;
+}
+
+#frame div.page input[type="range"] {
+	display: block;
+	width: 180px;
+	margin-left: 8px;
+}
+
+#frame dt, #frame dd {
+	font-size: 12px;
+}
+
+#frame dt {
+	clear: both;
+	float: left;
+	padding-left: 16px;
+}
+
+#frame dd {
+	margin-left: 40px;
+}
+
+#frame p {
+	font-size: 12px;
+	line-height: 18px;
+}
+
+div.station-info {
+	position: absolute;
+	border: 1px solid white;
+	background-color: #222222;
+	color: white;
+	padding: 4px;
+	z-index: 200;
+}
+
+
+.overlay-branding {
+	color: white;
+	margin: 4px;
+	position: absolute;
+	right: 0;
+	top: 0;
+}
+div#scene {
+			width: 100%;
+			height: 90%;		}
+
+</style>
+
+<script type="text/javascript" src="/javascript/CaveView/js/CaveView.js" ></script>
+<script type="text/javascript" src="/javascript/CaveView/lib/proj4.js" ></script>
+
+
+<script type="text/javascript" >
+
+	function onLoad () {
+
+		// display the user interface - and a blank canvas
+		// the configuration object specifies the location of CaveView, surveys and terrain files
+		CV.UI.init( 'scene', {
+			home: '/javascript/CaveView/',
+			surveyDirectory: '/cave/3d/',
+		terrainDirectory: '/loser/surface/terrain/'
+		} );
+
+		// load a single survey to display
+		CV.UI.loadCave( '{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{ cave.unofficial_number }}{% endif %}.3d' );
+	}
+        window.onload = onLoad;
+</script>
+{% endif %}
+{% endblock %}
+
+
 {% load wiki_markup %}
 {% block content %}
 {% block contentheader %}
@@ -131,7 +564,8 @@
 {% endif %}
 {% if cave.survex_file %}
     <h2>Survex File</h2>
-    {{ cave.survex_file|safe }}
+    {{ cave.survex_file|safe }} <a href="{% if cave.kataster_number %}{% url "cave3d" cave.kataster_number %}{% else %}{% url "cave3d" cave.unofficial_number %}{% endif %}">3d file</a>
+    <div id='scene'></div>
 {% endif %}
 {% if cave.notes %}
     <h2>Notes</h2>

templates/caveindex.html

@@ -7,29 +7,31 @@
 
 <h1>Cave Index</h1>
 
+
 <h3>Notable caves</h3>
 <ul>
 {% for cave in notablecaves %}
-  <li> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> </li>
+  <li> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{% endif %} {{cave.official_name|safe}}</a> </li>
 {% endfor %}
 </ul>
 
 <h3>1623</h3>
 
-<ul class="searchable">
+<table class="searchable">
 {% for cave in caves1623 %}
 
-  <li> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> </li>
+  <tr><td> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }} {{cave.official_name|safe}}</a> {% if cave.unofficial_number %}({{cave.unofficial_number }}){% endif %}{% else %}{{cave.unofficial_number }} {{cave.official_name|safe}}</a> {% endif %}</td></tr>
 
 {% endfor %}
-</ul>
+</table>
 
 <h3>1626</h3>
 
 <ul class="searchable">
 {% for cave in caves1626 %}
 
-  <li> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> </li>
+  <li>  <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }} {{cave.official_name|safe}}</a> {% if cave.unofficial_number %}({{cave.unofficial_number }}){% endif %}{% else %}{{cave.unofficial_number }} {{cave.official_name|safe}}</a> {% endif %}
+ </li>
 
 {% endfor %}
 </ul>

templates/cavemillenial.html

@@ -0,0 +1,32 @@
+{% extends "cavebase.html" %}
+{% load wiki_markup %}
+
+{% block title %}Cave Index{% endblock %}
+
+{% block content %}
+
+<h1>Cave Index</h1>
+
+<h3>1623</h3>
+
+<table class="searchable">
+{% for cave in caves1623 %}
+
+  <tr><td> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> </td></tr>
+
+{% endfor %}
+</table>
+
+<h3>1626</h3>
+
+<ul class="searchable">
+{% for cave in caves1626 %}
+
+  <li> <a href="{{ cave.url }}">{% if cave.kataster_number %}{{ cave.kataster_number }}{% else %}{{cave.unofficial_number }}{%endif %} {{cave.official_name|safe}}</a> </li>
+
+{% endfor %}
+</ul>
+
+<a href="{% url "newcave" %}">New Cave</a>
+
+{% endblock %}

templates/controlPanel.html

@@ -23,25 +23,45 @@
 
 
 <form name="reset" method="post" action="">
-<h3>Wipe:</h3>
+    <h3>Wipe:</h3>
-
+    <table>
-<table>
+        <tr>
-<tr><td>Wipe entire database and recreate tables: </td><td><input type="checkbox" name="reload_db" /></td><td>      <input type="submit" id="Import" value="I really want to delete all information in troggle, and accept all responsibility."></td></tr>
+            <td>Wipe entire database and recreate tables: </td>
-</table>
+            <td><input type="checkbox" name="reload_db" /></td>
+            <td>
+                <input type="submit" id="Import" value="I really want to delete all information in troggle, and accept all responsibility.">
+            </td>
+        </tr>
+    </table>
 </form>
 <h3>Import (non-destructive):</h3>
 <form name="import" method="post" action="">
-<table>
+    <table>
-<tr><td>people from folk.csv using parsers\people.py</td><td><input type="checkbox" name="import_people"/></td></tr>
+        <tr>
-<tr><td>caves from cavetab2.csv using parsers\cavetab.py</td><td> <input type="checkbox" class="parser" name="import_cavetab"/></td></tr>
+            <td>people from folk.csv using parsers\people.py</td>
-<tr><td>logbook entries using parsers\logbooks.py</td><td><input type="checkbox" name="import_logbooks"/></td></tr>
+            <td><input type="checkbox" name="import_people"/></td>
-<tr><td>QMs using parsers\QMs.py</td><td><input type="checkbox" name="import_QMs" /></td></tr>
+        </tr>
-<tr><td>survey scans using parsers\surveys.py</td><td><input type="checkbox" name="import_surveys" /></td></tr>
+        <tr>
-<tr><td>survex data using parsers\survex.py</td><td><input type="checkbox" name="import_survex" /></td></tr>
+            <td>caves from cavetab2.csv using parsers\cavetab.py</td>
-
+            <td> <input type="checkbox" class="parser" name="import_cavetab"/></td>
-</table>
+        </tr>
-
+        <tr>
-
+            <td>logbook entries using parsers\logbooks.py</td>
+            <td><input type="checkbox" name="import_logbooks"/></td>
+        </tr>
+        <tr>
+            <td>QMs using parsers\QMs.py</td>
+            <td><input type="checkbox" name="import_QMs" /></td>
+        </tr>
+        <tr>
+            <td>survey scans using parsers\surveys.py</td>
+            <td><input type="checkbox" name="import_surveys" /></td>
+        </tr>
+        <tr>
+            <td>survex data using parsers\survex.py</td>
+            <td><input type="checkbox" name="import_survex" /></td>
+        </tr>
+    </table>
     <p>
       <input type="submit" id="Import" value="Import">
       

templates/core/expedition_list.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+{% load wiki_markup %}
+{% load link %}
+
+{% block content %}
+<h1>Expeditions</h1>
+<ul>
+{% for expedition in object_list %}
+    <li>{{ expedition.year }} - <a href="{{ expedition.get_absolute_url }}">{{ expedition.name }}</a></li>
+{% empty %}
+    <li>No articles yet.</li>
+{% endfor %}
+</ul>
+{% endblock %}

templates/dataformat/cave.xml

@@ -1,4 +1,4 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<!DOCTYPE html>
 <!-- Only put one cave in this file -->
 <!-- If you edit this file, make sure you update the websites database -->
 <html lang="en">

templates/dataformat/entrance.xml

@@ -1,4 +1,4 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<!DOCTYPE html>
 <!-- Only put one entrance in this file -->
 <!-- If you edit this file, make sure you update the websites database -->
 <html lang="en">

templates/dataformat/logbookentry.html

@@ -1,4 +1,5 @@
 {% autoescape off %}
+<!DOCTYPE html>
 <html>
 <head>
 <style type="text/css">.author {text-decoration:underline}</style>

templates/editcave2.html

@@ -9,6 +9,7 @@
 <script src="{{ settings.TINY_MCE_MEDIA_URL }}tiny_mce.js" type="text/javascript"></script>
 {% endblock %}
 {% block content %}
+<h1>Edit Cave</h1>
 <form action="" method="post">{% csrf_token %}
     <table>{{ form }}{{caveAndEntranceFormSet}}</table>
     {{ versionControlForm }}

templates/editflatpage.html

@@ -3,6 +3,11 @@
 {% block extrahead %}
 {% load csrffaker %} 
 <script src="{{ settings.TINY_MCE_MEDIA_URL }}tiny_mce.js" type="text/javascript"></script>
+<script type="text/javascript">
+    tinyMCE.init({
+      mode : "textareas"
+    });
+</script>
 {% endblock %}
 {% block body %}
 <h1>Edit {{ path }}</h1>

templates/expedition.html

@@ -10,10 +10,6 @@
 
 {% block content %}
 
-{% if message %}
-<p>debug message: {{message}}</p>
-{% endif %}
-
 <h2>{{expedition.name}}</h2>
 
 <p><b>Other years:</b>
@@ -41,13 +37,13 @@ an "S" for a survey trip.  The colours are the same for people on the same trip.
 </tr>
 {% for personexpeditionday in personexpeditiondays %}
   <tr>
-  <td><a href="{{ personexpeditionday.personexpedition.get_absolute_url }}">{{personexpeditionday.personexpedition.person}}</a></td>
+  <td><a href="{{ personexpeditionday.personexpedition.get_absolute_url }}">{{personexpeditionday.personexpedition.person|safe}}</a></td>
   {% for persondayactivities in personexpeditionday.personrow %}
 
   {% if persondayactivities.persontrips or persondayactivities.survexblocks %}
   <td class="persondayactivity">
     {% for persontrip in persondayactivities.persontrips %}
-      <a href="{{persontrip.logbook_entry.get_absolute_url}}" class="dayindexlog-{{persontrip.logbook_entry.DayIndex}}">T</a>
+      <a href="{{persontrip.logbook_entry.get_absolute_url}}" class="dayindexlog-1">T</a>
     {% endfor %}
    <br/>
     {% for survexblock in persondayactivities.survexblocks %}
@@ -73,7 +69,7 @@ an "S" for a survey trip.  The colours are the same for people on the same trip.
 {% regroup dateditems|dictsort:"date" by date as dates %}
 {% for date in dates %}
 <tr>
-<td>{{date.grouper}}</td>
+<td>{{date.grouper|date:"D d M Y"}}</td>
 <td>{% for item in date.list %}
         {% if item.isLogbookEntry %}<a href="{{ item.get_absolute_url }}">{{item.title|safe}}</a><br/>{% endif %}
     {% endfor %}</td>

templates/flatpage.html

@@ -3,5 +3,13 @@
 {% block bodyattrs %}{% if homepage %} id="homepage"{% endif %}{% endblock %}
 {% block body %}
 {{ body|safe }}
-{% if homepage %}{% if editable %}<a href="{% url "editflatpage" path %}">Edit</a>{% endif %}{%else %}{% include "menu.html" %}{% endif %}
+{% if homepage %}
+  {% if editable %}
+    <a href="{% url "editflatpage" path %}">Edit</a>
+  {% endif %}
+  {% else %}
+    {% if not has_menu %}
+      {% include "menu.html" %}
+    {% endif %}
+  {% endif %}
 {% endblock %}

templates/frontpage.html

@@ -16,7 +16,7 @@
                 {% if entry.is_deletion %}
                     {{ entry.object_repr }}
                 {% else %}
-                    <a href="admin/{{ entry.get_admin_url }}">{{ entry.object_repr }}</a>
+                    <a href="admin/{{ entry.get_admin_url }}/">{{ entry.object_repr }}</a>
                 {% endif %}
                 <br/>
                 {% if entry.content_type %}
@@ -42,24 +42,13 @@ This is Troggle, the information portal for Cambridge University Caving Club's E
 </p>
 
 <p class="indent">
-Here you will find information about the {{expedition.objects.count}} expeditions the club has undertaken since 1976. Browse survey information, photos, and description wikis for {{cave.objects.count}} caves, {{subcave.objects.count}} areas within those caves, and {{extantqms.count}} going leads yet to be explored. We have {{photo.objects.count}} photos and {{logbookentry.objects.count}} logbook entries.
+Here you will find information about the {{expedition.objects.count}} expeditions the club has undertaken since 1976. Browse survey information, photos, and description wikis for {{Cave.objects.count}} caves, {{subcave.objects.count}} areas within those caves, and {{extantqms.count}} going leads yet to be explored. We have {{Photo.objects.count}} photos and {{Logbookentry.objects.count}} logbook entries.
 </p>
 
 <p class="indent">
 If you are an expedition member, please sign up using the link to the top right and begin editing.
 </p>
 
-<h3>News</h3>
-
-<p class="indent">
-Everyone is gearing up for the 2009 expedition; please see the link below for the main expedition website.
-</p>
-
-<h3>Troggle development</h3>
-<p class="indent">
-Troggle is still under development. Check out the <a href="http://troggle.googlecode.com">development page</a> on google code, where you can file bug reports, make suggestions, and help develop the code. There is also an old todo list at <a href="{%url "todo"%}">here</a>.
-</p>
-</div>
 {% endblock content %}
 
 {% block margins %}

templates/logbookentry.html

@@ -2,12 +2,12 @@
 {% load wiki_markup %}
 
 {% block title %}Logbook {{logbookentry.id}}{% endblock %}
-{% block editLink %}<a href={{logbookentry.get_admin_url}}>Edit logbook entry {{logbookentry|wiki_to_html_short}}</a>{% endblock %}
+{% block editLink %}<a href={{logbookentry.get_admin_url}}/>Edit logbook entry {{logbookentry|wiki_to_html_short}}</a>{% endblock %}
 {% block content %}
 
 {% block related %}{% endblock %}
 {% block nav %}{% endblock %}
-<h2>{{logbookentry.title}}</h2>
+<h2>{{logbookentry.title|safe}}</h2>
 
 <div id="related">
 <p><a href="{{ logbookentry.expedition.get_absolute_url }}">{{logbookentry.expedition.name}}</a></p>
@@ -20,10 +20,10 @@
 
 <p>
   {% if logbookentry.get_previous_by_date %}
-    <a href="{{ logbookentry.get_previous_by_date.get_absolute_url }}">{{logbookentry.get_previous_by_date.date}}</a>
+    <a href="{{ logbookentry.get_previous_by_date.get_absolute_url }}">{{logbookentry.get_previous_by_date.date|date:"D d M Y"}}</a>
   {% endif %}
   {% if logbookentry.get_next_by_date %}
-    <a href="{{ logbookentry.get_next_by_date.get_absolute_url }}">{{logbookentry.get_next_by_date.date}}</a>
+    <a href="{{ logbookentry.get_next_by_date.get_absolute_url }}">{{logbookentry.get_next_by_date.date|date:"D d M Y"}}</a>
   {% endif %}
 </p>
 
@@ -47,12 +47,12 @@
 
   <td>
   {% if persontrip.persontrip_prev %}
-    <a href="{{ persontrip.persontrip_prev.logbook_entry.get_absolute_url }}">{{persontrip.persontrip_prev.logbook_entry.date}}</a>
+    <a href="{{ persontrip.persontrip_prev.logbook_entry.get_absolute_url }}">{{persontrip.persontrip_prev.logbook_entry.date|date:"D d M Y"}}</a>
   {% endif %}
   </td>
   <td>
   {% if persontrip.persontrip_next %}
-    <a href="{{ persontrip.persontrip_next.logbook_entry.get_absolute_url }}">{{persontrip.persontrip_next.logbook_entry.date}}</a>
+    <a href="{{ persontrip.persontrip_next.logbook_entry.get_absolute_url }}">{{persontrip.persontrip_next.logbook_entry.date|date:"D d M Y"}}</a>
   {% endif %}
   </td>
 
@@ -65,9 +65,14 @@
 </div>
 
 <div id="col1">
-<div class="logbookentry">
+  <div class="logbookentry">
-<b>{{logbookentry.date}}</b>
+    <b>{{logbookentry.date|date:"D d M Y"}}</b>
-    {{logbookentry.text|wiki_to_html}}</div>
+    {% if logbookentry.entry_type == "html" %}
+      <p>{{logbookentry.text|safe}}</p>
+    {% else %}
+      {{logbookentry.text|wiki_to_html}}
+    {% endif %}
+  </div>
 </div>
 </div>
 

templates/personexpedition.html

@@ -7,7 +7,7 @@
 
 {% block content %}
 <h1>
-  <a href="{{personexpedition.person.get_absolute_url}}">{{personexpedition.person}}</a> : 
+  <a href="{{personexpedition.person.get_absolute_url}}">{{personexpedition.person|safe}}</a> : 
   <a href="{{personexpedition.expedition.get_absolute_url}}">{{personexpedition.expedition}}</a>
 </h1>
 

templates/personindex.html

@@ -8,12 +8,12 @@
 <h2>Notable expoers</h2>
 <table class="searchable">
 <tr><th>Person</th><th>First</th><th>Last</th><th>Notability</th></tr>
-{% for person in notablepersons %}
+{% for person in notablepersons|dictsortreversed:"notability" %}
 <tr>
   <td><a href="{{ person.get_absolute_url }}">{{person|wiki_to_html_short}}</a></td>
   <td><a href="{{ person.first.get_absolute_url }}">{{ person.first.expedition.year }}</a></td> 
   <td><a href="{{ person.last.get_absolute_url }}">{{ person.last.expedition.year }}</a></td> 
-  <td>{{person.notability}}</td>
+  <td>{{person.notability|floatformat:2}}</td>
 </tr>
 {% endfor %}
 </table>
@@ -31,8 +31,8 @@
 <tr>
   <td><a href="{{ person.get_absolute_url }}">{{person|wiki_to_html_short}}</a></td>
   <td><a href="{{ person.first.get_absolute_url }}">{{person.first.expedition.year}}</a></td> 
-  <td><a href="{{ person.last.get_absolute_url }}">{{person.last.expedition.year}}</a></td> 
+  <td><a href="{{ person.last.get_absolute_url }}">{{person.last.expedition.year}}</a></td>
-  <td>{{ person.surveyedleglength }}</td>
+  <td></td>
 </tr>
 {% endfor %}
 </table>

templates/svxcavesingle.html

@@ -41,7 +41,7 @@
   <td>{{survexblock.name}}</td>
   <td>
   {% if survexblock.expedition %}
-    <a href="{{survexblock.expedition.get_absolute_url}}">{{survexblock.date}}</a>
+    <a href="{{survexblock.expedition.get_absolute_url}}">{{survexblock.date|date:"D d M Y"}}</a>
   {% else %}
     {{survexblock.date}}
   {% endif %} 

templates/svxfile.html

@@ -46,7 +46,7 @@ $(document).ready(function()
 </p>
 {% endif %}
 
-<form id="codewikiform" action="" method="POST">
+<form id="codewikiform" action="" method="POST">{% csrf_token %}
     <div class="codeframebit">{{form.code}}</div>
     <div style="display:none">{{form.filename}} {{form.dirname}} {{form.datetime}} {{form.outputtype}}</div>
     <input type="submit" name="diff" value="Diffy" />

urls.py

@@ -15,23 +15,32 @@ admin.autodiscover()
 
 # type url probably means it's used.
 
+#	HOW DOES THIS WORK:
+#	url( <regular expression that matches the thing in the web browser>,
+#   <reference to python function in 'core' folder>,
+#	<name optional argument for URL reversing (doesn't do much)>)
+
 actualurlpatterns = patterns('',
+	
+    url(r'^testingurl/?$' , views_caves.millenialcaves, name="testing"),
+
+    url(r'^millenialcaves/?$',     views_caves.millenialcaves,	name="millenialcaves"),
     
     url(r'^troggle$',              views_other.frontpage,      name="frontpage"),
     url(r'^todo/$',              views_other.todo,      name="todo"),
     
-    url(r'^caves/?$',     views_caves.caveindex,      name="caveindex"),
+    url(r'^caves/?$',     views_caves.caveindex,      name="caveindex"),	
     url(r'^people/?$',   views_logbooks.personindex, name="personindex"),
 
     url(r'^newqmnumber/?$',              views_other.ajax_QM_number,  ),
     url(r'^lbo_suggestions/?$',              logbook_entry_suggestions),    
     #(r'^person/(?P<person_id>\d*)/?$', views_logbooks.person),
-    url(r'^person/(?P<first_name>[A-Z]*[a-z\-\']*)[^a-zA-Z]*(?P<last_name>[a-z\-\']*[^a-zA-Z]*[A-Z]*[a-z\-]*)/?', views_logbooks.person, name="person"),
+    url(r'^person/(?P<first_name>[A-Z]*[a-z\-\'&;]*)[^a-zA-Z]*(?P<last_name>[a-z\-\']*[^a-zA-Z]*[A-Z]*[a-z\-&;]*)/?', views_logbooks.person, name="person"),
     #url(r'^person/(\w+_\w+)$',       views_logbooks.person,      name="person"),
     
     url(r'^expedition/(\d+)$',  views_logbooks.expedition,  name="expedition"),
-    url(r'^expeditions/?$',  ListView,  {'queryset':Expedition.objects.all(),'template_name':'object_list.html'},name="expeditions"),
+    url(r'^expeditions/?$',  views_logbooks.ExpeditionListView.as_view(), name="expeditions"),
-    url(r'^personexpedition/(?P<first_name>[A-Z]*[a-z]*)[^a-zA-Z]*(?P<last_name>[A-Z]*[a-z]*)/(?P<year>\d+)/?$', views_logbooks.personexpedition, name="personexpedition"),
+    url(r'^personexpedition/(?P<first_name>[A-Z]*[a-z&;]*)[^a-zA-Z]*(?P<last_name>[A-Z]*[a-zA-Z&;]*)/(?P<year>\d+)/?$', views_logbooks.personexpedition, name="personexpedition"),
     url(r'^logbookentry/(?P<date>.*)/(?P<slug>.*)/?$', views_logbooks.logbookentry,name="logbookentry"),
     url(r'^newlogbookentry/(?P<expeditionyear>.*)$', views_logbooks.newLogbookEntry,     name="newLogBookEntry"),
     url(r'^editlogbookentry/(?P<expeditionyear>[^/]*)/(?P<pdate>[^/]*)/(?P<pslug>[^/]*)/$', views_logbooks.newLogbookEntry,     name="editLogBookEntry"),
@@ -44,7 +53,7 @@ actualurlpatterns = patterns('',
     url(r'^getLogBookEntries/(?P<expeditionslug>.*)', views_logbooks.get_logbook_entries, name = "get_logbook_entries"),
 
     
-    url(r'^cave/new/$', edit_cave, name="newcave"),
+    url(r'^cave/new/$', views_caves.edit_cave, name="newcave"),
     url(r'^cave/(?P<cave_id>[^/]+)/?$', views_caves.cave, name="cave"),
     url(r'^caveslug/([^/]+)/?$', views_caves.caveSlug, name="caveSlug"),
     url(r'^cave/entrance/([^/]+)/?$', views_caves.caveEntrance),
@@ -56,12 +65,13 @@ actualurlpatterns = patterns('',
     #url(r'^cavedescription/(?P<cavedescription_name>[^/]+)/?$', views_caves.cave_description, name="cavedescription"),
     #url(r'^cavedescription/?$', object_list, {'queryset':CaveDescription.objects.all(),'template_name':'object_list.html'}, name="cavedescriptions"),
     #url(r'^cavehref/(.+)$',     views_caves.cave,       name="cave"),url(r'cave'),
+    url(r'^cave/3d/(?P<cave_id>[^/]+).3d$', views_caves.cave3d, name="cave3d"),
 
 #    url(r'^jgtfile/(.*)$',      view_surveys.jgtfile,       name="jgtfile"),
 #    url(r'^jgtuploadfile$',     view_surveys.jgtuploadfile, name="jgtuploadfile"),
 
     url(r'^cave/(?P<cave_id>[^/]+)/?(?P<ent_letter>[^/])$', ent),
-    url(r'^cave/(?P<slug>[^/]+)/edit/$', edit_cave, name="edit_cave"),
+    url(r'^cave/(?P<slug>[^/]+)/edit/$', views_caves.edit_cave, name="edit_cave"),
     #(r'^cavesearch', caveSearch),
 
     
@@ -128,7 +138,7 @@ actualurlpatterns = patterns('',
     #(r'^survey_scans/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.SURVEY_SCANS, 'show_indexes':True}),
     url(r'^survey_scans/$',                        view_surveys.surveyscansfolders, name="surveyscansfolders"), 
     url(r'^survey_scans/(?P<path>[^/]+)/$',        view_surveys.surveyscansfolder,  name="surveyscansfolder"), 
-    url(r'^survey_scans/(?P<path>[^/]+)/(?P<file>[^/]+(?:png|jpg))$', 
+    url(r'^survey_scans/(?P<path>[^/]+)/(?P<file>[^/]+(?:png|jpg|jpeg))$', 
                                                    view_surveys.surveyscansingle,   name="surveyscansingle"), 
     
     url(r'^tunneldata/$',                          view_surveys.tunneldata,         name="tunneldata"), 

utils.py

@@ -1,4 +1,5 @@
 from django.conf import settings
+from django.shortcuts import render
 import random, re, logging
 from troggle.core.models import CaveDescription
 
@@ -58,21 +59,6 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
     if not created and not instance.new_since_parsing:
         logging.info(str(instance) + " existed in the database unchanged since last parse. It was overwritten by the current script. \n")
     return (instance, created)
-
-def render_with_context(req, *args, **kwargs):
-    """this is the snippet from http://www.djangosnippets.org/snippets/3/
-
-    Django uses Context, not RequestContext when you call render_to_response.
-    We always want to use RequestContext, so that django adds the context from
-    settings.TEMPLATE_CONTEXT_PROCESSORS. This way we automatically get
-    necessary settings variables passed to each template. So we use a custom
-    method, render_response instead of render_to_response. Hopefully future
-    Django releases will make this unnecessary."""
-
-    from django.shortcuts import render_to_response
-    from django.template import RequestContext
-    kwargs['context_instance'] = RequestContext(req)
-    return render_to_response(*args, **kwargs)
     
 re_body = re.compile(r"\<body[^>]*\>(.*)\</body\>", re.DOTALL)
 re_title = re.compile(r"\<title[^>]*\>(.*)\</title\>", re.DOTALL)