working maps: cave -> desc, survey -> cave, expedition -> person. Added /millnialpeople/ page.

core/models_millenial.py

@@ -70,8 +70,8 @@ class ExpeditionM(models.Model): #instance of this class corresponds to one expo
 
 class SurveyM(models.Model): #instance of this class corresponds to one .svx file - one trip
     date = models.CharField(max_length=100) #date of the trip in format YYYY.MM.DD (dated:=date given by .svx file) 
-    maxdepth = models.FloatField() #represents max depth of a node in this survey
+    survex_file = models.TextField()
-
+    
 class Logbook_entryM(models.Model): #instance of this class corresponds to one bit of logbook (c.f. expo.survex.com/years/2015/logbook.html or simil)
     date = models.CharField(max_length=100) #date as typed into logbook
     contents = models.TextField() #contents of the logbook chunk

core/views_caves.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation, CaveM, Cave_descriptionM
+from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation
 from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
 import troggle.core.models as models
 import troggle.settings as settings
@@ -21,6 +21,13 @@ import settings
 from PIL import Image, ImageDraw, ImageFont
 import string, os, sys, subprocess
 
+#
+#   NEW CONTENT
+#
+
+
+from troggle.core.models import CaveM, Cave_descriptionM, ExpeditionM
+
 def millenialcaves(request):
     #RW messing around area
     caves = CaveM.objects.all()
@@ -31,6 +38,16 @@ def millenialdescription(request, slug):
     desc = Cave_descriptionM.objects.get(slug=slug)
     return render_with_context(request,'cave_uground_description.html', {'cave': desc})
 
+def millenialpeople(request):
+    expos = ExpeditionM.objects.all()
+    return render_with_context(request,'peoplemillenial.html' , {'expos': expos})
+
+
+#
+#   END NEW CONTENT
+#
+
+
 
 def getCave(cave_id):
     """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm."""

databaseResetM.py

@@ -50,10 +50,19 @@ def load_redirects():
         f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL)
         f.save()
 
+def load_surveys():
+    SurveyM.objects.all().delete()
+    import troggle.parsers.surveysM
+    troggle.parsers.surveysM.load()
+
 def load_caves():
     import troggle.parsers.cavesM
     troggle.parsers.cavesM.load()
 
+def load_people():
+    import troggle.parsers.peopleM
+    troggle.parsers.peopleM.load()
+
 def load_all():
     load_caves()
     load_surveys()

parsers/cavesM.py

@@ -21,6 +21,9 @@ def load():
     print('Loading caves of 1623 area')
     loadarea('1623')
 
+    print('Loading caves of 1626 area')
+    loadarea('1626')
+
 
 def loadarea(areacode):
 

parsers/peopleM.py

@@ -0,0 +1,27 @@
+from django.conf import settings
+import troggle.core.models as models
+
+def load():
+    folkfile = open(settings.EXPOWEB+"noinfo/folk.csv")
+    personlines = folkfile.read().splitlines()
+    persontable = [x.split(',') for x in personlines]
+    years = [persontable[0][i] for i in range(5,len(persontable[0]))]
+    for year in years:
+        newexpedition = models.ExpeditionM( date = year )
+        newexpedition.save()
+    for row in persontable[1:]: #skip header
+        attendedid = [i for i, x in enumerate(row) if '1' in x]
+        attendedyears = [persontable[0][i] for i in attendedid if i >= 5]
+        name = row[0]
+        print(name+' has attended: '+', '.join(attendedyears))
+        newperson = models.PersonM(
+        name = name)
+        newperson.save()
+        for year in attendedyears:
+            target = models.ExpeditionM.objects.get(date=year)
+            newperson.expos_attended.add( target )
+    print('Person -> Expo table created!')
+
+        
+        
+    

parsers/surveysM.py

@@ -0,0 +1,65 @@
+from django.conf import settings
+import subprocess, re
+import troggle.core.models as models
+
+def load():
+    print('Load survex files and relations')
+    load_area('1623')
+
+def load_area(areacode):
+
+    print('Searching all cave dirs files')
+    basedir = settings.SURVEX_DATA+'caves-'+areacode+'/'
+
+    cavedirs = bash("find "+basedir+" -maxdepth 1 -type d").splitlines() #this command finds all directories
+    print('Obtained list of directories! (#dirs='+str(len(cavedirs))+')')
+    
+    for cavedir in cavedirs:
+        if cavedir==basedir:
+            continue #skip the basedir - a non-proper subdirectory
+        parentname = bash('echo '+cavedir+' | rev | cut -f1 -d \'/\' | rev').splitlines()[0] #get final bit of the directory
+        parentcave = models.CaveM.objects.filter(survex_file__icontains=cavedir)
+        if len(parentcave)>1:
+            print('Non unique parent - skipping. Name:'+parentname)
+        elif len(parentcave)==0:
+            print('Error! parent not created:'+parentname)
+            continue
+        else: #exaclty one match
+            print('Adding relations of:'+parentname)
+            parentcave = parentcave[0]
+            
+            surveyfiles = bash('find '+cavedir+' -name \'*.svx\'').splitlines()
+            for fn in surveyfiles:
+                print(fn)
+                svxcontents = open(fn,'r').read().splitlines()
+                try:
+                    dateline = [x for x in svxcontents if ('*date' in x)][0]
+                    date = re.findall('\\d\\d\\d\\d\\.\\d\\d\\.\\d\\d', dateline, re.S)[0]
+                    
+                    
+                except:
+                    if( len( [x for x in svxcontents if ('*date' in x)] ) == 0 ):
+                        continue #skip dateless files                     
+                    print('Date format error in '+fn)
+                    print('Dateline = '+ '"'.join([x for x in svxcontents if ('*date' in x)]))
+                    date = '1900.01.01' 
+                
+            
+                newsurvex = models.SurveyM(survex_file=fn, date=date)
+                newsurvex.save()
+                parentcave.surveys.add(newsurvex)
+                parentcave.save()
+                        
+
+def file_exists(filename):
+    test = bash('if [ ! -f '+filename+' ] ; then echo MISSING; fi')#test for file exisence
+    if 'MISSING' in test: #send error message to the database
+        return False
+    return True       
+
+def bash(cmd): #calls command in bash shell, returns output
+    process = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
+    output, error = process.communicate()
+    return output
+        
+    

urls.py

@@ -24,6 +24,7 @@ actualurlpatterns = patterns('',
 	
     
     url(r'^millenialcaves/?$',     views_caves.millenialcaves,	name="millenialcaves"),
+    url(r'^millenialpeople/?$',     views_caves.millenialpeople,	name="millenialpeople"),
     url(r'^cave/descriptionM/([^/]+)/?$', views_caves.millenialdescription),
     #url(r'^cave/description/([^/]+)/?$', views_caves.caveDescription),