[svn] now with ability to make new svx file

core/admin.py

@@ -118,7 +118,6 @@ admin.site.register(Expedition)
 admin.site.register(Person,PersonAdmin)
 admin.site.register(PersonRole)
 admin.site.register(PersonExpedition,PersonExpeditionAdmin)
-admin.site.register(Role)
 admin.site.register(LogbookEntry, LogbookEntryAdmin)
 #admin.site.register(PersonTrip)
 admin.site.register(QM, QMAdmin)

core/models_survex.py

@@ -6,57 +6,69 @@ import os
 ###########################################################
 # These will allow browsing and editing of the survex data
 ###########################################################
-
-
 # Needs to add: 
-#   SurvexFile
 #   Equates
 #   reloading
 
+class SurvexDirectory(models.Model):
+    path = models.CharField(max_length=200)
+    cave = models.ForeignKey('Cave', blank=True, null=True)
+    primarysurvexfile = models.ForeignKey('SurvexFile', related_name='primarysurvexfile', blank=True, null=True)
+    # could also include files in directory but not referenced
+    
+    class Meta:
+        ordering = ('id',)
+    
+class SurvexFile(models.Model):
+    path = models.CharField(max_length=200)
+    survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True)
+    cave = models.ForeignKey('Cave', blank=True, null=True)
+    
+    class Meta:
+        ordering = ('id',)
+    
+    def exists(self):
+        fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx")
+        return os.path.isfile(fname)
+    
+    def OpenFile(self):
+        fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx")
+        return open(fname)
+    
+    def SetDirectory(self):
+        dirpath = os.path.split(self.path)[0]
+        survexdirectorylist = SurvexDirectory.objects.filter(cave=self.cave, path=dirpath)
+        if survexdirectorylist:
+            self.survexdirectory = survexdirectorylist[0]
+        else:
+            survexdirectory = SurvexDirectory(path=dirpath, cave=self.cave, primarysurvexfile=self)
+            survexdirectory.save()
+            self.survexdirectory = survexdirectory
+        self.save()
+
 #
 # Single SurvexBlock 
 # 
 class SurvexBlock(models.Model):
-    name = models.CharField(max_length=100, blank=True, null=True)
+    name = models.CharField(max_length=100)
     parent = models.ForeignKey('SurvexBlock', blank=True, null=True)
     text = models.TextField()
-    
+    cave = models.ForeignKey('Cave', blank=True, null=True)
-    # non-useful representation of incomplete data
-    start_year = models.IntegerField(blank=True, null=True)
-    start_month = models.IntegerField(blank=True, null=True)
-    start_day = models.IntegerField(blank=True, null=True)
-    end_year = models.IntegerField(blank=True, null=True)
-    end_month = models.IntegerField(blank=True, null=True)
-    end_day = models.IntegerField(blank=True, null=True)
     
     date    = models.DateField(blank=True, null=True)
-    survexpath  = models.CharField(max_length=100)
+    expedition = models.ForeignKey('Expedition', blank=True, null=True)
         
-    # superfluous
+    survexfile  = models.ForeignKey("SurvexFile", blank=True, null=True)
-    person  = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True)
+    begin_char = models.IntegerField()  # code for where in the survex data files this block sits
-   
+    survexpath = models.CharField(max_length=200)   # the path for the survex stations
-    # code for where in the survex data files this block sits
+    refscandir = models.CharField(max_length=100)
-    begin_file = models.CharField(max_length=200)
-    begin_char = models.IntegerField()
-    end_file = models.CharField(max_length=200, blank=True, null=True)
-    end_char = models.IntegerField(blank=True, null=True)
     
     class Meta:
-        ordering = ('date', 'survexpath')
+        ordering = ('id',)
 
     def __unicode__(self):
         return self.name and unicode(self.name) or 'no name'
     
-    def filewithoutsvx(self):
-        return self.begin_file[:-4]
-    
-    def filecontents(self):
-        f = os.path.join(settings.SURVEX_DATA, self.begin_file)
-        fin = open(f, "rb")
-        res = fin.read().decode("latin1")
-        fin.close()
-        return res
-        
     def GetPersonroles(self):
         res = [ ]
         for personrole in self.personrole_set.order_by('personexpedition'):
@@ -66,22 +78,16 @@ class SurvexBlock(models.Model):
                 res.append({'person':personrole.personexpedition.person, 'expeditionyear':personrole.personexpedition.expedition.year, 'roles':str(personrole.role)})
         return res
 
-
+class SurvexTitle(models.Model):
-#
+    survexblock = models.ForeignKey('SurvexBlock')
-# Replace this with a choice string in PersonRole
+    title = models.CharField(max_length=200)
-#
+    cave = models.ForeignKey('Cave', blank=True, null=True)
-class Role(models.Model):
-    name = models.CharField(max_length=50)
-    def __unicode__(self):
-        return unicode(self.name)
-
 
 #
 # member of a SurvexBlock
 #
 class PersonRole(models.Model):
     survex_block        = models.ForeignKey('SurvexBlock')
-    role                = models.ForeignKey('Role')  # to go
     
     ROLE_CHOICES = (
         ('insts','Instruments'),
@@ -89,15 +95,20 @@ class PersonRole(models.Model):
         ('notes','Notes'),
         ('pics','Pictures'),
         ('tape','Tape measure'),
+        ('useless','Useless'),
+        ('helper','Helper'),
+        ('disto','Disto'),
+        ('consultant','Consultant'),
         )
     nrole = models.CharField(choices=ROLE_CHOICES, max_length=200, blank=True, null=True)
 
     # increasing levels of precision
-    person              = models.ForeignKey('Person')
+    personname          = models.CharField(max_length=100)
-    personexpedition    = models.ForeignKey('PersonExpedition')
+    person              = models.ForeignKey('Person', blank=True, null=True)
+    personexpedition    = models.ForeignKey('PersonExpedition', blank=True, null=True)
     persontrip          = models.ForeignKey('PersonTrip', blank=True, null=True)  
 
     def __unicode__(self):
-        return unicode(self.person) + " - " + unicode(self.survex_block) + " - " + unicode(self.role)
+        return unicode(self.person) + " - " + unicode(self.survex_block) + " - " + unicode(self.nrole)
         
     

core/templatetags/link.py

@@ -6,3 +6,4 @@ register = template.Library()
 @register.filter()
 def link(value):
     return mark_safe("<a href=\'%s\'>"%value.get_absolute_url()+unicode(value)+"</a>")
+

core/templatetags/wiki_markup.py

@@ -7,9 +7,16 @@ from core.models import QM, Photo, LogbookEntry, Cave
 import re, urlparse
 
 register = template.Library()
+url_root=settings.URL_ROOT
 if settings.URL_ROOT.endswith('/'):
     url_root=settings.URL_ROOT[:-1]
                 
+
+@register.filter()
+def plusone(n):
+    return n + 1
+
+
 def wiki_list(line, listdepth):
         l = ""
         for d in listdepth:

core/views_caves.py

@@ -51,6 +51,8 @@ def ent(request, cave_id, ent_letter):
                                                 'letter': cave_and_ent.entrance_letter,})
 
 def survexblock(request, survexpath):
+    survexpath = re.sub("/", ".", survexpath)
+    print "jjjjjj", survexpath
     survexblock = models.SurvexBlock.objects.get(survexpath=survexpath)
     #ftext = survexblock.filecontents()
     ftext = survexblock.text

core/views_logbooks.py

@@ -81,9 +81,9 @@ def GetPersonChronology(personexpedition):
         survexpath = personrole.survex_block.survexpath
         
         if b.get(survexpath):
-            b[survexpath] += ", " + str(personrole.role)
+            b[survexpath] += ", " + str(personrole.nrole)
         else:
-            b[survexpath] = str(personrole.role)
+            b[survexpath] = str(personrole.nrole)
     
     # build up the tables
     rdates = res.keys()
@@ -95,7 +95,7 @@ def GetPersonChronology(personexpedition):
         persontrips = res[rdate].get("persontrips", [])
         personroles = list(res[rdate].get("personroles", {}).items())
         for n in range(max(len(persontrips), len(personroles))):
-            res2.append(((n == 0 and rdate or ""), (n < len(persontrips) and persontrips[n]), (n < len(personroles) and personroles[n])))
+            res2.append(((n == 0 and rdate or "--"), (n < len(persontrips) and persontrips[n]), (n < len(personroles) and personroles[n])))
             
     return res2
 

core/views_survex.py

@@ -7,7 +7,38 @@ import os
 import datetime
 import difflib
 
+from troggle.core.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry, Cave
+from troggle.core.models import SurvexBlock, PersonRole, SurvexFile, SurvexDirectory, SurvexTitle
+from parsers.people import GetPersonExpeditionNameLookup
+
 import troggle.settings as settings
+import parsers.survex
+
+survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau (kataster group 1623)
+; Cave: 
+
+*begin [surveyname]
+
+*export [connecting stations]
+
+*title "area title"
+*date 2999.99.99
+*team Insts [Caver]
+*team Insts [Caver]
+*team Notes [Caver]
+*instrument [set number]
+
+;ref.: 2009#NN
+
+*calibrate tape +0.0   ; +ve if tape was too short, -ve if too long
+
+*data normal from to tape compass clino
+1   2   3.90    298 -20
+
+*data passage station left right up down ignoreall
+1   [L]    [R]    [U]   [D]    comment
+
+*end [surveyname]"""        
         
         
 def ReplaceTabs(stext):
@@ -36,7 +67,7 @@ class SvxForm(forms.Form):
     def GetDiscCode(self):
         fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
         if not os.path.isfile(fname):
-            return None
+            return survextemplatefile
         fin = open(fname, "rb")
         svxtext = fin.read().decode("latin1")   # unicode(a, "latin1")
         svxtext = ReplaceTabs(svxtext).strip()
@@ -52,11 +83,19 @@ class SvxForm(forms.Form):
     def SaveCode(self, rcode):
         fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
         if not os.path.isfile(fname):
-            return False
+            # only save if appears valid
+            if re.search("\[|\]", rcode):   
+                return "Error: clean up all []s from the text"
+            mbeginend = re.search("(?s)\*begin\s+(\w+).*?\*end\s+(\w+)", rcode)
+            if not mbeginend:
+                return "Error: no begin/end block here"
+            if mbeginend.group(1) != mbeginend.group(2):
+                return "Error: mismatching beginend"
+                
         fout = open(fname, "w")
         res = fout.write(rcode.encode("latin1"))
         fout.close()
-        return True
+        return "SAVED"
 
     def Process(self):
         print "....\n\n\n....Processing\n\n\n"
@@ -91,7 +130,7 @@ def svx(request, survex_file):
             rcode = rform.cleaned_data['code']
             outputtype = rform.cleaned_data['outputtype']
             difflist = form.DiffCode(rcode)
-            print "ssss", rform.data
+            #print "ssss", rform.data
             
             if "revert" in rform.data:
                 pass
@@ -105,11 +144,8 @@ def svx(request, survex_file):
                     form.data['code'] = rcode
             if "save" in rform.data:
                 print "sssavvving"
-                if form.SaveCode(rcode):
+                message = form.SaveCode(rcode)
-                    message = "SAVVVED"
+                if message != "SAVED":
-                    # we will reload later
-                else:
-                    message = "FAILED TO SAVE"
                     form.data['code'] = rcode
             if "diff" in rform.data:
                 form.data['code'] = rcode
@@ -124,8 +160,8 @@ def svx(request, survex_file):
     if message:
         difflist.insert(0, message)
     
-    print [ form.data['code'] ]
+    #print [ form.data['code'] ]
-    svxincludes = re.findall('\*include\s+"?(.*?)(?:\.svx)?"?\s*?\n(?i)', form.data['code'] or "")
+    svxincludes = re.findall('\*include\s+(\S+)(?i)', form.data['code'] or "")
     
     vmap = {'settings': settings,
             'has_3d': os.path.isfile(settings.SURVEX_DATA + survex_file + ".3d"),
@@ -138,12 +174,11 @@ def svx(request, survex_file):
         return render_to_response('svxfiledifflistonly.html', vmap)
     return render_to_response('svxfile.html', vmap)
 
-def Dsvx(request, survex_file):
+def svxraw(request, survex_file):
-    svx = open(settings.SURVEX_DATA + survex_file + ".svx", "rb")
+    svx = open(os.path.join(settings.SURVEX_DATA, survex_file+".svx"), "rb")
     return HttpResponse(svx, mimetype="text")
 
 
-
 # The cavern running function
 def process(survex_file):
     cwd = os.getcwd()
@@ -179,19 +214,34 @@ def identifycavedircontents(gcavedir):
     subsvx = [ ]
     primesvx = None
     for f in os.listdir(gcavedir):
-        if os.path.isdir(os.path.join(gcavedir, f)):
+        if name == "204" and (f in ["skel.svx", "template.svx", "204withents.svx"]):
+            pass
+        elif name == "136" and (f in ["136-noents.svx"]):
+            pass
+        elif name == "115" and (f in ["115cufix.svx", "115fix.svx"]):
+            pass
+        
+        elif os.path.isdir(os.path.join(gcavedir, f)):
             if f[0] != ".":
                 subdirs.append(f)
         elif f[-4:] == ".svx":
             nf = f[:-4]
-            if nf == name:
+            
-                assert not primesvx
+            if nf.lower() == name.lower() or nf[:3] == "all" or (name, nf) in [("144arge", "144"), ("resurvey2005", "145-2005"), ("cucc", "cu115")]:
-                primesvx = nf
+                if primesvx:
+                    if nf[:3] == "all":
+                        assert primesvx[:3] != "all", (name, nf, primesvx, gcavedir, subsvx)
+                        primesvx = nf
+                    else:
+                        assert primesvx[:3] == "all", (name, nf, primesvx, gcavedir, subsvx)
+                else:
+                    primesvx = nf
             else:
                 subsvx.append(nf)
         else:
-            assert re.match(".*?(?:.3d|.log|.err|.txt|.espec|~)$", f), (gcavedir, f)
+            assert re.match(".*?(?:.3d|.log|.err|.txt|.tmp|.diff|.e?spec|~)$", f), (gcavedir, f)
     subsvx.sort()
+    assert primesvx, (gcavedir, subsvx)
     if primesvx:
         subsvx.insert(0, primesvx)
     return subdirs, subsvx
@@ -206,25 +256,58 @@ def survexcaveslist(request):
     
     onefilecaves = [ ]
     multifilecaves = [ ]
+    subdircaves = [ ]
     
     # first sort the file list
-    fnumlist = [ (int(re.match("\d*", f).group(0) or "99999"), f)  for f in os.listdir(cavesdir) ]
+    fnumlist = [ (-int(re.match("\d*", f).group(0) or "0"), f)  for f in os.listdir(cavesdir) ]
     fnumlist.sort()
     
     # go through the list and identify the contents of each cave directory
     for num, cavedir in fnumlist:
+        if cavedir in ["144", "40"]:
+            continue
+            
         gcavedir = os.path.join(cavesdir, cavedir)
         if os.path.isdir(gcavedir) and cavedir[0] != ".":
             subdirs, subsvx = identifycavedircontents(gcavedir)
             survdirobj = [ ]
+            
             for lsubsvx in subsvx:
                 survdirobj.append(("caves/"+cavedir+"/"+lsubsvx, lsubsvx))
-            if len(survdirobj) == 1:
+            
-                onefilecaves.append(survdirobj[0])
+            # caves with subdirectories
-            else:
+            if subdirs:
+                subsurvdirs = [ ]
+                for subdir in subdirs:
+                    dsubdirs, dsubsvx = identifycavedircontents(os.path.join(gcavedir, subdir))
+                    assert not dsubdirs
+                    lsurvdirobj = [ ]
+                    for lsubsvx in dsubsvx:
+                        lsurvdirobj.append(("caves/"+cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))
+                    subsurvdirs.append((lsurvdirobj[0], lsurvdirobj[1:]))
+                subdircaves.append((cavedir, (survdirobj[0], survdirobj[1:]), subsurvdirs))
+            
+            # multifile caves
+            elif len(survdirobj) > 1:
                 multifilecaves.append((survdirobj[0], survdirobj[1:]))
+            # single file caves
+            else:
+                onefilecaves.append(survdirobj[0])
+    
+    return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves })
+    
+
+        
+
+
+
+# parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people
+# doesn't use recursion.  just writes it twice
+def survexcavesingle(request, survex_cave):
+    cave = Cave.objects.get(kataster_number=survex_cave)
+    parsers.survex.ReloadSurvexCave(survex_cave)
+    return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })
 
-    return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves})
     
     
     

databaseReset.py

@@ -30,6 +30,7 @@ def make_dirs():
 
 def import_cavetab():
     import parsers.cavetab
+    print "importing cavetab"
     parsers.cavetab.LoadCaveTab()
 
 def import_people():
@@ -74,8 +75,8 @@ def reset():
     make_dirs()
     import_cavetab()
     import_people()
-    import_logbooks()
     import_survex()
+    import_logbooks()
     import_QMs()
     import_surveys()
     import_descriptions()
@@ -97,11 +98,16 @@ def export_cavetab():
     outfile.close()
 
 if __name__ == "__main__":
+    import core.models
     import sys
     if "desc" in sys.argv:
         resetdesc()
     elif "reset" in sys.argv:
         reset()
+    elif "survex" in sys.argv:
+        management.call_command('syncdb', interactive=False)  # this sets the path so that import settings works in import_survex
+#        import_survex()
+        import_logbooks()
     else:
         print "Do 'python databaseReset.py reset'"
 

media/css/main3.css

@@ -205,7 +205,10 @@ a.redtext:link {
 	
 }
 
-
+td.survexnewfile 
+{
+    background-color:#f99; 
+}
 
 
 .behind {
@@ -233,13 +236,13 @@ img.thumbnail {
 }
 
 div#header {
-	position:absolute;
+	Dposition:absolute;
 	left:100px;
 	right:100px;
 	top:0;
 	margin-left:auto;
 	margin-right:auto;
-	height:50px;
+	Dheight:50px;
 	background-image: url( ../204plan.gif);
 	border-bottom:thin solid #000;
 	font-family: Arial, Helvetica, sans-serif;
@@ -248,7 +251,7 @@ div#header {
 
 
 div#editLinks {
-	position:absolute;
+	Zposition:absolute;
     background: #999;
 	bottom:0px;
 	right:0px;
@@ -263,7 +266,7 @@ div#editLinks a{
 }
 
 div#content {
-	margin-top: 50px;
+	Zmargin-top: 50px;
 	Zmargin-left: 120px;
 	Zmargin-right: 120px;
 	padding-top: 10px;
@@ -273,10 +276,15 @@ div#content {
 	background:#CCC;
 }
 
+.toolbarlinks
+{
+    padding:0;
+}
 
 .footer {
 	Dposition:fixed;
-	width:100%;
+ 	visibility:none; 
+    width:100%;
 	bottom:0;
 	left:0;
 }
@@ -357,9 +365,12 @@ div.codeframebit
     border:thin black solid;
     background-color: #ffffdf;
 }
-CodeMirror-line-numbers
+.CodeMirror-line-numbers
 {
-    background-color: #bbb;
+  background-color: #bbb;
+  font-family: monospace;
+  font-size: 10pt;
+  padding: .4em;
 }
 div#difflistajax
 {

parsers/logbooks.py

@@ -89,7 +89,7 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_
     for tripperson, time_underground in trippersons:
         lookupAttribs={'person_expedition':tripperson, 'logbook_entry':lbo}
         nonLookupAttribs={'time_underground':time_underground, 'date':date, 'is_logbook_entry_author':(tripperson == author)}
-        print nonLookupAttribs
+        #print nonLookupAttribs
         save_carefully(models.PersonTrip, lookupAttribs, nonLookupAttribs)
 
 
@@ -326,7 +326,7 @@ def LoadLogbookForExpedition(expedition):
         if lyear == year:
             break
     fin = open(os.path.join(expowebbase, lloc))
-    txt = fin.read()
+    txt = fin.read().decode("latin1")
     fin.close()
     parsefunc(year, expedition, txt)
     SetDatesFromLogbookEntries(expedition)

parsers/survex.py

@@ -2,158 +2,107 @@ import troggle.settings as settings
 import troggle.core.models as models
 
 from troggle.parsers.people import GetPersonExpeditionNameLookup
-
 import re
 import os
 
-roles = {"Insts": "Insts",
-         "insts": "Insts",
-         "Instruments": "Insts",
-         "instruments": "Insts",
-         "Inst": "Insts",
-         "inst": "Insts",
-         "dog": "Other",
-         "Dog": "Other",
-         "other": "Other",
-         "Other": "Other",
-         "Notes": "Notes",
-         "notes": "notes",
-         "pics": "Pics",
-         "Pics": "Pics",
-         "Tape": "Tape",
-         "tape": "Tape"}
 
-re_include_extension = re.compile(r"^\s*\*include\s+([^\s]*).svx\s*$", re.IGNORECASE)
+def RecursiveLoad(survexblock, survexfile, fin, textlines):
-re_include_no_extension = re.compile(r"^\s*\*include\s+([^\s]*)\s*$", re.IGNORECASE)
+    iblankbegins = 0
-flags = {"begin": re.compile(r"^\s*\*begin\s+(.*?)\s*$", re.IGNORECASE),
+    text = [ ]
-         "end": re.compile(r"^\s*\*end\s+(.*?)\s*$", re.IGNORECASE),
+    teammembers = [ ]
-         "date": re.compile(r"^\s*\*date\s+(.*?)\s*$", re.IGNORECASE),
+    while True:
-         "team": re.compile(r"^\s*\*team\s+(.*?)\s*$", re.IGNORECASE),
+        svxline = fin.readline().decode("latin1")
-         "title": re.compile(r"^\s*\*title\s+(.*?)\s*$", re.IGNORECASE)}
+        if not svxline:
+            return
+        textlines.append(svxline)
+        mstar = re.match('\s*\*(\w+)\s+(.*?)\s*(?:;.*)?$', svxline)
         
-def fileIterator(directory, filename):
+        #;ref.: 2008#18
-    survex_file = os.path.join(directory, filename + ".svx")
+        mref = re.match('.*?ref.*?(\d+#\d+)', svxline)
-    try:
+        if mref:
-        f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb")
+            survexblock.refscandir = mref.group(1)
-    except:
+            survexblock.save()
-        f = open(os.path.join(settings.SURVEX_DATA, survex_file).lower(), "rb")
+             
-    char = 0
+        if mstar:
-    for line in f.readlines():
+            cmd, line = mstar.groups()
-        line = unicode(line, "latin1")
+            
-        include_extension = re_include_extension.match(line)
+            if re.match("include$(?i)", cmd):
-        include_no_extension = re_include_no_extension.match(line)
+                includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub("\.svx$", "", line))
-        def a(include):
+                includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave)
-            link = re.split(r"/|\\", include)
+                includesurvexfile.save()
-            return fileIterator(os.path.join(directory, *link[:-1]), link[-1])
+                includesurvexfile.SetDirectory()
-        if include_extension:
+                if includesurvexfile.exists():
-            for sf, c, l in a(include_extension.groups()[0]):
+                    fininclude = includesurvexfile.OpenFile()
-                yield sf, c, l
+                    RecursiveLoad(survexblock, includesurvexfile, fininclude, textlines)
-        elif include_no_extension:
+            
-            for sf, c, l in a(include_no_extension.groups()[0]):
+            elif re.match("begin$(?i)", cmd):
-                yield sf, c, l
+                if line: 
-        else:
+                    name = line.lower()
-            yield survex_file, char, line
+                    survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile)
-        char = char + len(line)
+                    survexblockdown.save()
+                    textlinesdown = [ ]
+                    RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown)
+                else:
+                    iblankbegins += 1
+            
+            elif re.match("end$(?i)", cmd):
+                if iblankbegins:
+                    iblankbegins -= 1
+                else:
+                    survexblock.text = "".join(textlines)
+                    survexblock.save()
+                    return
+            
+            elif re.match("date$(?i)", cmd):
+                if len(line) == 10:
+                    survexblock.date = re.sub("\.", "-", line)
+                    expeditions = models.Expedition.objects.filter(year=line[:4])
+                    if expeditions:
+                        survexblock.expedition = expeditions[0]
+            
+            elif re.match("team$(?i)", cmd):
+                mteammember = re.match("(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line)
+                if mteammember:
+                    for tm in re.split(" and | / |, | & | \+ |^both$|^none$(?i)", mteammember.group(2)):
+                        if tm:
+                            personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
+                            if (personexpedition, tm) not in teammembers:
+                                teammembers.append((personexpedition, tm))
+                                personrole = models.PersonRole(survex_block=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
+                                personrole.save()
+                                
+            elif cmd == "title":
+                survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave)
+                survextitle.save()
+                
+            else:
+                assert cmd.lower() in [ "sd", "equate", "include", "units", "entrance", "fix", "data", "flags", "title", "export", "instrument", "calibrate", ], (cmd, line, survexblock)
         
 
-def make_model(name, parent, iter_lines, sf, c, l):
+def ReloadSurvexCave(survex_cave):
-    m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l)
+    cave = models.Cave.objects.get(kataster_number=survex_cave)
-    m.survexpath = m.name
+    cave.survexblock_set.all().delete()
-    if parent:
+    cave.survexfile_set.all().delete()
-        m.parent = parent
+    cave.survexdirectory_set.all().delete()
-        m.survexpath = m.parent.survexpath + "." + m.name
-    m.save()
     
-    # horrible local function
+    survexfile = models.SurvexFile(path="caves/" + survex_cave + "/" + survex_cave, cave=cave)
-    def saveEnd(survex_file, count):
+    survexfile.save()
-          if m.start_year and team:
+    survexfile.SetDirectory()
-              try:
-                  explist = models.Expedition.objects.filter(year = str(m.start_year))
-                  if not explist:
-                      return   # help hack
-                  exp = explist[0]
-                  for file_, (role, names) in team:
-                      if names.strip("\t").strip(" ") == "both" or names.strip("\t").strip(" ") == "Both":
-                          names = reduce(lambda x, y: x + u" & " + y,
-                                         [names for file_, (role, names) in team
-                                                if names.strip("\t").strip(" ") != "both"
-                                                   and names.strip("\t").strip(" ") != "Both"])
-                      for name in re.split("&|/|\+|,|;", names):
-                          sname = name.strip(". ").lower()
-                          try:
-                              personexpedition = GetPersonExpeditionNameLookup(exp).get(sname)
-                              if personexpedition:
-                                  models.PersonRole(personexpedition = personexpedition,
-                                                person = personexpedition.person,
-                                                survex_block = m,
-                                                role = models.Role.objects.get(name = roles[role])).save()
-                              else:
-                                  print ("no person", exp, sname, role)
-                          except AttributeError:
-                              print ("Person not found: " + name + " in " + file_ + "  " + role).encode('ascii', 'xmlcharrefreplace')
-              except AssertionError, inst:
-                  print (unicode(inst) + ": " + unicode(file_year[0])).encode('ascii', 'xmlcharrefreplace')
-              #except models.Expedition.DoesNotExist:
-              #    print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace')
-
-          m.end_file = survex_file
-          m.end_char = count
-
-          if m.start_day:
-              m.date = "%04d-%02d-%02d" % (int(m.start_year), int(m.start_month), int(m.start_day))
-
-          m.save()
-
-    team = []
-    file_year = None
-    for survex_file, count, line in iter_lines:
-      #Dictionary compreshension
-      res = dict([(key, regex.match(line.split(";")[0])) for key, regex in flags.iteritems()])
-      if res["begin"]:
-          make_model(res["begin"].groups()[0], m, iter_lines, survex_file, count, line)
-      else:
-          m.text = m.text + line
-      if res["end"]:
-          saveEnd(survex_file, count)
-          assert (res["end"].groups()[0]).lower() == (name).lower()
-          return None
-      elif res["date"]:
-          datere = re.match("(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:-(\d+))?(?:\.(\d+))?(?:\.(\d+))?",
-                            res["date"].groups()[0])
-          if datere is not None:
-              startYear, startMonth, startDay, endYear, endMonth, endDay = datere.groups()
-              m.start_year = startYear
-              m.start_month = startMonth
-              m.start_day = startDay
-              m.end_year = endYear
-              m.end_month = endMonth
-              m.end_day = endDay
-          file_year = survex_file, startYear
-      elif res["team"]:
-          h = re.match("((?:[Ii]nst(?:s|ruments)?)|(?:[Pp]ics)|(?:[Tt]ape)|(?:[Nn]otes)|(?:[Oo]ther))\s*(.*)",
-                       res["team"].groups()[0])
-          if h:
-              team.append((survex_file, h.groups()))
-          else:
-              print ("Role not found: " +  line + " in: " + sf).encode('ascii', 'xmlcharrefreplace')
-      elif res["title"]:
-          nsb, success = models.NewSubCave.objects.get_or_create(name = res["title"].groups()[0])
-
-    m.text = m.text + line
-    saveEnd(survex_file, count)
-
-
-#def LoadSurvexBlocks():
-#    survex_file = os.path.join(directory, filename + ".svx")
-#    f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb")
     
+    survexblockroot = models.SurvexBlock(name="root", survexpath="caves", begin_char=0, cave=cave, survexfile=survexfile)
+    survexblockroot.save()
+    fin = survexfile.OpenFile()
+    textlines = [ ]
+    RecursiveLoad(survexblockroot, survexfile, fin, textlines)
+    survexblockroot.text = "".join(textlines)
+    survexblockroot.save()
 
 def LoadAllSurvexBlocks():
-    models.Role.objects.all().delete()
+    caves = models.Cave.objects.all()
-    models.SurvexBlock.objects.all().delete()
+    for cave in caves:
-    for role in ["Insts", "Notes", "Pics", "Tape", "Other"]:
+        if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves", cave.kataster_number)):
-        models.Role(name = role).save()
+            if cave.kataster_number not in ['40']:
-    filename = "all"
+                print "loading", cave
-    make_model("all", None, fileIterator("", filename), filename, 0, "")
+                ReloadSurvexCave(cave.kataster_number)
     
 

templates/base.html

@@ -31,6 +31,15 @@
     {% endblock %}
    </div>
 </div>
+<div class="toolbarlinks">
+    <a href="{% url survexcaveslist %}">Cave Survex</a> | 
+    <a href="{% url survexcavessingle 161 %}">161</a> | 
+    <a href="{% url survexcavessingle 204 %}">204</a> | 
+    <a href="{% url survexcavessingle 258 %}">258</a> | 
+    <a href="{% url expedition 2008 %}">Expo2008</a> | 
+    <a href="{% url expedition 2009 %}">Expo2009</a> | 
+    <a href="/admin">Django admin</a>
+</div>
     
 <div id="nav">
   {% block nav %}
@@ -60,7 +69,7 @@
 
   </div>
 
-
+    <div class="footer">
 	<ul class="dropdown" id="footerLinks">
 
 		<li><a href="#">External links</a>
@@ -93,6 +102,6 @@
         <li class="toggleMenu"><a href="#">hide menu</a></li>
 
 	</ul>
-        <div class="toggleMenu" style="display:none; position:fixed; bottom:0; right:130px"><a href="#">Show menu</a></li>
+    </div>
 </body>
 </html>

templates/personexpedition.html

@@ -43,7 +43,7 @@
     <td class="survexblock"><a href="{% url survexblock persondate.2.0 %}">{{persondate.2.0}}</a></td>
     <td class="roles">{{persondate.2.1}}</td>
   {% else %}
-     <td colspan="2"> </td>
+    <td colspan="2"> </td>
   {% endif %}
 
 </tr>

templates/survexblock.html

@@ -8,7 +8,7 @@
 {% block content %}
 <h2>Survex Block {{survexblock.survexpath}}</h2>
 
-<p>Link to <a href="{% url svx survexblock.filewithoutsvx %}">{{survexblock.begin_file}}</a></p>
+<p>Link to <a href="{% url svx survexblock.survexfile.path %}">{{survexblock.survexfile.path}}</a></p>
 
 <p>Needs duplicates removed from right hand column</p>
 <p>Needs links to survex file presentation</p>

templates/svxcavesingle.html

@@ -0,0 +1,79 @@
+{% extends "base.html" %}
+{% load wiki_markup %}
+{% load link %}
+
+{% block title %}List of survex files{% endblock %}
+
+{% block content %}
+
+<h1>Surveys for {{cave}}</h1>
+
+<p>
+{% for survexdirectory in cave.survexdirectory_set.all %}
+  <a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>
+{% endfor %}
+</p>
+
+{% for survexdirectory in cave.survexdirectory_set.all %}
+<h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
+
+<table>
+<tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>Titles</th><th>Scans</th></tr>
+
+{% for survexfile in survexdirectory.survexfile_set.all %}
+<tr>
+  {% if survexfile.exists %}
+    <td rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
+  {% else %}
+    <td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
+  {% endif %}
+
+  {% ifequal survexfile survexdirectory.primarysurvexfile %}
+    <a href="{% url svx survexfile.path %}"><b>{{survexfile.path}}</b></a>
+  {% else %}
+    <a href="{% url svx survexfile.path %}">{{survexfile.path}}</a>
+  {% endifequal %}
+  </td>
+</tr>
+
+{% for survexblock in survexfile.survexblock_set.all %}
+<tr>
+  <td>{{survexblock.name}}</td>
+  <td>
+  {% if survexblock.expedition %}
+    <a href="{{survexblock.expedition.get_absolute_url}}">{{survexblock.date}}</a>
+  {% else %}
+    {{survexblock.date}}
+  {% endif %} 
+  </td>
+
+  <td>
+  {% for personrole in survexblock.personrole_set.all %}
+    {% if personrole.personexpedition %}
+      <a href="{{personrole.personexpedition.get_absolute_url}}">{{personrole.personname}}</a>
+    {% else %}
+      {{personrole.personname}}
+    {% endif %}
+  {% endfor %}
+  </td>
+
+  <td>
+  {% for survextitle in survexblock.survextitle_set.all %}
+    {{survextitle.title}}
+  {% endfor %}
+  </td>
+
+  <td>
+  {% if survexblock.refscandir %}
+    <b>{{survexblock.refscandir}}</b>
+  {% endif %}
+  </td>
+</tr>
+{% endfor %}
+{% endfor %}
+</table>
+
+{% endfor %}
+
+{% endblock %}
+

templates/svxfile.html

@@ -18,7 +18,7 @@ $(document).ready(function()
         stylesheet: "{{settings.MEDIA_URL}}CodeMirror-0.62/css/survexcolors.css",
         path: "{{settings.MEDIA_URL}}CodeMirror-0.62/js/",
         textWrapping: false, 
-        lineNumbers: true, 
+        lineNumbers: false, 
         indentUnit: 4, 
         tabMode: "spaces"
     });
@@ -36,7 +36,15 @@ $(document).ready(function()
 {% endblock %}
 
 {% block content %}
-<h1>Survex File: {{ title }} .svx</h1>
+<h1>Survex File: {{ title }}</h1>
+
+{% if svxincludes %}
+<p><b>Included files:</b>
+{% for svxinclude in svxincludes %}
+ <a href="{{svxinclude}}.svx">{{svxinclude}}</a>
+{% endfor %}
+</p>
+{% endif %}
 
 <form id="codewikiform" action="" method="POST">
     <div class="codeframebit">{{form.code}}</div>
@@ -66,12 +74,4 @@ LOGMESSAGES
 {% endif %}
 </div>
 
-{% if svxincludes %}
-<p><b>Included files:</b>
-{% for svxinclude in svxincludes %}
- <a href="{{svxinclude}}.svx">{{svxinclude}}</a>
-{% endfor %}
-</p>
-{% endif %}
-
 {% endblock %}

templates/svxfilecavelist.html

@@ -5,27 +5,56 @@
 {% block title %}List of survex files{% endblock %}
 
 {% block content %}
-<h1>List of survex directories</h1>
+<p><a href="#cdir">caves with subdirectories</a> | <a href="#cmult">caves with multiple files</a> | <a href="#csing">caves with single files</a></p>
 
-<p>{{message}}</p>
+<h2 id="cdir">Caves with subdirectories</h2>
-
-<h2>Caves of multiple files</h2>
 
+{% for subdircave, cavefiles, subsurvdirs in subdircaves %}
+<h3>{{cavefiles.0.1}} - <a href="{% url survexcavessingle cavefiles.0.1 %}">dates and explorers</a></h3>
 <table>
-<tr><th>Primary file</th><th>Survex files</th></tr>
+<tr>
-{% for primarycavefile, subcavefiles in multifilecaves %}
+  <td><b><a href="{% url svx cavefiles.0.0 %}">{{cavefiles.0.1}}</a></b></td>
-  <tr>
+  <td>
+  {% for cavepath, cavename in cavefiles.1 %}
+    <a href="{% url svx cavepath %}">{{cavename}}</a>
+  {% endfor %}
+  </td>
+</tr>
+
+{% for primarycavefile, subcavefiles in subsurvdirs %}
+<tr>
   <td><a href="{% url svx primarycavefile.0 %}">{{primarycavefile.1}}</a></td>
   <td>
     {% for cavepath, cavename in subcavefiles %}
        <a href="{% url svx cavepath %}">{{cavename}}</a>
     {% endfor %}
   </td>
-  </tr>
+</tr>  
 {% endfor %}
 </table>
 
-<h2>Caves of one file</h2>
+{% endfor %}
+
+
+<h2 id="cmult">Caves of multiple files</h2>
+<table>
+<tr><th>Dates and explorers</th><th>Survex files</th></tr>
+{% for primarycavefile, subcavefiles in multifilecaves %}
+<tr>
+  <td>
+     <a href="{% url survexcavessingle primarycavefile.1 %}">{{primarycavefile.1}}</a>
+  </td>
+  <td>
+    <a href="{% url svx primarycavefile.0 %}">{{primarycavefile.1}}</a> - 
+    {% for cavepath, cavename in subcavefiles %}
+       <a href="{% url svx cavepath %}">{{cavename}}</a>
+    {% endfor %}
+  </td>
+</tr>
+{% endfor %}
+</table>
+
+<h2 id="csing">Caves of one file</h2>
 <p>
 {% for cavepath, cavename in onefilecaves %}
    <a href="{% url svx cavepath %}">{{cavename}}</a>

urls.py

@@ -81,7 +81,10 @@ urlpatterns = patterns('',
     url(r'^survexblock/(.+)$',                     views_caves.survexblock, name="survexblock"),
     url(r'^survexfile/(?P<survex_file>.*?)\.svx$', views_survex.svx,        name="svx"),
     url(r'^survexfile/(?P<survex_file>.*)\.3d$',   views_survex.threed,     name="threed"),
-    url(r'^survexfile/caves$',                     views_survex.survexcaveslist,name="survexcaveslist"),
+    url(r'^survexfile/caves$',                     views_survex.survexcaveslist,  name="survexcaveslist"),
+    url(r'^survexfile/caves/(?P<survex_cave>.*)$', views_survex.survexcavesingle, name="survexcavessingle"),
+    url(r'^survexfileraw/(?P<survex_file>.*?)\.svx$', views_survex.svxraw,        name="svxraw"),
+            
     (r'^survex/(?P<survex_file>.*)\.log$', log),
     (r'^survex/(?P<survex_file>.*)\.err$', err),