xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

[svn] now with ability to make new svx file

Browse Source
This commit is contained in:
goatchurch 2009-08-01 07:31:27 +01:00
parent b135ab64e7
commit 60dcb82ef6
18 changed files with 449 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/admin.py
View File

@ -118,7 +118,6 @@ admin.site.register(Expedition)
admin.site.register(Person,PersonAdmin) admin.site.register(Person,PersonAdmin)
admin.site.register(PersonRole) admin.site.register(PersonRole)
admin.site.register(PersonExpedition,PersonExpeditionAdmin) admin.site.register(PersonExpedition,PersonExpeditionAdmin)
admin.site.register(Role)
admin.site.register(LogbookEntry, LogbookEntryAdmin) admin.site.register(LogbookEntry, LogbookEntryAdmin)
#admin.site.register(PersonTrip) #admin.site.register(PersonTrip)
admin.site.register(QM, QMAdmin) admin.site.register(QM, QMAdmin)

105
core/models_survex.py
View File

@ -6,57 +6,69 @@ import os
########################################################### ###########################################################
# These will allow browsing and editing of the survex data # These will allow browsing and editing of the survex data
########################################################### ###########################################################
# Needs to add: # Needs to add:
# SurvexFile
# Equates # Equates
# reloading # reloading
class SurvexDirectory(models.Model):
path = models.CharField(max_length=200)
cave = models.ForeignKey('Cave', blank=True, null=True)
primarysurvexfile = models.ForeignKey('SurvexFile', related_name='primarysurvexfile', blank=True, null=True)
# could also include files in directory but not referenced
class Meta:
ordering = ('id',)
class SurvexFile(models.Model):
path = models.CharField(max_length=200)
survexdirectory = models.ForeignKey("SurvexDirectory", blank=True, null=True)
cave = models.ForeignKey('Cave', blank=True, null=True)
class Meta:
ordering = ('id',)
def exists(self):
fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx")
return os.path.isfile(fname)
def OpenFile(self):
fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx")
return open(fname)
def SetDirectory(self):
dirpath = os.path.split(self.path)[0]
survexdirectorylist = SurvexDirectory.objects.filter(cave=self.cave, path=dirpath)
if survexdirectorylist:
self.survexdirectory = survexdirectorylist[0]
else:
survexdirectory = SurvexDirectory(path=dirpath, cave=self.cave, primarysurvexfile=self)
survexdirectory.save()
self.survexdirectory = survexdirectory
self.save()
# #
# Single SurvexBlock # Single SurvexBlock
# #
class SurvexBlock(models.Model): class SurvexBlock(models.Model):
name = models.CharField(max_length=100, blank=True, null=True) name = models.CharField(max_length=100)
parent = models.ForeignKey('SurvexBlock', blank=True, null=True) parent = models.ForeignKey('SurvexBlock', blank=True, null=True)
text = models.TextField() text = models.TextField()
cave = models.ForeignKey('Cave', blank=True, null=True)
# non-useful representation of incomplete data
start_year = models.IntegerField(blank=True, null=True)
start_month = models.IntegerField(blank=True, null=True)
start_day = models.IntegerField(blank=True, null=True)
end_year = models.IntegerField(blank=True, null=True)
end_month = models.IntegerField(blank=True, null=True)
end_day = models.IntegerField(blank=True, null=True)
date = models.DateField(blank=True, null=True) date = models.DateField(blank=True, null=True)
survexpath = models.CharField(max_length=100) expedition = models.ForeignKey('Expedition', blank=True, null=True)
# superfluous survexfile = models.ForeignKey("SurvexFile", blank=True, null=True)
person = models.ManyToManyField('Person', through='PersonRole', blank=True, null=True) begin_char = models.IntegerField() # code for where in the survex data files this block sits
survexpath = models.CharField(max_length=200) # the path for the survex stations
# code for where in the survex data files this block sits refscandir = models.CharField(max_length=100)
begin_file = models.CharField(max_length=200)
begin_char = models.IntegerField()
end_file = models.CharField(max_length=200, blank=True, null=True)
end_char = models.IntegerField(blank=True, null=True)
class Meta: class Meta:
ordering = ('date', 'survexpath') ordering = ('id',)
def __unicode__(self): def __unicode__(self):
return self.name and unicode(self.name) or 'no name' return self.name and unicode(self.name) or 'no name'
def filewithoutsvx(self):
return self.begin_file[:-4]
def filecontents(self):
f = os.path.join(settings.SURVEX_DATA, self.begin_file)
fin = open(f, "rb")
res = fin.read().decode("latin1")
fin.close()
return res
def GetPersonroles(self): def GetPersonroles(self):
res = [ ] res = [ ]
for personrole in self.personrole_set.order_by('personexpedition'): for personrole in self.personrole_set.order_by('personexpedition'):
@ -66,22 +78,16 @@ class SurvexBlock(models.Model):
res.append({'person':personrole.personexpedition.person, 'expeditionyear':personrole.personexpedition.expedition.year, 'roles':str(personrole.role)}) res.append({'person':personrole.personexpedition.person, 'expeditionyear':personrole.personexpedition.expedition.year, 'roles':str(personrole.role)})
return res return res
class SurvexTitle(models.Model):
# survexblock = models.ForeignKey('SurvexBlock')
# Replace this with a choice string in PersonRole title = models.CharField(max_length=200)
# cave = models.ForeignKey('Cave', blank=True, null=True)
class Role(models.Model):
name = models.CharField(max_length=50)
def __unicode__(self):
return unicode(self.name)
# #
# member of a SurvexBlock # member of a SurvexBlock
# #
class PersonRole(models.Model): class PersonRole(models.Model):
survex_block = models.ForeignKey('SurvexBlock') survex_block = models.ForeignKey('SurvexBlock')
role = models.ForeignKey('Role') # to go
ROLE_CHOICES = ( ROLE_CHOICES = (
('insts','Instruments'), ('insts','Instruments'),
@ -89,15 +95,20 @@ class PersonRole(models.Model):
('notes','Notes'), ('notes','Notes'),
('pics','Pictures'), ('pics','Pictures'),
('tape','Tape measure'), ('tape','Tape measure'),
('useless','Useless'),
('helper','Helper'),
('disto','Disto'),
('consultant','Consultant'),
) )
nrole = models.CharField(choices=ROLE_CHOICES, max_length=200, blank=True, null=True) nrole = models.CharField(choices=ROLE_CHOICES, max_length=200, blank=True, null=True)
# increasing levels of precision # increasing levels of precision
person = models.ForeignKey('Person') personname = models.CharField(max_length=100)
personexpedition = models.ForeignKey('PersonExpedition') person = models.ForeignKey('Person', blank=True, null=True)
personexpedition = models.ForeignKey('PersonExpedition', blank=True, null=True)
persontrip = models.ForeignKey('PersonTrip', blank=True, null=True) persontrip = models.ForeignKey('PersonTrip', blank=True, null=True)
def __unicode__(self): def __unicode__(self):
return unicode(self.person) + " - " + unicode(self.survex_block) + " - " + unicode(self.role) return unicode(self.person) + " - " + unicode(self.survex_block) + " - " + unicode(self.nrole)

3
core/templatetags/link.py
View File

@ -5,4 +5,5 @@ register = template.Library()
@register.filter() @register.filter()
def link(value): def link(value):
return mark_safe("<a href=\'%s\'>"%value.get_absolute_url()+unicode(value)+"</a>") return mark_safe("<a href=\'%s\'>"%value.get_absolute_url()+unicode(value)+"</a>")

7
core/templatetags/wiki_markup.py
View File

@ -7,9 +7,16 @@ from core.models import QM, Photo, LogbookEntry, Cave
import re, urlparse import re, urlparse
register = template.Library() register = template.Library()
url_root=settings.URL_ROOT
if settings.URL_ROOT.endswith('/'): if settings.URL_ROOT.endswith('/'):
url_root=settings.URL_ROOT[:-1] url_root=settings.URL_ROOT[:-1]
@register.filter()
def plusone(n):
return n + 1
def wiki_list(line, listdepth): def wiki_list(line, listdepth):
l = "" l = ""
for d in listdepth: for d in listdepth:

2
core/views_caves.py
View File

@ -51,6 +51,8 @@ def ent(request, cave_id, ent_letter):
'letter': cave_and_ent.entrance_letter,}) 'letter': cave_and_ent.entrance_letter,})
def survexblock(request, survexpath): def survexblock(request, survexpath):
survexpath = re.sub("/", ".", survexpath)
print "jjjjjj", survexpath
survexblock = models.SurvexBlock.objects.get(survexpath=survexpath) survexblock = models.SurvexBlock.objects.get(survexpath=survexpath)
#ftext = survexblock.filecontents() #ftext = survexblock.filecontents()
ftext = survexblock.text ftext = survexblock.text

6
core/views_logbooks.py
View File

@ -81,9 +81,9 @@ def GetPersonChronology(personexpedition):
survexpath = personrole.survex_block.survexpath survexpath = personrole.survex_block.survexpath
if b.get(survexpath): if b.get(survexpath):
b[survexpath] += ", " + str(personrole.role) b[survexpath] += ", " + str(personrole.nrole)
else: else:
b[survexpath] = str(personrole.role) b[survexpath] = str(personrole.nrole)
# build up the tables # build up the tables
rdates = res.keys() rdates = res.keys()
@ -95,7 +95,7 @@ def GetPersonChronology(personexpedition):
persontrips = res[rdate].get("persontrips", []) persontrips = res[rdate].get("persontrips", [])
personroles = list(res[rdate].get("personroles", {}).items()) personroles = list(res[rdate].get("personroles", {}).items())
for n in range(max(len(persontrips), len(personroles))): for n in range(max(len(persontrips), len(personroles))):
res2.append(((n == 0 and rdate or ""), (n < len(persontrips) and persontrips[n]), (n < len(personroles) and personroles[n]))) res2.append(((n == 0 and rdate or "--"), (n < len(persontrips) and persontrips[n]), (n < len(personroles) and personroles[n])))
return res2 return res2

135
core/views_survex.py
View File

@ -7,8 +7,39 @@ import os
import datetime import datetime
import difflib import difflib
import troggle.settings as settings from troggle.core.models import Expedition, Person, PersonExpedition, PersonTrip, LogbookEntry, Cave
from troggle.core.models import SurvexBlock, PersonRole, SurvexFile, SurvexDirectory, SurvexTitle
from parsers.people import GetPersonExpeditionNameLookup
import troggle.settings as settings
import parsers.survex
survextemplatefile = """; Locn: Totes Gebirge, Austria - Loser/Augst-Eck Plateau (kataster group 1623)
; Cave:
*begin [surveyname]
*export [connecting stations]
*title "area title"
*date 2999.99.99
*team Insts [Caver]
*team Insts [Caver]
*team Notes [Caver]
*instrument [set number]
;ref.: 2009#NN
*calibrate tape +0.0 ; +ve if tape was too short, -ve if too long
*data normal from to tape compass clino
1 2 3.90 298 -20
*data passage station left right up down ignoreall
1 [L] [R] [U] [D] comment
*end [surveyname]"""
def ReplaceTabs(stext): def ReplaceTabs(stext):
res = [ ] res = [ ]
@ -36,7 +67,7 @@ class SvxForm(forms.Form):
def GetDiscCode(self): def GetDiscCode(self):
fname = settings.SURVEX_DATA + self.data['filename'] + ".svx" fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
if not os.path.isfile(fname): if not os.path.isfile(fname):
return None return survextemplatefile
fin = open(fname, "rb") fin = open(fname, "rb")
svxtext = fin.read().decode("latin1") # unicode(a, "latin1") svxtext = fin.read().decode("latin1") # unicode(a, "latin1")
svxtext = ReplaceTabs(svxtext).strip() svxtext = ReplaceTabs(svxtext).strip()
@ -52,11 +83,19 @@ class SvxForm(forms.Form):
def SaveCode(self, rcode): def SaveCode(self, rcode):
fname = settings.SURVEX_DATA + self.data['filename'] + ".svx" fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
if not os.path.isfile(fname): if not os.path.isfile(fname):
return False # only save if appears valid
if re.search("\[|\]", rcode):
return "Error: clean up all []s from the text"
mbeginend = re.search("(?s)\*begin\s+(\w+).*?\*end\s+(\w+)", rcode)
if not mbeginend:
return "Error: no begin/end block here"
if mbeginend.group(1) != mbeginend.group(2):
return "Error: mismatching beginend"
fout = open(fname, "w") fout = open(fname, "w")
res = fout.write(rcode.encode("latin1")) res = fout.write(rcode.encode("latin1"))
fout.close() fout.close()
return True return "SAVED"
def Process(self): def Process(self):
print "....\n\n\n....Processing\n\n\n" print "....\n\n\n....Processing\n\n\n"
@ -91,7 +130,7 @@ def svx(request, survex_file):
rcode = rform.cleaned_data['code'] rcode = rform.cleaned_data['code']
outputtype = rform.cleaned_data['outputtype'] outputtype = rform.cleaned_data['outputtype']
difflist = form.DiffCode(rcode) difflist = form.DiffCode(rcode)
print "ssss", rform.data #print "ssss", rform.data
if "revert" in rform.data: if "revert" in rform.data:
pass pass
@ -105,11 +144,8 @@ def svx(request, survex_file):
form.data['code'] = rcode form.data['code'] = rcode
if "save" in rform.data: if "save" in rform.data:
print "sssavvving" print "sssavvving"
if form.SaveCode(rcode): message = form.SaveCode(rcode)
message = "SAVVVED" if message != "SAVED":
# we will reload later
else:
message = "FAILED TO SAVE"
form.data['code'] = rcode form.data['code'] = rcode
if "diff" in rform.data: if "diff" in rform.data:
form.data['code'] = rcode form.data['code'] = rcode
@ -124,8 +160,8 @@ def svx(request, survex_file):
if message: if message:
difflist.insert(0, message) difflist.insert(0, message)
print [ form.data['code'] ] #print [ form.data['code'] ]
svxincludes = re.findall('\*include\s+"?(.*?)(?:\.svx)?"?\s*?\n(?i)', form.data['code'] or "") svxincludes = re.findall('\*include\s+(\S+)(?i)', form.data['code'] or "")
vmap = {'settings': settings, vmap = {'settings': settings,
'has_3d': os.path.isfile(settings.SURVEX_DATA + survex_file + ".3d"), 'has_3d': os.path.isfile(settings.SURVEX_DATA + survex_file + ".3d"),
@ -138,12 +174,11 @@ def svx(request, survex_file):
return render_to_response('svxfiledifflistonly.html', vmap) return render_to_response('svxfiledifflistonly.html', vmap)
return render_to_response('svxfile.html', vmap) return render_to_response('svxfile.html', vmap)
def Dsvx(request, survex_file): def svxraw(request, survex_file):
svx = open(settings.SURVEX_DATA + survex_file + ".svx", "rb") svx = open(os.path.join(settings.SURVEX_DATA, survex_file+".svx"), "rb")
return HttpResponse(svx, mimetype="text") return HttpResponse(svx, mimetype="text")
# The cavern running function # The cavern running function
def process(survex_file): def process(survex_file):
cwd = os.getcwd() cwd = os.getcwd()
@ -179,19 +214,34 @@ def identifycavedircontents(gcavedir):
subsvx = [ ] subsvx = [ ]
primesvx = None primesvx = None
for f in os.listdir(gcavedir): for f in os.listdir(gcavedir):
if os.path.isdir(os.path.join(gcavedir, f)): if name == "204" and (f in ["skel.svx", "template.svx", "204withents.svx"]):
pass
elif name == "136" and (f in ["136-noents.svx"]):
pass
elif name == "115" and (f in ["115cufix.svx", "115fix.svx"]):
pass
elif os.path.isdir(os.path.join(gcavedir, f)):
if f[0] != ".": if f[0] != ".":
subdirs.append(f) subdirs.append(f)
elif f[-4:] == ".svx": elif f[-4:] == ".svx":
nf = f[:-4] nf = f[:-4]
if nf == name:
assert not primesvx if nf.lower() == name.lower() or nf[:3] == "all" or (name, nf) in [("144arge", "144"), ("resurvey2005", "145-2005"), ("cucc", "cu115")]:
primesvx = nf if primesvx:
if nf[:3] == "all":
assert primesvx[:3] != "all", (name, nf, primesvx, gcavedir, subsvx)
primesvx = nf
else:
assert primesvx[:3] == "all", (name, nf, primesvx, gcavedir, subsvx)
else:
primesvx = nf
else: else:
subsvx.append(nf) subsvx.append(nf)
else: else:
assert re.match(".*?(?:.3d|.log|.err|.txt|.espec|~)$", f), (gcavedir, f) assert re.match(".*?(?:.3d|.log|.err|.txt|.tmp|.diff|.e?spec|~)$", f), (gcavedir, f)
subsvx.sort() subsvx.sort()
assert primesvx, (gcavedir, subsvx)
if primesvx: if primesvx:
subsvx.insert(0, primesvx) subsvx.insert(0, primesvx)
return subdirs, subsvx return subdirs, subsvx
@ -206,25 +256,58 @@ def survexcaveslist(request):
onefilecaves = [ ] onefilecaves = [ ]
multifilecaves = [ ] multifilecaves = [ ]
subdircaves = [ ]
# first sort the file list # first sort the file list
fnumlist = [ (int(re.match("\d*", f).group(0) or "99999"), f) for f in os.listdir(cavesdir) ] fnumlist = [ (-int(re.match("\d*", f).group(0) or "0"), f) for f in os.listdir(cavesdir) ]
fnumlist.sort() fnumlist.sort()
# go through the list and identify the contents of each cave directory # go through the list and identify the contents of each cave directory
for num, cavedir in fnumlist: for num, cavedir in fnumlist:
if cavedir in ["144", "40"]:
continue
gcavedir = os.path.join(cavesdir, cavedir) gcavedir = os.path.join(cavesdir, cavedir)
if os.path.isdir(gcavedir) and cavedir[0] != ".": if os.path.isdir(gcavedir) and cavedir[0] != ".":
subdirs, subsvx = identifycavedircontents(gcavedir) subdirs, subsvx = identifycavedircontents(gcavedir)
survdirobj = [ ] survdirobj = [ ]
for lsubsvx in subsvx: for lsubsvx in subsvx:
survdirobj.append(("caves/"+cavedir+"/"+lsubsvx, lsubsvx)) survdirobj.append(("caves/"+cavedir+"/"+lsubsvx, lsubsvx))
if len(survdirobj) == 1:
onefilecaves.append(survdirobj[0]) # caves with subdirectories
else: if subdirs:
subsurvdirs = [ ]
for subdir in subdirs:
dsubdirs, dsubsvx = identifycavedircontents(os.path.join(gcavedir, subdir))
assert not dsubdirs
lsurvdirobj = [ ]
for lsubsvx in dsubsvx:
lsurvdirobj.append(("caves/"+cavedir+"/"+subdir+"/"+lsubsvx, lsubsvx))
subsurvdirs.append((lsurvdirobj[0], lsurvdirobj[1:]))
subdircaves.append((cavedir, (survdirobj[0], survdirobj[1:]), subsurvdirs))
# multifile caves
elif len(survdirobj) > 1:
multifilecaves.append((survdirobj[0], survdirobj[1:])) multifilecaves.append((survdirobj[0], survdirobj[1:]))
# single file caves
else:
onefilecaves.append(survdirobj[0])
return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves}) return render_to_response('svxfilecavelist.html', {'settings': settings, "onefilecaves":onefilecaves, "multifilecaves":multifilecaves, "subdircaves":subdircaves })
# parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people
# doesn't use recursion. just writes it twice
def survexcavesingle(request, survex_cave):
cave = Cave.objects.get(kataster_number=survex_cave)
parsers.survex.ReloadSurvexCave(survex_cave)
return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })

8
databaseReset.py
View File

@ -30,6 +30,7 @@ def make_dirs():
def import_cavetab(): def import_cavetab():
import parsers.cavetab import parsers.cavetab
print "importing cavetab"
parsers.cavetab.LoadCaveTab() parsers.cavetab.LoadCaveTab()
def import_people(): def import_people():
@ -74,8 +75,8 @@ def reset():
make_dirs() make_dirs()
import_cavetab() import_cavetab()
import_people() import_people()
import_logbooks()
import_survex() import_survex()
import_logbooks()
import_QMs() import_QMs()
import_surveys() import_surveys()
import_descriptions() import_descriptions()
@ -97,11 +98,16 @@ def export_cavetab():
outfile.close() outfile.close()
if __name__ == "__main__": if __name__ == "__main__":
import core.models
import sys import sys
if "desc" in sys.argv: if "desc" in sys.argv:
resetdesc() resetdesc()
elif "reset" in sys.argv: elif "reset" in sys.argv:
reset() reset()
elif "survex" in sys.argv:
management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
# import_survex()
import_logbooks()
else: else:
print "Do 'python databaseReset.py reset'" print "Do 'python databaseReset.py reset'"

27
media/css/main3.css
View File

@ -205,7 +205,10 @@ a.redtext:link {
} }
td.survexnewfile
{
background-color:#f99;
}
.behind { .behind {
@ -233,13 +236,13 @@ img.thumbnail {
} }
div#header { div#header {
position:absolute; Dposition:absolute;
left:100px; left:100px;
right:100px; right:100px;
top:0; top:0;
margin-left:auto; margin-left:auto;
margin-right:auto; margin-right:auto;
height:50px; Dheight:50px;
background-image: url( ../204plan.gif); background-image: url( ../204plan.gif);
border-bottom:thin solid #000; border-bottom:thin solid #000;
font-family: Arial, Helvetica, sans-serif; font-family: Arial, Helvetica, sans-serif;
@ -248,7 +251,7 @@ div#header {
div#editLinks { div#editLinks {
position:absolute; Zposition:absolute;
background: #999; background: #999;
bottom:0px; bottom:0px;
right:0px; right:0px;
@ -263,7 +266,7 @@ div#editLinks a{
} }
div#content { div#content {
margin-top: 50px; Zmargin-top: 50px;
Zmargin-left: 120px; Zmargin-left: 120px;
Zmargin-right: 120px; Zmargin-right: 120px;
padding-top: 10px; padding-top: 10px;
@ -273,10 +276,15 @@ div#content {
background:#CCC; background:#CCC;
} }
.toolbarlinks
{
padding:0;
}
.footer { .footer {
Dposition:fixed; Dposition:fixed;
width:100%; visibility:none;
width:100%;
bottom:0; bottom:0;
left:0; left:0;
} }
@ -357,9 +365,12 @@ div.codeframebit
border:thin black solid; border:thin black solid;
background-color: #ffffdf; background-color: #ffffdf;
} }
CodeMirror-line-numbers .CodeMirror-line-numbers
{ {
background-color: #bbb; background-color: #bbb;
font-family: monospace;
font-size: 10pt;
padding: .4em;
} }
div#difflistajax div#difflistajax
{ {

4
parsers/logbooks.py
View File

@ -89,7 +89,7 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_
for tripperson, time_underground in trippersons: for tripperson, time_underground in trippersons:
lookupAttribs={'person_expedition':tripperson, 'logbook_entry':lbo} lookupAttribs={'person_expedition':tripperson, 'logbook_entry':lbo}
nonLookupAttribs={'time_underground':time_underground, 'date':date, 'is_logbook_entry_author':(tripperson == author)} nonLookupAttribs={'time_underground':time_underground, 'date':date, 'is_logbook_entry_author':(tripperson == author)}
print nonLookupAttribs #print nonLookupAttribs
save_carefully(models.PersonTrip, lookupAttribs, nonLookupAttribs) save_carefully(models.PersonTrip, lookupAttribs, nonLookupAttribs)
@ -326,7 +326,7 @@ def LoadLogbookForExpedition(expedition):
if lyear == year: if lyear == year:
break break
fin = open(os.path.join(expowebbase, lloc)) fin = open(os.path.join(expowebbase, lloc))
txt = fin.read() txt = fin.read().decode("latin1")
fin.close() fin.close()
parsefunc(year, expedition, txt) parsefunc(year, expedition, txt)
SetDatesFromLogbookEntries(expedition) SetDatesFromLogbookEntries(expedition)

243
parsers/survex.py
View File

@ -2,158 +2,107 @@ import troggle.settings as settings
import troggle.core.models as models import troggle.core.models as models
from troggle.parsers.people import GetPersonExpeditionNameLookup from troggle.parsers.people import GetPersonExpeditionNameLookup
import re import re
import os import os
roles = {"Insts": "Insts",
"insts": "Insts",
"Instruments": "Insts",
"instruments": "Insts",
"Inst": "Insts",
"inst": "Insts",
"dog": "Other",
"Dog": "Other",
"other": "Other",
"Other": "Other",
"Notes": "Notes",
"notes": "notes",
"pics": "Pics",
"Pics": "Pics",
"Tape": "Tape",
"tape": "Tape"}
re_include_extension = re.compile(r"^\s*\*include\s+([^\s]*).svx\s*$", re.IGNORECASE) def RecursiveLoad(survexblock, survexfile, fin, textlines):
re_include_no_extension = re.compile(r"^\s*\*include\s+([^\s]*)\s*$", re.IGNORECASE) iblankbegins = 0
flags = {"begin": re.compile(r"^\s*\*begin\s+(.*?)\s*$", re.IGNORECASE), text = [ ]
"end": re.compile(r"^\s*\*end\s+(.*?)\s*$", re.IGNORECASE), teammembers = [ ]
"date": re.compile(r"^\s*\*date\s+(.*?)\s*$", re.IGNORECASE), while True:
"team": re.compile(r"^\s*\*team\s+(.*?)\s*$", re.IGNORECASE), svxline = fin.readline().decode("latin1")
"title": re.compile(r"^\s*\*title\s+(.*?)\s*$", re.IGNORECASE)} if not svxline:
return
def fileIterator(directory, filename): textlines.append(svxline)
survex_file = os.path.join(directory, filename + ".svx") mstar = re.match('\s*\*(\w+)\s+(.*?)\s*(?:;.*)?$', svxline)
try:
f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb") #;ref.: 2008#18
except: mref = re.match('.*?ref.*?(\d+#\d+)', svxline)
f = open(os.path.join(settings.SURVEX_DATA, survex_file).lower(), "rb") if mref:
char = 0 survexblock.refscandir = mref.group(1)
for line in f.readlines(): survexblock.save()
line = unicode(line, "latin1")
include_extension = re_include_extension.match(line) if mstar:
include_no_extension = re_include_no_extension.match(line) cmd, line = mstar.groups()
def a(include):
link = re.split(r"/|\\", include) if re.match("include$(?i)", cmd):
return fileIterator(os.path.join(directory, *link[:-1]), link[-1]) includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub("\.svx$", "", line))
if include_extension: includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave)
for sf, c, l in a(include_extension.groups()[0]): includesurvexfile.save()
yield sf, c, l includesurvexfile.SetDirectory()
elif include_no_extension: if includesurvexfile.exists():
for sf, c, l in a(include_no_extension.groups()[0]): fininclude = includesurvexfile.OpenFile()
yield sf, c, l RecursiveLoad(survexblock, includesurvexfile, fininclude, textlines)
else:
yield survex_file, char, line elif re.match("begin$(?i)", cmd):
char = char + len(line) if line:
name = line.lower()
survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile)
def make_model(name, parent, iter_lines, sf, c, l): survexblockdown.save()
m = models.SurvexBlock(name = name, begin_file = sf, begin_char = c, text = l) textlinesdown = [ ]
m.survexpath = m.name RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown)
if parent: else:
m.parent = parent iblankbegins += 1
m.survexpath = m.parent.survexpath + "." + m.name
m.save() elif re.match("end$(?i)", cmd):
if iblankbegins:
# horrible local function iblankbegins -= 1
def saveEnd(survex_file, count): else:
if m.start_year and team: survexblock.text = "".join(textlines)
try: survexblock.save()
explist = models.Expedition.objects.filter(year = str(m.start_year)) return
if not explist:
return # help hack elif re.match("date$(?i)", cmd):
exp = explist[0] if len(line) == 10:
for file_, (role, names) in team: survexblock.date = re.sub("\.", "-", line)
if names.strip("\t").strip(" ") == "both" or names.strip("\t").strip(" ") == "Both": expeditions = models.Expedition.objects.filter(year=line[:4])
names = reduce(lambda x, y: x + u" & " + y, if expeditions:
[names for file_, (role, names) in team survexblock.expedition = expeditions[0]
if names.strip("\t").strip(" ") != "both"
and names.strip("\t").strip(" ") != "Both"]) elif re.match("team$(?i)", cmd):
for name in re.split("&|/|\+|,|;", names): mteammember = re.match("(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line)
sname = name.strip(". ").lower() if mteammember:
try: for tm in re.split(" and | / |, | & | \+ |^both$|^none$(?i)", mteammember.group(2)):
personexpedition = GetPersonExpeditionNameLookup(exp).get(sname) if tm:
if personexpedition: personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
models.PersonRole(personexpedition = personexpedition, if (personexpedition, tm) not in teammembers:
person = personexpedition.person, teammembers.append((personexpedition, tm))
survex_block = m, personrole = models.PersonRole(survex_block=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
role = models.Role.objects.get(name = roles[role])).save() personrole.save()
else:
print ("no person", exp, sname, role) elif cmd == "title":
except AttributeError: survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave)
print ("Person not found: " + name + " in " + file_ + " " + role).encode('ascii', 'xmlcharrefreplace') survextitle.save()
except AssertionError, inst:
print (unicode(inst) + ": " + unicode(file_year[0])).encode('ascii', 'xmlcharrefreplace') else:
#except models.Expedition.DoesNotExist: assert cmd.lower() in [ "sd", "equate", "include", "units", "entrance", "fix", "data", "flags", "title", "export", "instrument", "calibrate", ], (cmd, line, survexblock)
# print "Expo"+str(file_year[1]).encode('ascii', 'xmlcharrefreplace')
m.end_file = survex_file
m.end_char = count
if m.start_day:
m.date = "%04d-%02d-%02d" % (int(m.start_year), int(m.start_month), int(m.start_day))
m.save()
team = []
file_year = None
for survex_file, count, line in iter_lines:
#Dictionary compreshension
res = dict([(key, regex.match(line.split(";")[0])) for key, regex in flags.iteritems()])
if res["begin"]:
make_model(res["begin"].groups()[0], m, iter_lines, survex_file, count, line)
else:
m.text = m.text + line
if res["end"]:
saveEnd(survex_file, count)
assert (res["end"].groups()[0]).lower() == (name).lower()
return None
elif res["date"]:
datere = re.match("(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:-(\d+))?(?:\.(\d+))?(?:\.(\d+))?",
res["date"].groups()[0])
if datere is not None:
startYear, startMonth, startDay, endYear, endMonth, endDay = datere.groups()
m.start_year = startYear
m.start_month = startMonth
m.start_day = startDay
m.end_year = endYear
m.end_month = endMonth
m.end_day = endDay
file_year = survex_file, startYear
elif res["team"]:
h = re.match("((?:[Ii]nst(?:s|ruments)?)|(?:[Pp]ics)|(?:[Tt]ape)|(?:[Nn]otes)|(?:[Oo]ther))\s*(.*)",
res["team"].groups()[0])
if h:
team.append((survex_file, h.groups()))
else:
print ("Role not found: " + line + " in: " + sf).encode('ascii', 'xmlcharrefreplace')
elif res["title"]:
nsb, success = models.NewSubCave.objects.get_or_create(name = res["title"].groups()[0])
m.text = m.text + line
saveEnd(survex_file, count)
#def LoadSurvexBlocks():
# survex_file = os.path.join(directory, filename + ".svx")
# f = open(os.path.join(settings.SURVEX_DATA, survex_file), "rb")
def ReloadSurvexCave(survex_cave):
cave = models.Cave.objects.get(kataster_number=survex_cave)
cave.survexblock_set.all().delete()
cave.survexfile_set.all().delete()
cave.survexdirectory_set.all().delete()
survexfile = models.SurvexFile(path="caves/" + survex_cave + "/" + survex_cave, cave=cave)
survexfile.save()
survexfile.SetDirectory()
survexblockroot = models.SurvexBlock(name="root", survexpath="caves", begin_char=0, cave=cave, survexfile=survexfile)
survexblockroot.save()
fin = survexfile.OpenFile()
textlines = [ ]
RecursiveLoad(survexblockroot, survexfile, fin, textlines)
survexblockroot.text = "".join(textlines)
survexblockroot.save()
def LoadAllSurvexBlocks(): def LoadAllSurvexBlocks():
models.Role.objects.all().delete() caves = models.Cave.objects.all()
models.SurvexBlock.objects.all().delete() for cave in caves:
for role in ["Insts", "Notes", "Pics", "Tape", "Other"]: if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves", cave.kataster_number)):
models.Role(name = role).save() if cave.kataster_number not in ['40']:
filename = "all" print "loading", cave
make_model("all", None, fileIterator("", filename), filename, 0, "") ReloadSurvexCave(cave.kataster_number)

13
templates/base.html
View File

@ -31,6 +31,15 @@
{% endblock %} {% endblock %}
</div> </div>
</div> </div>
<div class="toolbarlinks">
<a href="{% url survexcaveslist %}">Cave Survex</a> |
<a href="{% url survexcavessingle 161 %}">161</a> |
<a href="{% url survexcavessingle 204 %}">204</a> |
<a href="{% url survexcavessingle 258 %}">258</a> |
<a href="{% url expedition 2008 %}">Expo2008</a> |
<a href="{% url expedition 2009 %}">Expo2009</a> |
<a href="/admin">Django admin</a>
</div>
<div id="nav"> <div id="nav">
{% block nav %} {% block nav %}
@ -60,7 +69,7 @@
</div> </div>
<div class="footer">
<ul class="dropdown" id="footerLinks"> <ul class="dropdown" id="footerLinks">
<li><a href="#">External links</a> <li><a href="#">External links</a>
@ -93,6 +102,6 @@
<li class="toggleMenu"><a href="#">hide menu</a></li> <li class="toggleMenu"><a href="#">hide menu</a></li>
</ul> </ul>
<div class="toggleMenu" style="display:none; position:fixed; bottom:0; right:130px"><a href="#">Show menu</a></li> </div>
</body> </body>
</html> </html>

2
templates/personexpedition.html
View File

@ -43,7 +43,7 @@
<td class="survexblock"><a href="{% url survexblock persondate.2.0 %}">{{persondate.2.0}}</a></td> <td class="survexblock"><a href="{% url survexblock persondate.2.0 %}">{{persondate.2.0}}</a></td>
<td class="roles">{{persondate.2.1}}</td> <td class="roles">{{persondate.2.1}}</td>
{% else %} {% else %}
<td colspan="2"> </td> <td colspan="2"> </td>
{% endif %} {% endif %}
</tr> </tr>

2
templates/survexblock.html
View File

@ -8,7 +8,7 @@
{% block content %} {% block content %}
<h2>Survex Block {{survexblock.survexpath}}</h2> <h2>Survex Block {{survexblock.survexpath}}</h2>
<p>Link to <a href="{% url svx survexblock.filewithoutsvx %}">{{survexblock.begin_file}}</a></p> <p>Link to <a href="{% url svx survexblock.survexfile.path %}">{{survexblock.survexfile.path}}</a></p>
<p>Needs duplicates removed from right hand column</p> <p>Needs duplicates removed from right hand column</p>
<p>Needs links to survex file presentation</p> <p>Needs links to survex file presentation</p>

79
templates/svxcavesingle.html Normal file
View File

@ -0,0 +1,79 @@
{% extends "base.html" %}
{% load wiki_markup %}
{% load link %}
{% block title %}List of survex files{% endblock %}
{% block content %}
<h1>Surveys for {{cave}}</h1>
<p>
{% for survexdirectory in cave.survexdirectory_set.all %}
<a href="#T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</a>
{% endfor %}
</p>
{% for survexdirectory in cave.survexdirectory_set.all %}
<h3 id="T_{{survexdirectory.primarysurvexfile.path}}">{{survexdirectory.path}}</h3>
<table>
<tr><th>Survex file</th><th>Block</th><th>Date</th><th>Explorers</th><th>Titles</th><th>Scans</th></tr>
{% for survexfile in survexdirectory.survexfile_set.all %}
<tr>
{% if survexfile.exists %}
<td rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
{% else %}
<td class="survexnewfile" rowspan="{{survexfile.survexblock_set.all|length|plusone}}">
{% endif %}
{% ifequal survexfile survexdirectory.primarysurvexfile %}
<a href="{% url svx survexfile.path %}"><b>{{survexfile.path}}</b></a>
{% else %}
<a href="{% url svx survexfile.path %}">{{survexfile.path}}</a>
{% endifequal %}
</td>
</tr>
{% for survexblock in survexfile.survexblock_set.all %}
<tr>
<td>{{survexblock.name}}</td>
<td>
{% if survexblock.expedition %}
<a href="{{survexblock.expedition.get_absolute_url}}">{{survexblock.date}}</a>
{% else %}
{{survexblock.date}}
{% endif %}
</td>
<td>
{% for personrole in survexblock.personrole_set.all %}
{% if personrole.personexpedition %}
<a href="{{personrole.personexpedition.get_absolute_url}}">{{personrole.personname}}</a>
{% else %}
{{personrole.personname}}
{% endif %}
{% endfor %}
</td>
<td>
{% for survextitle in survexblock.survextitle_set.all %}
{{survextitle.title}}
{% endfor %}
</td>
<td>
{% if survexblock.refscandir %}
<b>{{survexblock.refscandir}}</b>
{% endif %}
</td>
</tr>
{% endfor %}
{% endfor %}
</table>
{% endfor %}
{% endblock %}

20
templates/svxfile.html
View File

@ -18,7 +18,7 @@ $(document).ready(function()
stylesheet: "{{settings.MEDIA_URL}}CodeMirror-0.62/css/survexcolors.css", stylesheet: "{{settings.MEDIA_URL}}CodeMirror-0.62/css/survexcolors.css",
path: "{{settings.MEDIA_URL}}CodeMirror-0.62/js/", path: "{{settings.MEDIA_URL}}CodeMirror-0.62/js/",
textWrapping: false, textWrapping: false,
lineNumbers: true, lineNumbers: false,
indentUnit: 4, indentUnit: 4,
tabMode: "spaces" tabMode: "spaces"
}); });
@ -36,7 +36,15 @@ $(document).ready(function()
{% endblock %} {% endblock %}
{% block content %} {% block content %}
<h1>Survex File: {{ title }} .svx</h1> <h1>Survex File: {{ title }}</h1>
{% if svxincludes %}
<p><b>Included files:</b>
{% for svxinclude in svxincludes %}
<a href="{{svxinclude}}.svx">{{svxinclude}}</a>
{% endfor %}
</p>
{% endif %}
<form id="codewikiform" action="" method="POST"> <form id="codewikiform" action="" method="POST">
<div class="codeframebit">{{form.code}}</div> <div class="codeframebit">{{form.code}}</div>
@ -66,12 +74,4 @@ LOGMESSAGES
{% endif %} {% endif %}
</div> </div>
{% if svxincludes %}
<p><b>Included files:</b>
{% for svxinclude in svxincludes %}
<a href="{{svxinclude}}.svx">{{svxinclude}}</a>
{% endfor %}
</p>
{% endif %}
{% endblock %} {% endblock %}

47
templates/svxfilecavelist.html
View File

@ -5,27 +5,56 @@
{% block title %}List of survex files{% endblock %} {% block title %}List of survex files{% endblock %}
{% block content %} {% block content %}
<h1>List of survex directories</h1> <p><a href="#cdir">caves with subdirectories</a> | <a href="#cmult">caves with multiple files</a> | <a href="#csing">caves with single files</a></p>
<p>{{message}}</p> <h2 id="cdir">Caves with subdirectories</h2>
<h2>Caves of multiple files</h2>
{% for subdircave, cavefiles, subsurvdirs in subdircaves %}
<h3>{{cavefiles.0.1}} - <a href="{% url survexcavessingle cavefiles.0.1 %}">dates and explorers</a></h3>
<table> <table>
<tr><th>Primary file</th><th>Survex files</th></tr> <tr>
{% for primarycavefile, subcavefiles in multifilecaves %} <td><b><a href="{% url svx cavefiles.0.0 %}">{{cavefiles.0.1}}</a></b></td>
<tr> <td>
{% for cavepath, cavename in cavefiles.1 %}
<a href="{% url svx cavepath %}">{{cavename}}</a>
{% endfor %}
</td>
</tr>
{% for primarycavefile, subcavefiles in subsurvdirs %}
<tr>
<td><a href="{% url svx primarycavefile.0 %}">{{primarycavefile.1}}</a></td> <td><a href="{% url svx primarycavefile.0 %}">{{primarycavefile.1}}</a></td>
<td> <td>
{% for cavepath, cavename in subcavefiles %} {% for cavepath, cavename in subcavefiles %}
<a href="{% url svx cavepath %}">{{cavename}}</a> <a href="{% url svx cavepath %}">{{cavename}}</a>
{% endfor %} {% endfor %}
</td> </td>
</tr> </tr>
{% endfor %} {% endfor %}
</table> </table>
<h2>Caves of one file</h2> {% endfor %}
<h2 id="cmult">Caves of multiple files</h2>
<table>
<tr><th>Dates and explorers</th><th>Survex files</th></tr>
{% for primarycavefile, subcavefiles in multifilecaves %}
<tr>
<td>
<a href="{% url survexcavessingle primarycavefile.1 %}">{{primarycavefile.1}}</a>
</td>
<td>
<a href="{% url svx primarycavefile.0 %}">{{primarycavefile.1}}</a> -
{% for cavepath, cavename in subcavefiles %}
<a href="{% url svx cavepath %}">{{cavename}}</a>
{% endfor %}
</td>
</tr>
{% endfor %}
</table>
<h2 id="csing">Caves of one file</h2>
<p> <p>
{% for cavepath, cavename in onefilecaves %} {% for cavepath, cavename in onefilecaves %}
<a href="{% url svx cavepath %}">{{cavename}}</a> <a href="{% url svx cavepath %}">{{cavename}}</a>

5
urls.py
View File

@ -81,7 +81,10 @@ urlpatterns = patterns('',
url(r'^survexblock/(.+)$', views_caves.survexblock, name="survexblock"), url(r'^survexblock/(.+)$', views_caves.survexblock, name="survexblock"),
url(r'^survexfile/(?P<survex_file>.*?)\.svx$', views_survex.svx, name="svx"), url(r'^survexfile/(?P<survex_file>.*?)\.svx$', views_survex.svx, name="svx"),
url(r'^survexfile/(?P<survex_file>.*)\.3d$', views_survex.threed, name="threed"), url(r'^survexfile/(?P<survex_file>.*)\.3d$', views_survex.threed, name="threed"),
url(r'^survexfile/caves$', views_survex.survexcaveslist,name="survexcaveslist"), url(r'^survexfile/caves$', views_survex.survexcaveslist, name="survexcaveslist"),
url(r'^survexfile/caves/(?P<survex_cave>.*)$', views_survex.survexcavesingle, name="survexcavessingle"),
url(r'^survexfileraw/(?P<survex_file>.*?)\.svx$', views_survex.svxraw, name="svxraw"),
(r'^survex/(?P<survex_file>.*)\.log$', log), (r'^survex/(?P<survex_file>.*)\.log$', log),
(r'^survex/(?P<survex_file>.*)\.err$', err), (r'^survex/(?P<survex_file>.*)\.err$', err),