xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

pathlib for path management & cavelist fixes

Browse Source
This commit is contained in:
Philip Sargent
2021-03-24 15:46:35 +00:00
parent 7f37327bcd
commit 9a91487375
5 changed files with 69 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
core/views_survex.py
View File

@@ -2,15 +2,17 @@ import re
import os
import datetime
import difflib
from pathlib import Path
from django import forms
from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import render_to_response, render
#from django.core.context_processors import csrf
from django.template.context_processors import csrf
from django.core.exceptions import ObjectDoesNotExist
from django.http import HttpResponse, Http404
from django.core.exceptions import ObjectDoesNotExist
import troggle.settings as settings
import parsers.survex
from troggle.core.models import Expedition, Person, PersonExpedition
@@ -18,6 +20,7 @@ from troggle.core.models_survex import SurvexBlock, SurvexPersonRole, SurvexFile
from troggle.core.models_caves import Cave, PersonTrip, LogbookEntry
from troggle.parsers.people import GetPersonExpeditionNameLookup
survexdatasetpath = Path(settings.SURVEX_DATA)
survextemplatefile = """; *** THIS IS A TEMPLATE FILE NOT WHAT YOU MIGHT BE EXPECTING ***
@@ -99,8 +102,9 @@ class SvxForm(forms.Form):
code = forms.CharField(widget=forms.Textarea(attrs={"cols":150, "rows":36}))
def GetDiscCode(self):
fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
fname = survexdatasetpath / (self.data['filename'] + ".svx")
if not os.path.isfile(fname):
print(">>> >>> WARNING - svx file not found, showiung TEMPLATE SVX",fname, flush=True)
return survextemplatefile
fin = open(fname, "rt",encoding='utf8',newline='')
svxtext = fin.read()
@@ -114,7 +118,7 @@ class SvxForm(forms.Form):
return difflist
def SaveCode(self, rcode):
fname = settings.SURVEX_DATA + self.data['filename'] + ".svx"
fname = survexdatasetpath / (self.data['filename'] + ".svx")
if not os.path.isfile(fname):
if re.search(r"\[|\]", rcode):
return "Error: remove all []s from the text. They are only template guidance."
@@ -129,7 +133,7 @@ class SvxForm(forms.Form):
fout = open(fname, "wt", encoding='utf8',newline='\n')
except FileNotFoundError:
pth = os.path.dirname(self.data['filename'])
newpath = os.path.join(settings.SURVEX_DATA, pth)
newpath = survexdatasetpath / pth
if not os.path.exists(newpath):
os.makedirs(newpath)
fout = open(fname, "wt", encoding='utf8',newline='\n')
@@ -141,11 +145,13 @@ class SvxForm(forms.Form):
def Process(self):
print("....\n\n\n....Processing\n\n\n")
froox = os.fspath(survexdatasetpath / (self.data['filename'] + ".svx"))
froog = os.fspath(survexdatasetpath / (self.data['filename'] + ".log"))
cwd = os.getcwd()
os.chdir(os.path.split(settings.SURVEX_DATA + self.data['filename'])[0])
os.system(settings.CAVERN + " --log " + settings.SURVEX_DATA + self.data['filename'] + ".svx")
os.chdir(os.path.split(froox)[0])
os.system(settings.CAVERN + " --log " + froox )
os.chdir(cwd)
fin = open(settings.SURVEX_DATA + self.data['filename'] + ".log", "rt",encoding='utf8')
fin = open(froog, "rt",encoding='utf8')
log = fin.read()
fin.close()
log = re.sub("(?s).*?(Survey contains)", "\\1", log)
@@ -206,7 +212,7 @@ def svx(request, survex_file):
svxincludes = re.findall(r'\*include\s+(\S+)(?i)', form.data['code'] or "")
vmap = {'settings': settings,
'has_3d': os.path.isfile(settings.SURVEX_DATA + survex_file + ".3d"),
'has_3d': os.path.isfile(survexdatasetpath / survex_file / ".3d"),
'title': survex_file,
'svxincludes': svxincludes,
'difflist': difflist,
@@ -218,37 +224,37 @@ def svx(request, survex_file):
return render_to_response('svxfile.html', vmap)
def svxraw(request, survex_file):
svx = open(os.path.join(settings.SURVEX_DATA, survex_file+".svx"), "rt",encoding='utf8')
svx = open(os.path.join(survexdatasetpath / survex_file / ".svx"), "rt",encoding='utf8')
return HttpResponse(svx, content_type="text")
# The cavern running function
def process(survex_file):
cwd = os.getcwd()
os.chdir(os.path.split(settings.SURVEX_DATA + survex_file)[0])
os.system(settings.CAVERN + " --log " + settings.SURVEX_DATA + survex_file + ".svx")
os.chdir(os.path.split(os.fspath(survexdatasetpath / survex_file))[0])
os.system(settings.CAVERN + " --log " + survexdatasetpath / survex_file / ".svx")
os.chdir(cwd)
def threed(request, survex_file):
process(survex_file)
try:
threed = open(settings.SURVEX_DATA + survex_file + ".3d", "rt",encoding='utf8')
threed = open(survexdatasetpath / survex_file / ".3d", "rt",encoding='utf8')
return HttpResponse(threed, content_type="model/3d")
except:
log = open(settings.SURVEX_DATA + survex_file + ".log", "rt",encoding='utf8')
log = open(survexdatasetpath / survex_file / ".log", "rt",encoding='utf8')
return HttpResponse(log, content_type="text")
def log(request, survex_file):
process(survex_file)
log = open(settings.SURVEX_DATA + survex_file + ".log", "rt",encoding='utf8')
log = open(survexdatasetpath / survex_file / ".log", "rt",encoding='utf8')
return HttpResponse(log, content_type="text")
def err(request, survex_file):
process(survex_file)
err = open(settings.SURVEX_DATA + survex_file + ".err", "rt",encoding='utf8')
err = open(survexdatasetpath / survex_file / ".err", "rt",encoding='utf8')
return HttpResponse(err, content_type="text")
@@ -292,12 +298,36 @@ def identifycavedircontents(gcavedir):
subsvx.insert(0, primesvx)
return subdirs, subsvx
def check_cave_registered(survex_cave):
'''Checks whether a cave has been properly registered when it is found in the Loser repo
This should be called by Databasereset not here in a view
Currently Caves are only registered if they are listed in :expoweb: settings.CAVEDESCRIPTIONS
so we need to add in any mroe here.
'''
try:
cave = Cave.objects.get(kataster_number=survex_cave)
return survex_cave
except ObjectDoesNotExist:
pass
try:
cave = Cave.objects.get(unofficial_number=survex_cave)
if cave.kataster_number:
return cave.kataster_number
else:
return None
except ObjectDoesNotExist:
pass
# direct local non-database browsing through the svx file repositories
# perhaps should use the database and have a reload button for it
# why is caves-1623 HARD CODED here ?! That must be wrong..
def survexcaveslist(request):
cavesdir = os.path.join(settings.SURVEX_DATA, "caves-1623")
'''This reads the entire list of caves in the Loser repo directory and produces a complete report.
It can find caves which have not yet been properly registered in the system by Databasereset.py because
someone may have uploaded the survex files without doing the rest of the integration process.
'''
cavesdir = survexdatasetpath / "caves-1623"
#cavesdircontents = { }
onefilecaves = [ ]
@@ -307,21 +337,25 @@ def survexcaveslist(request):
# first sort the file list
fnumlist = sorted([ (-int(re.match(r"\d*", f).group(0) or "0"), f) for f in os.listdir(cavesdir) ])
print(fnumlist)
#print(fnumlist)
# go through the list and identify the contents of each cave directory
for num, cavedir in fnumlist:
# these have sub dirs /cucc/ /arge/ /old/ but that is no reason to hide them in this webpage
# so these are now treated the same as 142 and 113 which also had a /cucc/ sub dir
# so these are now treated the same as 142 and 113 which also have a /cucc/ sub dir
#if cavedir in ["144", "40"]:
# continue
# This all assumes that the first .svx file has the same name as the cave name,
# which usually but not always true. e.g. caves-1623/78/allkaese.svx not caves-1623/78/78.svx
# which is why we now also pass through the cavedir
gcavedir = os.path.join(cavesdir, cavedir)
if os.path.isdir(gcavedir) and cavedir[0] != ".":
subdirs, subsvx = identifycavedircontents(gcavedir)
survdirobj = [ ]
katast = check_cave_registered(cavedir) # should do this only once per database load or it will be slow
survdirobj = [ ]
for lsubsvx in subsvx:
survdirobj.append(("caves-1623/"+cavedir+"/"+lsubsvx, lsubsvx))
@@ -340,7 +374,7 @@ def survexcaveslist(request):
# multifile caves
elif len(survdirobj) > 1:
multifilecaves.append((survdirobj[0], survdirobj[1:]))
multifilecaves.append((survdirobj[0], cavedir, survdirobj[1:]))
# single file caves
elif len(survdirobj) == 1:
onefilecaves.append(survdirobj[0])
@@ -349,16 +383,18 @@ def survexcaveslist(request):
# parsing all the survex files of a single cave and showing that it's consistent and can find all the files and people
# doesn't use recursion. just writes it twice
# currently not showing Explorers or Titles. link test from SurvexFile page is "dates and explorers"
# Should explicity fix the kataster number thing.
def survexcavesingle(request, survex_cave):
print(">>>", survex_cave)
breload = False
if breload:
parsers.survex.ReloadSurvexCave(survex_cave) # does not exit now, needs re-writing to work.
try:
cave = Cave.objects.get(kataster_number=survex_cave)
for survexdirectory in cave.survexdirectory_set.all:
print(">>> >>>", survexdirectory, flush=True)
return render_to_response('svxcavesingle.html', {'settings': settings, "cave":cave })
except ObjectDoesNotExist:
# can get here if the survex file is in a directory labelled with unofficial number not kataster number.

10
flatpages/views.py
View File

@@ -1,5 +1,6 @@
import os
import re
from pathlib import Path
from django.shortcuts import render, redirect
from django.http import HttpResponse, HttpResponseRedirect, Http404
@@ -52,15 +53,16 @@ def flatpage(request, path):
print(("flat path noinfo", path))
return HttpResponseRedirect(reverse("auth_login") + '?next=%s' % request.path)
expowebpath = Path(settings.EXPOWEB)
if path.endswith("/") or path == "":
#print(" - FLATPAGES the file: {} ENDSWITH ...".format(path))
try:
o = open(os.path.normpath(settings.EXPOWEB + path + "index.html"), "rb")
o = open(os.path.normpath(expowebpath / path / "index.html"), "rb")
path = path + "index.html"
except IOError:
try:
o = open(os.path.normpath(settings.EXPOWEB + path + "index.htm"), "rb")
o = open(os.path.normpath(expowebpath / path / "index.htm"), "rb")
path = path + "index.htm"
except IOError:
return render(request, 'pagenotfound.html', {'path': path})
@@ -77,8 +79,8 @@ def flatpage(request, path):
path = path.replace("static", settings.MEDIA_ROOT)
filetobeopened = os.path.normpath(path)
else:
#print(" - NO _ROOT: {} ...".format(settings.EXPOWEB))
filetobeopened = os.path.normpath(settings.EXPOWEB + path)
#print(" - NO _ROOT: {} ...".format(expowebpath))
filetobeopened = os.path.normpath(expowebpath / path)
#print(" - FLATPAGES full path : {} ...".format(filetobeopened))
o = open(filetobeopened, "rb")

4
parsers/survex.py
View File

@@ -4,7 +4,7 @@ import re
import time
import copy
from unipath import Path
from pathlib import Path
from datetime import datetime, timedelta
from subprocess import call, run
@@ -1257,7 +1257,7 @@ def LoadPositions():
print(" - Regenerating {} {}.3d in {}".format(settings.SURVEXPORT, topdata, settings.SURVEX_DATA))
call([settings.SURVEXPORT, '--pos', '{}.3d'.format(topdata)], cwd = settings.SURVEX_DATA)
topdata = settings.SURVEX_DATA.child(settings.SURVEX_TOPNAME)
topdata = os.fspath(Path(settings.SURVEX_DATA) / settings.SURVEX_TOPNAME)
print((' - Generating a list of Pos from %s.svx and then loading...' % (topdata)))
found = 0

2
settings.py
View File

@@ -27,7 +27,7 @@ print("* importing troggle/settings.py")
# read https://docs.djangoproject.com/en/3.0/topics/settings/
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
#BASE_DIR = os.path.dirname(os.path.dirname(__file__))
# Django settings for troggle project.

4
templates/svxfilecavelist.html
View File

@@ -41,10 +41,10 @@
<h2 id="cmult">Caves of multiple files</h2>
<table>
<tr><th>Dates and explorers</th><th>Survex files</th></tr>
{% for primarycavefile, subcavefiles in multifilecaves %}
{% for primarycavefile, cavedir, subcavefiles in multifilecaves %}
<tr>
<td>
<a href="{% url "survexcavessingle" primarycavefile.1 %}">{{primarycavefile.1}}</a>
<a href="{% url "survexcavessingle" cavedir %}">{{cavedir}}</a> <!-- formerly primarycavefile.1 -->
</td>
<td>
<a href="{% url "svx" primarycavefile.0 %}">{{primarycavefile.1}}</a> -