diff --git a/core/admin.py b/core/admin.py
index c820b77..2fc635a 100644
--- a/core/admin.py
+++ b/core/admin.py
@@ -7,7 +7,7 @@ from django.core import serializers
from troggle.core.views_other import downloadLogbook
from troggle.core.models import Person, PersonExpedition, Expedition, DataIssue
from troggle.core.models_caves import Cave, Area, Entrance, CaveAndEntrance, NewSubCave, OtherCaveName, CaveDescription, LogbookEntry, PersonTrip, QM
-from troggle.core.models_survex import SurvexBlock, SurvexPersonRole, SurvexStation, SurvexScansFolder, SurvexScanSingle
+from troggle.core.models_survex import SurvexBlock, SurvexPersonRole, SurvexStation, ScansFolder, SurvexScanSingle
class TroggleModelAdmin(admin.ModelAdmin):
@@ -120,7 +120,7 @@ admin.site.register(PersonExpedition,PersonExpeditionAdmin)
admin.site.register(LogbookEntry, LogbookEntryAdmin)
admin.site.register(QM, QMAdmin)
admin.site.register(SurvexStation)
-admin.site.register(SurvexScansFolder)
+admin.site.register(ScansFolder)
admin.site.register(SurvexScanSingle)
admin.site.register(DataIssue)
diff --git a/core/migrations/0001_initial.py b/core/migrations/0001_initial.py
index cbcb4a7..5e14a6c 100644
--- a/core/migrations/0001_initial.py
+++ b/core/migrations/0001_initial.py
@@ -296,6 +296,17 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='ScansFolder',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('fpath', models.CharField(max_length=200)),
+ ('walletname', models.CharField(max_length=200)),
+ ],
+ options={
+ 'ordering': ('walletname',),
+ },
+ ),
migrations.CreateModel(
name='SurvexBlock',
fields=[
@@ -359,24 +370,13 @@ class Migration(migrations.Migration):
('survexblock', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.SurvexBlock')),
],
),
- migrations.CreateModel(
- name='SurvexScansFolder',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('fpath', models.CharField(max_length=200)),
- ('walletname', models.CharField(max_length=200)),
- ],
- options={
- 'ordering': ('walletname',),
- },
- ),
migrations.CreateModel(
name='SurvexScanSingle',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('ffile', models.CharField(max_length=200)),
('name', models.CharField(max_length=200)),
- ('survexscansfolder', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='core.SurvexScansFolder')),
+ ('survexscansfolder', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='core.ScansFolder')),
],
options={
'ordering': ('name',),
@@ -412,9 +412,9 @@ class Migration(migrations.Migration):
('bfontcolours', models.BooleanField(default=False)),
('filesize', models.IntegerField(default=0)),
('npaths', models.IntegerField(default=0)),
+ ('manyscansfolders', models.ManyToManyField(to='core.ScansFolder')),
('survexblocks', models.ManyToManyField(to='core.SurvexBlock')),
('survexscans', models.ManyToManyField(to='core.SurvexScanSingle')),
- ('survexscansfolders', models.ManyToManyField(to='core.SurvexScansFolder')),
('survextitles', models.ManyToManyField(to='core.SurvexTitle')),
('tunnelcontains', models.ManyToManyField(to='core.TunnelFile')),
],
@@ -435,7 +435,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='survexblock',
name='survexscansfolder',
- field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='core.SurvexScansFolder'),
+ field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='core.ScansFolder'),
),
migrations.AddField(
model_name='qm',
diff --git a/core/models_survex.py b/core/models_survex.py
index 2b1e48f..75cdec8 100644
--- a/core/models_survex.py
+++ b/core/models_survex.py
@@ -123,7 +123,7 @@ class SurvexBlock(models.Model):
# begin_char = models.IntegerField() # code for where in the survex data files this block sits
survexpath = models.CharField(max_length=200) # the path for the survex stations
- survexscansfolder = models.ForeignKey("SurvexScansFolder", null=True,on_delete=models.CASCADE)
+ survexscansfolder = models.ForeignKey("ScansFolder", null=True,on_delete=models.CASCADE)
#refscandir = models.CharField(max_length=100)
legsall = models.IntegerField(null=True) # summary data for this block
@@ -195,7 +195,7 @@ class SurvexPersonRole(models.Model):
def __str__(self):
return str(self.person) + " - " + str(self.survexblock) + " - " + str(self.nrole)
-class SurvexScansFolder(models.Model):
+class ScansFolder(models.Model):
fpath = models.CharField(max_length=200)
walletname = models.CharField(max_length=200)
@@ -211,7 +211,7 @@ class SurvexScansFolder(models.Model):
class SurvexScanSingle(models.Model):
ffile = models.CharField(max_length=200)
name = models.CharField(max_length=200)
- survexscansfolder = models.ForeignKey("SurvexScansFolder", null=True,on_delete=models.CASCADE)
+ survexscansfolder = models.ForeignKey("ScansFolder", null=True,on_delete=models.CASCADE)
class Meta:
ordering = ('name',)
@@ -226,7 +226,7 @@ class TunnelFile(models.Model):
tunnelpath = models.CharField(max_length=200)
tunnelname = models.CharField(max_length=200)
bfontcolours = models.BooleanField(default=False)
- survexscansfolders = models.ManyToManyField("SurvexScansFolder")
+ manyscansfolders = models.ManyToManyField("ScansFolder")
survexscans = models.ManyToManyField("SurvexScanSingle")
survexblocks = models.ManyToManyField("SurvexBlock")
tunnelcontains = models.ManyToManyField("TunnelFile") # case when its a frame type
diff --git a/core/view_surveys.py b/core/view_surveys.py
index 76f7104..fbe5c6f 100644
--- a/core/view_surveys.py
+++ b/core/view_surveys.py
@@ -6,7 +6,7 @@ from django.conf import settings
from django.shortcuts import render_to_response
from django.http import HttpResponse, Http404
-from troggle.core.models_survex import SurvexScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile
+from troggle.core.models_survex import ScansFolder, SurvexScanSingle, SurvexBlock, TunnelFile
from troggle.flatpages import views as flatviews
import parsers.surveys
#from . import fileAbstraction
@@ -69,12 +69,12 @@ def SaveImageInDir(name, imgdir, project, fdata, bbinary):
return res.replace("\\", "/")
def surveyscansfolder(request, path):
- #print [ s.walletname for s in SurvexScansFolder.objects.all() ]
- survexscansfolder = SurvexScansFolder.objects.get(walletname=urllib.parse.unquote(path))
+ #print [ s.walletname for s in ScansFolder.objects.all() ]
+ survexscansfolder = ScansFolder.objects.get(walletname=urllib.parse.unquote(path))
return render_to_response('survexscansfolder.html', { 'survexscansfolder':survexscansfolder, 'settings': settings })
def surveyscansingle(request, path, file):
- survexscansfolder = SurvexScansFolder.objects.get(walletname=urllib.parse.unquote(path))
+ survexscansfolder = ScansFolder.objects.get(walletname=urllib.parse.unquote(path))
survexscansingle = SurvexScanSingle.objects.get(survexscansfolder=survexscansfolder, name=file)
print("SSS {} {} :{}:".format(path, file, flatviews.getmimetype(file)))
return HttpResponse(content=open(survexscansingle.ffile,"rb"), content_type=flatviews.getmimetype(file))
@@ -90,8 +90,8 @@ def cssfilessingle(request, filepath):
return HttpResponse(content=open(settings.MEDIA_ROOT+fn,"r"),content_type="text/css")
def surveyscansfolders(request):
- survexscansfolders = SurvexScansFolder.objects.all()
- return render_to_response('survexscansfolders.html', { 'survexscansfolders':survexscansfolders, 'settings': settings })
+ manyscansfolders = ScansFolder.objects.all()
+ return render_to_response('manyscansfolders.html', { 'manyscansfolders':manyscansfolders, 'settings': settings })
def tunneldata(request):
diff --git a/lines-of-python.txt b/lines-of-python.txt
index 74c6a89..d8104bb 100644
--- a/lines-of-python.txt
+++ b/lines-of-python.txt
@@ -24,32 +24,32 @@
49 ./parsers/subcaves.py
52 ./reset-django.py
63 ./logbooksdump.py
+68 ./urls.py
69 ./core/TESTS/tests.py
-69 ./urls.py
73 ./localsettings.py
73 ./localsettingsWSL.py
-85 ./settings.py
-92 ./core/views_statistics.py
+73 ./settings.py
+91 ./core/views_statistics.py
97 ./core/forms.py
97 ./core/view_surveys.py
98 ./core/admin.py
-99 ./parsers/QMs.py
+98 ./parsers/QMs.py
102 ./parsers/people.py
124 ./core/templatetags/wiki_markup.py
135 ./utils.py
-150 ./parsers/surveys.py
+147 ./parsers/surveys.py
160 ./core/models_survex.py
164 ./flatpages/views.py
164 ./modelviz.py
167 ./core/models.py
175 ./core/views_other.py
198 ./parsers/caves.py
-217 ./core/views_logbooks.py
+216 ./core/views_logbooks.py
256 ./core/views_survex.py
276 ./profiles/views.py
280 ./databaseReset.py
387 ./core/views_caves.py
-431 ./parsers/survex.py
+444 ./parsers/survex.py
450 ./core/models_caves.py
515 ./parsers/logbooks.py
-5960
+5954
diff --git a/parsers/QMs.py b/parsers/QMs.py
index 025438e..d96b665 100644
--- a/parsers/QMs.py
+++ b/parsers/QMs.py
@@ -1,7 +1,7 @@
import csv
import os
import re
-#from datetime import *
+from datetime import date
from django.conf import settings
diff --git a/parsers/survex.py b/parsers/survex.py
index f236ec8..a290a83 100644
--- a/parsers/survex.py
+++ b/parsers/survex.py
@@ -2,6 +2,8 @@ import sys
import os
import re
import time
+import resource
+
from datetime import datetime, timedelta
from subprocess import call, Popen, PIPE
@@ -29,14 +31,12 @@ ROOTBLOCK = "rootblock"
def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
+ """This reads compass, clino and tape data but only keeps the tape lengths,
+ the rest is discarded after error-checking.
+ """
global survexlegsalllength
global survexlegsnumber
- # The try catches here need replacing as they are relatively expensive
ls = sline.lower().split()
- #ssfrom = survexblock.MakeSurvexStation(ls[stardata["from"]])
- #ssto = survexblock.MakeSurvexStation(ls[stardata["to"]])
-
-# survexleg = models_survex.SurvexLeg(block=survexblock, stationfrom=ssfrom, stationto=ssto)
survexleg = models_survex.SurvexLeg()
# this next fails for two surface survey svx files which use / for decimal point
# e.g. '29/09' in the tape measurement, or use decimals but in brackets, e.g. (06.05)
@@ -97,11 +97,9 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
if cave:
survexleg.cave = cave
- # only save proper legs
# No need to save as we are measuring lengths only on parsing now.
# delete the object so that django autosaving doesn't save it.
survexleg = None
- #survexleg.save()
itape = stardata.get("tape")
if itape:
@@ -111,7 +109,6 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment, cave):
except ValueError:
print("! Length not added")
# No need to save as we are measuring lengths only on parsing now.
- #survexblock.save()
def LoadSurvexEquate(survexblock, sline):
@@ -146,12 +143,12 @@ rx_starref = re.compile(r'(?i)^\s*\*ref[\s.:]*((?:19[6789]\d)|(?:20[0123]\d))\s*
# $(?i)""", re.X) # the end (do the whole thing case insensitively)
rx_team = re.compile(r"(?i)(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$")
-rx_team_member = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$")
+rx_person = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$")
rx_qm = re.compile(r'^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')
insp = ""
callcount = 0
-def RecursiveLoad(survexblock, survexfile, fin, textlines):
+def RecursiveLoad(survexblock, survexfile, fin):
"""Follows the *include links in all the survex files from the root file 1623.svx
and reads in the survex blocks, other data and the wallet references (survexscansfolder) as it
goes. This part of the data import process is where the maximum memory is used and where it
@@ -165,7 +162,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
global callcount
global survexlegsnumber
- print(insp+" - Reading file: " + survexblock.survexfile.path + " <> " + survexfile.path)
+ print(insp+" - MEM:{} Reading. parent:{} <> {} ".format(get_process_memory(),survexblock.survexfile.path,survexfile.path))
stamp = datetime.now()
lineno = 0
@@ -199,12 +196,12 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
if len(wallet)<2:
wallet = "0" + wallet
refscan = "%s#%s%s" % (yr, letterx, wallet )
- survexscansfolders = models_survex.SurvexScansFolder.objects.filter(walletname=refscan)
- if survexscansfolders:
- survexblock.survexscansfolder = survexscansfolders[0]
+ manyscansfolders = models_survex.ScansFolder.objects.filter(walletname=refscan)
+ if manyscansfolders:
+ survexblock.survexscansfolder = manyscansfolders[0]
survexblock.save()
else:
- message = ' ! Wallet ; ref {} - NOT found in survexscansfolders {}'.format(refscan, survexblock.survexfile.path)
+ message = ' ! Wallet ; ref {} - NOT found in manyscansfolders {}'.format(refscan, survexblock.survexfile.path)
print((insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
@@ -265,12 +262,12 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
assert (int(yr)>1960 and int(yr)<2039), "Wallet year out of bounds: %s" % yr
assert (int(wallet)<100), "Wallet number more than 100: %s" % wallet
refscan = "%s#%s%s" % (yr, letterx, wallet)
- survexscansfolders = models_survex.SurvexScansFolder.objects.filter(walletname=refscan)
- if survexscansfolders:
- survexblock.survexscansfolder = survexscansfolders[0]
+ manyscansfolders = models_survex.ScansFolder.objects.filter(walletname=refscan)
+ if manyscansfolders:
+ survexblock.survexscansfolder = manyscansfolders[0]
survexblock.save()
else:
- message = ' ! Wallet *REF {} - NOT found in survexscansfolders {}'.format(refscan, survexblock.survexfile.path)
+ message = ' ! Wallet *REF {} - NOT found in manyscansfolders {}'.format(refscan, survexblock.survexfile.path)
print((insp+message))
models.DataIssue.objects.create(parser='survex', message=message)
continue
@@ -291,17 +288,18 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
cmd = cmd.lower()
if re.match("include$(?i)", cmd):
includepath = os.path.normpath(os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line)))
- print((insp+' - Include path found including - ' + includepath))
- # Try to find the cave in the DB if not use the string as before
+ print((insp+' - Include path found, including - ' + includepath))
+ # Try to find the cave in the DB. if not, use the string as before
path_match = re.search(r"caves-(\d\d\d\d)/(\d+|\d\d\d\d-?\w+-\d+)/", includepath)
if path_match:
pos_cave = '%s-%s' % (path_match.group(1), path_match.group(2))
- # print(insp+pos_cave)
+ print((insp+' - Match in DB (i) for cave {}.'.format(pos_cave)))
cave = models_caves.getCaveByReference(pos_cave)
if cave:
survexfile.cave = cave
else:
- print((insp+' - No match in DB (i) for %s, so loading..' % includepath))
+ print((insp+' - NO Match in DB (i) for a cave for {}'.format(includepath)))
+
includesurvexfile = models_survex.SurvexFile(path=includepath)
includesurvexfile.save()
includesurvexfile.SetDirectory()
@@ -309,10 +307,17 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
survexblock.save()
fininclude = includesurvexfile.OpenFile()
insp += "> "
- RecursiveLoad(survexblock, includesurvexfile, fininclude, textlines)
+ RecursiveLoad(survexblock, includesurvexfile, fininclude)
+ #--------------------------------------------------------
+ fininclude.close()
insp = insp[2:]
+ else:
+ print((insp+' ! ERROR *include file not found for %s' % includesurvexfile))
elif re.match("begin$(?i)", cmd):
+ # On a *begin statement we start a new survexblock.
+ # There should not be any *include inside a begin-end block, so this is a simple
+ # load not a recursive load. But there may be many blocks in one file.
if line:
newsvxpath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line))
# Try to find the cave in the DB if not use the string as before
@@ -328,16 +333,16 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
previousnlegs = survexlegsnumber
name = line.lower()
- print((insp+' - Begin found for: ' + name))
-# survexblockdown = models_survex.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexfile.cave, survexfile=survexfile, totalleglength=0.0)
+ print(insp+' - Begin found for:{}, creating new SurvexBlock '.format(name))
survexblockdown = models_survex.SurvexBlock(name=name, parent=survexblock, survexpath=survexblock.survexpath+"."+name,
cave=survexfile.cave, survexfile=survexfile, legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
survexblockdown.save()
survexblock.save()
survexblock = survexblockdown
- textlinesdown = [ ]
insp += "> "
- RecursiveLoad(survexblockdown, survexfile, fin, textlinesdown)
+ RecursiveLoad(survexblockdown, survexfile, fin)
+ #--------------------------------------------------------
+ # do not close the file as there may be more blocks in this one
insp = insp[2:]
else:
iblankbegins += 1
@@ -346,7 +351,6 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
if iblankbegins:
iblankbegins -= 1
else:
- # .text not used, using it for number of legs per block
legsinblock = survexlegsnumber - previousnlegs
print(insp+"LEGS: {} (previous: {}, now:{})".format(legsinblock,previousnlegs,survexlegsnumber))
survexblock.legsall = legsinblock
@@ -370,7 +374,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
# print(insp+' - Team found: ')
mteammember = rx_team.match(line)
if mteammember:
- for tm in rx_team_member.split(mteammember.group(2)):
+ for tm in rx_person.split(mteammember.group(2)):
if tm:
personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower())
if (personexpedition, tm) not in teammembers:
@@ -431,6 +435,31 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines):
timetaken = endstamp - stamp
# print(insp+' - Time to process: ' + str(timetaken))
+def get_process_memory():
+ usage=resource.getrusage(resource.RUSAGE_SELF)
+ return usage[2]/1024.0
+
+
+def FindAndLoadAllSurvex(survexblockroot, survexfileroot):
+ """Follows the *include links recursively to find files
+ """
+ print(' - redirecting stdout to loadsurvexblks.log...')
+ stdout_orig = sys.stdout
+ # Redirect sys.stdout to the file
+ sys.stdout = open('loadsurvexblks.log', 'w')
+
+ finroot = survexfileroot.OpenFile()
+ RecursiveLoad(survexblockroot, survexfileroot, finroot)
+ finroot.close()
+
+ # Close the logging file
+ sys.stdout.close()
+ print("+", file=sys.stderr)
+ sys.stderr.flush();
+ # Restore sys.stdout to our old saved file handler
+ sys.stdout = stdout_orig
+
+
def LoadAllSurvexBlocks():
global survexlegsalllength
global survexlegsnumber
@@ -441,47 +470,28 @@ def LoadAllSurvexBlocks():
models_survex.SurvexFile.objects.all().delete()
models_survex.SurvexDirectory.objects.all().delete()
models_survex.SurvexEquate.objects.all().delete()
- #models_survex.SurvexLeg.objects.all().delete()
models_survex.SurvexTitle.objects.all().delete()
models_survex.SurvexPersonRole.objects.all().delete()
models_survex.SurvexStation.objects.all().delete()
print(" - Data flushed")
- # Clear the data issues as we are reloading
models.DataIssue.objects.filter(parser='survex').delete()
- print(' - Loading All Survex Blocks...')
-
- print(' - redirecting stdout to loadsurvexblks.log...')
- stdout_orig = sys.stdout
- # Redirect sys.stdout to the file
- sys.stdout = open('loadsurvexblks.log', 'w')
-
- survexfile = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
- survexfile.save()
- survexfile.SetDirectory()
-
- #Load all
- # this is the first so id=1
- survexblockroot = models_survex.SurvexBlock(name=ROOTBLOCK, survexpath="", cave=None, survexfile=survexfile,
+
+ survexfileroot = models_survex.SurvexFile(path=settings.SURVEX_TOPNAME, cave=None)
+ survexfileroot.save()
+ survexfileroot.SetDirectory()
+ survexblockroot = models_survex.SurvexBlock(name=ROOTBLOCK, survexpath="", cave=None, survexfile=survexfileroot,
legsall=0, legssplay=0, legssurfc=0, totalleglength=0.0)
+ # this is the first so id=1
survexblockroot.save()
- fin = survexfile.OpenFile()
- textlines = [ ]
- # The real work starts here
- RecursiveLoad(survexblockroot, survexfile, fin, textlines)
- fin.close()
+
+ print(' - Loading All Survex Blocks...')
+ FindAndLoadAllSurvex(survexblockroot, survexfileroot)
+
survexblockroot.totalleglength = survexlegsalllength
survexblockroot.legsall = survexlegsnumber
- #survexblockroot.text = "".join(textlines) these are all blank
survexblockroot.save()
- # Close the file
- sys.stdout.close()
- print("+", file=sys.stderr)
- sys.stderr.flush();
-
- # Restore sys.stdout to our old saved file handler
- sys.stdout = stdout_orig
print(" - total number of survex legs: {}m".format(survexlegsnumber))
print(" - total leg lengths loaded: {}m".format(survexlegsalllength))
print(' - Loaded All Survex Blocks.')
@@ -500,48 +510,6 @@ def LoadPos():
topdata = settings.SURVEX_DATA + settings.SURVEX_TOPNAME
print((' - Generating a list of Pos from %s.svx and then loading...' % (topdata)))
- # TO DO - remove the cache file apparatus. Not needed. Only laser points and entrances loaded now.
-
- # Be careful with the cache file.
- # If LoadPos has been run before,
- # but without cave import being run before,
- # then *everything* may be in the fresh 'not found' cache file.
-
- # cachefile = settings.SURVEX_DATA + "posnotfound.cache"
- # notfoundbefore = {}
- # if os.path.isfile(cachefile):
- # # this is not a good test. 1623.svx may never change but *included files may have done.
- # # When the *include is unrolled, we will be able to get a proper timestamp to use
- # # and can increase the timeout from 3 days to 30 days.
- # updtsvx = os.path.getmtime(topdata + ".svx")
- # updtcache = os.path.getmtime(cachefile)
- # age = updtcache - updtsvx
- # print((' svx: %s cache: %s not-found cache is fresher by: %s' % (updtsvx, updtcache, str(timedelta(seconds=age) ))))
-
- # now = time.time()
- # if now - updtcache > 3*24*60*60:
- # print(" cache is more than 3 days old. Deleting.")
- # os.remove(cachefile)
- # elif age < 0 :
- # print(" cache is stale. Deleting.")
- # os.remove(cachefile)
- # else:
- # print(" cache is fresh. Reading...")
- # try:
- # with open(cachefile, "r") as f:
- # for line in f:
- # l = line.rstrip()
- # if l in notfoundbefore:
- # notfoundbefore[l] +=1 # should not be duplicates
- # print(" DUPLICATE ", line, notfoundbefore[l])
- # else:
- # notfoundbefore[l] =1
- # except:
- # print(" FAILURE READ opening cache file %s" % (cachefile))
- # raise
-
-
-# notfoundnow =[]
found = 0
skip = {}
print("\n") # extra line because cavern overwrites the text buffer somehow
@@ -615,18 +583,5 @@ def LoadPos():
models.DataIssue.objects.create(parser='survex', message=message)
raise
- #print(" - %s failed lookups of SurvexStation.objects. %s found. %s skipped." % (len(notfoundnow),found, len(skip)))
print(" - {} SurvexStation entrances found.".format(found))
- # if found > 10: # i.e. a previous cave import has been done
- # try:
- # with open(cachefile, "w") as f:
- # c = len(notfoundnow)+len(skip)
- # for i in notfoundnow:
- # pass #f.write("%s\n" % i)
- # for j in skip:
- # pass #f.write("%s\n" % j) # NB skip not notfoundbefore
- # print((' Not-found cache file written: %s entries' % c))
- # except:
- # print(" FAILURE WRITE opening cache file %s" % (cachefile))
- # raise
\ No newline at end of file
diff --git a/parsers/surveys.py b/parsers/surveys.py
index baa8725..f3edebe 100644
--- a/parsers/surveys.py
+++ b/parsers/surveys.py
@@ -1,6 +1,3 @@
-from __future__ import (absolute_import, division,
- print_function, unicode_literals)
-
import sys
import os
import types
@@ -17,7 +14,7 @@ from functools import reduce
import settings
#from troggle.core.models import *
#from troggle.core.models_caves import *
-#from troggle.core.models_survex import *
+from troggle.core.models_survex import SurvexScanSingle, ScansFolder, TunnelFile
def get_or_create_placeholder(year):
""" All surveys must be related to a logbookentry. We don't have a way to
@@ -83,15 +80,15 @@ def LoadListScans():
print(' - Loading Survey Scans')
SurvexScanSingle.objects.all().delete()
- SurvexScansFolder.objects.all().delete()
+ ScansFolder.objects.all().delete()
print(' - deleting all scansFolder and scansSingle objects')
# first do the smkhs (large kh survey scans) directory
- survexscansfoldersmkhs = SurvexScansFolder(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs")
+ manyscansfoldersmkhs = ScansFolder(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs")
print("smkhs", end=' ')
- if os.path.isdir(survexscansfoldersmkhs.fpath):
- survexscansfoldersmkhs.save()
- LoadListScansFile(survexscansfoldersmkhs)
+ if os.path.isdir(manyscansfoldersmkhs.fpath):
+ manyscansfoldersmkhs.save()
+ LoadListScansFile(manyscansfoldersmkhs)
# iterate into the surveyscans directory
@@ -106,13 +103,13 @@ def LoadListScans():
for fy, ffy, fisdiry in GetListDir(ff):
if fisdiry:
assert fisdiry, ffy
- survexscansfolder = SurvexScansFolder(fpath=ffy, walletname=fy)
+ survexscansfolder = ScansFolder(fpath=ffy, walletname=fy)
survexscansfolder.save()
LoadListScansFile(survexscansfolder)
# do the
elif f != "thumbs":
- survexscansfolder = SurvexScansFolder(fpath=ff, walletname=f)
+ survexscansfolder = ScansFolder(fpath=ff, walletname=f)
survexscansfolder.save()
LoadListScansFile(survexscansfolder)
@@ -121,7 +118,7 @@ def FindTunnelScan(tunnelfile, path):
scansfolder, scansfile = None, None
mscansdir = re.search(rb"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg))$", path)
if mscansdir:
- scansfolderl = SurvexScansFolder.objects.filter(walletname=mscansdir.group(1))
+ scansfolderl = ScansFolder.objects.filter(walletname=mscansdir.group(1))
if len(scansfolderl):
assert len(scansfolderl) == 1
scansfolder = scansfolderl[0]
@@ -136,7 +133,7 @@ def FindTunnelScan(tunnelfile, path):
scansfile = scansfilel[0]
if scansfolder:
- tunnelfile.survexscansfolders.add(scansfolder)
+ tunnelfile.manyscansfolders.add(scansfolder)
if scansfile:
tunnelfile.survexscans.add(scansfile)
diff --git a/templates/survexscansfolders.html b/templates/survexscansfolders.html
index 05c65ba..6049750 100644
--- a/templates/survexscansfolders.html
+++ b/templates/survexscansfolders.html
@@ -13,7 +13,7 @@ hand-drawn passage sections are drawn. These hand-drawn passages will eventually
traced to produce Tunnel or Therion drawings and eventually the final complete cave survey.
<table>
<tr><th>Scans folder</th><th>Files</th><th>Survex blocks</th></tr>
-{% for survexscansfolder in survexscansfolders %}
+{% for survexscansfolder in manyscansfolders %}
<tr>
<td><a href="{{survexscansfolder.get_absolute_url}}">{{survexscansfolder.walletname}}</a></td>
<td>{{survexscansfolder.survexscansingle_set.all|length}}</td>
diff --git a/templates/tunnelfiles.html b/templates/tunnelfiles.html
index 843eefd..83db4b0 100644
--- a/templates/tunnelfiles.html
+++ b/templates/tunnelfiles.html
@@ -17,7 +17,7 @@
<td>{{tunnelfile.npaths}}</td>
<td>
- {% for survexscansfolder in tunnelfile.survexscansfolders.all %}
+ {% for survexscansfolder in tunnelfile.manyscansfolders.all %}
<a href="{{survexscansfolder.get_absolute_url}}">{{survexscansfolder.walletname}}</a>
{% endfor %}
</td>