forked from expo/troggle
140 lines
4.7 KiB
Python
140 lines
4.7 KiB
Python
import sys
|
|
import os
|
|
import types
|
|
import stat
|
|
import csv
|
|
import re
|
|
import datetime
|
|
|
|
from functools import reduce
|
|
|
|
import settings
|
|
from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
|
|
from troggle.core.models.troggle import DataIssue
|
|
from troggle.core.utils import save_carefully, GetListDir
|
|
|
|
'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
|
|
'''
|
|
|
|
contentsjson = "contents.json"
|
|
indexhtml = "walletindex.html"
|
|
|
|
wallet_blank_json = {
|
|
"cave": "",
|
|
"date": "",
|
|
"description url": "/caves",
|
|
"description written": False,
|
|
"electronic survey": False,
|
|
"elev drawn": False,
|
|
"elev not required": False,
|
|
"name": "",
|
|
"people": [
|
|
"Unknown"
|
|
],
|
|
"plan drawn": False,
|
|
"plan not required": False,
|
|
"qms written": False,
|
|
"survex file": [],
|
|
"survex not required": False,
|
|
"website updated": False}
|
|
|
|
wallet_blank_html = '''<html><body><H1>Wallet WALLET</H1>
|
|
<p>List of trips: <a href="http://expo.survex.com/expedition/YEAR">expedition/YEAR</a>
|
|
- troggle-processed .svx files and logbook entries on server</p>
|
|
<p>Date: </p><p>People: Unknown,</p>
|
|
<p>Cave <a href='http://expo.survex.com/caves/'>Guidebook description</a>
|
|
- A description is indicated as being needed, so may need adding into this cave page.
|
|
<p>Survex file: not identified yet
|
|
<H2>Issues</H2>
|
|
<p>The description needs writing</p>
|
|
<p>The QMs needs writing</p><p>The website is marked as needing updating (using the guidebook description)</p>
|
|
<p>Tunnel / Therion drawing files need drawing</p>
|
|
<H2>Files</H2>
|
|
<UL>
|
|
</UL>
|
|
</body></html>
|
|
'''
|
|
|
|
def get_or_create_placeholder(year):
|
|
""" All surveys must be related to a logbookentry. We don't have a way to
|
|
automatically figure out which survey went with which logbookentry,
|
|
so we create a survey placeholder logbook entry for each year. This
|
|
function always returns such a placeholder, and creates it if it doesn't
|
|
exist yet.
|
|
"""
|
|
lookupAttribs={'date__year':int(year), 'title':"placeholder for surveys",}
|
|
nonLookupAttribs={'text':"surveys temporarily attached to this should be re-attached to their actual trips", 'date':datetime.date(int(year),1,1)}
|
|
placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs)
|
|
return placeholder_logbook_entry
|
|
|
|
def listdir(*directories):
|
|
try:
|
|
return os.listdir(os.path.join(settings.SURVEYS, *directories))
|
|
except:
|
|
import urllib.request, urllib.parse, urllib.error
|
|
url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories))
|
|
folders = urllib.request.urlopen(url.replace("#", "%23")).readlines()
|
|
return [folder.rstrip(r"/") for folder in folders]
|
|
|
|
def LoadListScansFile(wallet):
|
|
gld = [ ]
|
|
# flatten out any directories in these wallet folders - should not be any
|
|
for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath):
|
|
if fisdiryf:
|
|
gld.extend(GetListDir(ffyf))
|
|
else:
|
|
gld.append((fyf, ffyf, fisdiryf))
|
|
|
|
c=0
|
|
for (fyf, ffyf, fisdiryf) in gld:
|
|
if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif)(?i)$", fyf):
|
|
singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet)
|
|
singlescan.save()
|
|
c+=1
|
|
if c>=10:
|
|
print(".", end='')
|
|
c = 0
|
|
|
|
|
|
# this iterates through the scans directories (either here or on the remote server)
|
|
# and builds up the models we can access later
|
|
def load_all_scans():
|
|
|
|
print(' - Loading Survey Scans')
|
|
|
|
SingleScan.objects.all().delete()
|
|
Wallet.objects.all().delete()
|
|
print(' - deleting all scansFolder and scansSingle objects')
|
|
DataIssue.objects.filter(parser='scans').delete()
|
|
|
|
# first do the smkhs (large kh survey scans) directory
|
|
manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs")
|
|
print("smkhs", end=' ')
|
|
if os.path.isdir(manywallets_smkhs.fpath):
|
|
manywallets_smkhs.save()
|
|
LoadListScansFile(manywallets_smkhs)
|
|
|
|
|
|
# iterate into the surveyscans directory
|
|
print(' - ', end=' ')
|
|
for f, ff, fisdir in GetListDir(settings.SURVEY_SCANS):
|
|
if not fisdir:
|
|
continue
|
|
|
|
# do the year folders
|
|
if re.match(r"\d\d\d\d$", f):
|
|
print("%s" % f, end=' ')
|
|
for fy, ffy, fisdiry in GetListDir(ff):
|
|
if fisdiry:
|
|
wallet = Wallet(fpath=ffy, walletname=fy)
|
|
wallet.save()
|
|
LoadListScansFile(wallet)
|
|
|
|
# do the
|
|
elif f != "thumbs":
|
|
wallet = Wallet(fpath=ff, walletname=f)
|
|
wallet.save()
|
|
LoadListScansFile(wallet)
|
|
|
|
print("", flush=True)
|