forked from expo/troggle
102 lines
3.5 KiB
Python
102 lines
3.5 KiB
Python
import sys
|
|
import os
|
|
import types
|
|
import stat
|
|
import csv
|
|
import re
|
|
import datetime
|
|
|
|
from PIL import Image
|
|
from functools import reduce
|
|
|
|
import settings
|
|
from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
|
|
from troggle.core.models.troggle import DataIssue
|
|
from troggle.core.utils import save_carefully, GetListDir
|
|
|
|
'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
|
|
'''
|
|
|
|
|
|
def get_or_create_placeholder(year):
|
|
""" All surveys must be related to a logbookentry. We don't have a way to
|
|
automatically figure out which survey went with which logbookentry,
|
|
so we create a survey placeholder logbook entry for each year. This
|
|
function always returns such a placeholder, and creates it if it doesn't
|
|
exist yet.
|
|
"""
|
|
lookupAttribs={'date__year':int(year), 'title':"placeholder for surveys",}
|
|
nonLookupAttribs={'text':"surveys temporarily attached to this should be re-attached to their actual trips", 'date':datetime.date(int(year),1,1)}
|
|
placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs)
|
|
return placeholder_logbook_entry
|
|
|
|
def listdir(*directories):
|
|
try:
|
|
return os.listdir(os.path.join(settings.SURVEYS, *directories))
|
|
except:
|
|
import urllib.request, urllib.parse, urllib.error
|
|
url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories))
|
|
folders = urllib.request.urlopen(url.replace("#", "%23")).readlines()
|
|
return [folder.rstrip(r"/") for folder in folders]
|
|
|
|
def LoadListScansFile(wallet):
|
|
gld = [ ]
|
|
# flatten out any directories in these wallet folders - should not be any
|
|
for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath):
|
|
if fisdiryf:
|
|
gld.extend(GetListDir(ffyf))
|
|
else:
|
|
gld.append((fyf, ffyf, fisdiryf))
|
|
|
|
c=0
|
|
for (fyf, ffyf, fisdiryf) in gld:
|
|
if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif)(?i)$", fyf):
|
|
singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet)
|
|
singlescan.save()
|
|
c+=1
|
|
if c>=10:
|
|
print(".", end='')
|
|
c = 0
|
|
|
|
|
|
# this iterates through the scans directories (either here or on the remote server)
|
|
# and builds up the models we can access later
|
|
def load_all_scans():
|
|
|
|
print(' - Loading Survey Scans')
|
|
|
|
SingleScan.objects.all().delete()
|
|
Wallet.objects.all().delete()
|
|
print(' - deleting all scansFolder and scansSingle objects')
|
|
|
|
# first do the smkhs (large kh survey scans) directory
|
|
manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs")
|
|
print("smkhs", end=' ')
|
|
if os.path.isdir(manywallets_smkhs.fpath):
|
|
manywallets_smkhs.save()
|
|
LoadListScansFile(manywallets_smkhs)
|
|
|
|
|
|
# iterate into the surveyscans directory
|
|
print(' - ', end=' ')
|
|
for f, ff, fisdir in GetListDir(settings.SURVEY_SCANS):
|
|
if not fisdir:
|
|
continue
|
|
|
|
# do the year folders
|
|
if re.match(r"\d\d\d\d$", f):
|
|
print("%s" % f, end=' ')
|
|
for fy, ffy, fisdiry in GetListDir(ff):
|
|
if fisdiry:
|
|
wallet = Wallet(fpath=ffy, walletname=fy)
|
|
wallet.save()
|
|
LoadListScansFile(wallet)
|
|
|
|
# do the
|
|
elif f != "thumbs":
|
|
wallet = Wallet(fpath=ff, walletname=f)
|
|
wallet.save()
|
|
LoadListScansFile(wallet)
|
|
|
|
print("", flush=True)
|