forked from expo/troggle
149 lines
6.1 KiB
Python
149 lines
6.1 KiB
Python
import csv
|
|
import datetime
|
|
import filecmp
|
|
import os
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import types
|
|
from functools import reduce
|
|
from pathlib import Path
|
|
|
|
import settings
|
|
from troggle.core.models.survex import DrawingFile, SingleScan, Wallet
|
|
from troggle.core.models.troggle import DataIssue
|
|
from troggle.core.utils import save_carefully
|
|
from troggle.core.views.scans import datewallet
|
|
|
|
'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
|
|
'''
|
|
|
|
contentsjson = "contents.json"
|
|
|
|
git = settings.GIT
|
|
|
|
# to do: Actually read all the JSON files and set the survex file field appropriately!
|
|
|
|
def setwalletyear(wallet):
|
|
_ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear
|
|
|
|
def load_all_scans():
|
|
'''This iterates through the scans directories (either here or on the remote server)
|
|
and builds up the models we can access later.
|
|
|
|
It does NOT read or validate anything in the JSON data attached to each wallet. Those checks
|
|
are done at runtime, when a wallet is accessed, not at import time.
|
|
|
|
'''
|
|
print(' - Loading Survey Scans')
|
|
|
|
SingleScan.objects.all().delete()
|
|
Wallet.objects.all().delete()
|
|
print(' - deleting all Wallet and SingleScan objects')
|
|
DataIssue.objects.filter(parser='scans').delete()
|
|
|
|
# These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet.
|
|
valids = [".top",".txt",".tif",".png",".jpg",".jpeg",".pdf",".svg",".gif",".xvi",
|
|
".json",".autosave",".sxd",".svx",".th",".th2",".tdr",".sql",".zip",".dxf",".3d",
|
|
".ods",".csv",".xcf",".xml"]
|
|
validnames = ["thconfig","manifest"]
|
|
|
|
# iterate into the surveyscans directory
|
|
# Not all folders with files in them are wallets.
|
|
# they are if they are /2010/2010#33
|
|
# or /1996-1999NotKHbook/
|
|
# but not if they are /2010/2010#33/therion/ : the wallet is /2010#33/ not /therion/
|
|
print(' ', end='')
|
|
scans_path = Path(settings.SCANS_ROOT)
|
|
seen = []
|
|
c=0
|
|
wallets = {}
|
|
for p in scans_path.rglob('*'):
|
|
if p.is_file():
|
|
if p.suffix.lower() not in valids and p.name.lower() not in validnames:
|
|
# print(f"'{p}'", end='\n')
|
|
pass
|
|
elif p.parent == scans_path: # skip files directly in /surveyscans/
|
|
pass
|
|
else:
|
|
|
|
c+=1
|
|
if c % 15 == 0 :
|
|
print(".", end='')
|
|
if c % 750 == 0 :
|
|
print("\n ", end='')
|
|
|
|
if p.parent.parent.parent.parent == scans_path:
|
|
# print(f"too deep {p}", end='\n')
|
|
fpath = p.parent.parent
|
|
walletname = p.parent.parent.name # wallet is one level higher
|
|
else:
|
|
fpath = p.parent
|
|
walletname = p.parent.name
|
|
|
|
if walletname in wallets:
|
|
wallet = wallets[walletname]
|
|
else:
|
|
print("", flush=True, end='')
|
|
# Create the wallet object. But we don't have a date for it yet.
|
|
wallet = Wallet(fpath=fpath, walletname=walletname)
|
|
setwalletyear(wallet)
|
|
wallet.save()
|
|
wallets[walletname] = wallet
|
|
|
|
singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet)
|
|
singlescan.save()
|
|
|
|
|
|
# only printing progress:
|
|
tag = p.parent
|
|
if len(walletname)>4:
|
|
if walletname[4] == "#":
|
|
tag = p.parent.parent
|
|
|
|
if tag not in seen:
|
|
print(f" {tag.name} ", end='')
|
|
if len(str(tag.name)) > 17:
|
|
print('\n ', end='')
|
|
seen.append(tag)
|
|
|
|
|
|
print(f'\n - found and loaded {c:,} acceptable scan files in {len(wallets):,} wallets')
|
|
|
|
# but we also need to check if JSON exists, even if there are no uploaded scan files.
|
|
# Here we know there is a rigid folder structure, so no need to look for sub folders
|
|
print(f"\n - Checking for wallets where JSON exists, but there may be no uploaded scan files:")
|
|
print(' ', end='')
|
|
wjson = 0
|
|
contents_path = Path(settings.DRAWINGS_DATA, "walletjson")
|
|
for yeardir in contents_path.iterdir():
|
|
if yeardir.is_dir():
|
|
for walletpath in yeardir.iterdir():
|
|
if Path(walletpath, contentsjson).is_file():
|
|
walletname = walletpath.name
|
|
|
|
if walletname not in wallets:
|
|
wjson += 1
|
|
if wjson % 10 == 0 :
|
|
print("\n ", end='')
|
|
|
|
print(f"{walletname} ", end='')
|
|
fpath = Path(settings.SCANS_ROOT, str(yeardir.stem), walletname)
|
|
# The wallets found from JSON should all have dates already
|
|
wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
|
|
wallets[walletname] = wallet
|
|
# could now also load the json but we don't. Do later, on-demand
|
|
# wallet.walletdate = wallet.date()
|
|
# could check if link to svx file is valid too.. but do on-demand later
|
|
# But we *do* set the walletyear:
|
|
setwalletyear(wallet)
|
|
if not created:
|
|
print(f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?")
|
|
wallet.save()
|
|
print(f'\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets')
|
|
wallets = Wallet.objects.filter(walletyear=None)
|
|
for w in wallets:
|
|
w.walletyear = datetime.date(1999, 1, 1)
|