troggle-unchained/parsers/scans.py

import sys
import os
import subprocess
import types
import stat
import csv
import re
import datetime
import shutil, filecmp

from functools import reduce
from pathlib import Path

import settings
from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
from troggle.core.models.troggle import DataIssue
from troggle.core.utils import save_carefully
from troggle.core.views.scans import datewallet

'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
'''

contentsjson = "contents.json"

git = settings.GIT

# to do: Actually read all the JSON files and set the survex file field appropriately!

def setwalletyear(wallet):
    _ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear

def load_all_scans():
    '''This iterates through the scans directories (either here or on the remote server)
    and builds up the models we can access later.
    
    It does NOT read or validate anything in the JSON data attached to each wallet. Those checks
    are done at runtime, when a wallet is accessed, not at import time.
    
    '''
    print(' - Loading Survey Scans')

    SingleScan.objects.all().delete()
    Wallet.objects.all().delete()
    print('  - deleting all Wallet and SingleScan objects')
    DataIssue.objects.filter(parser='scans').delete()
    
    # These are valid old file types to be visible, they are not necessarily allowed to be uploaded to a new wallet.
    valids = [".top",".txt",".tif",".png",".jpg",".jpeg",".pdf",".svg",".gif",".xvi",
        ".json",".autosave",".sxd",".svx",".th",".th2",".tdr",".sql",".zip",".dxf",".3d",
        ".ods",".csv",".xcf",".xml"]
    validnames = ["thconfig","manifest"]

    # iterate into the surveyscans directory
    # Not all folders with files in them are wallets.
    # they are if they   are /2010/2010#33 
    #     or /1996-1999NotKHbook/
    #     but not if they are /2010/2010#33/therion/  : the wallet is /2010#33/ not /therion/
    print('   ', end='')
    scans_path = Path(settings.SCANS_ROOT) 
    seen = []
    c=0
    wallets = {}
    for p in scans_path.rglob('*'):
        if p.is_file():
            if p.suffix.lower() not in valids and p.name.lower() not in validnames:
                # print(f"'{p}'", end='\n')
                pass
            elif p.parent == scans_path: # skip files directly in /surveyscans/
                pass
            else:
                
                c+=1
                if c % 15 == 0 :
                    print(".", end='')
                if c % 750 == 0 :
                    print("\n   ", end='')

                if p.parent.parent.parent.parent == scans_path:
                    # print(f"too deep {p}", end='\n')
                    fpath = p.parent.parent
                    walletname = p.parent.parent.name # wallet is one level higher
                else: 
                    fpath = p.parent
                    walletname = p.parent.name
                
                if walletname in wallets:
                    wallet = wallets[walletname]
                else:
                    print("", flush=True, end='')
                    # Create the wallet object. But we don't have a date for it yet.
                    wallet = Wallet(fpath=fpath, walletname=walletname)
                    setwalletyear(wallet)
                    wallet.save()
                    wallets[walletname] = wallet
                
                singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet)
                singlescan.save()
                
                
                # only printing progress:
                tag = p.parent
                if len(walletname)>4:
                    if walletname[4] == "#":
                        tag = p.parent.parent
                    
                if tag not in seen:
                    print(f" {tag.name} ", end='')
                    if len(str(tag.name)) > 17:
                        print('\n   ', end='')
                    seen.append(tag)
                     
                    
    print(f'\n  - found and loaded {c:,} acceptable scan files in {len(wallets):,} wallets')
     
    # but we also need to check if JSON exists, even if there are no uploaded scan files.
    # Here we know there is a rigid folder structure, so no need to look for sub folders
    print(f"\n  - Checking for wallets where JSON exists, but there may be no uploaded scan files:")
    print('   ', end='')
    wjson = 0
    contents_path = Path(settings.DRAWINGS_DATA, "walletjson") 
    for yeardir in contents_path.iterdir(): 
        if yeardir.is_dir():
            for walletpath in yeardir.iterdir(): 
                if Path(walletpath, contentsjson).is_file():
                    walletname = walletpath.name
                    
                    if walletname not in wallets:
                        wjson += 1
                        if wjson % 10 == 0 :
                            print("\n   ", end='')

                        print(f"{walletname} ", end='')
                        fpath = Path(settings.SCANS_ROOT, str(yeardir.stem), walletname) 
                        # The wallets found from JSON should all have dates already
                        wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
                        wallets[walletname] = wallet
                        # could now also load the json but we don't. Do later, on-demand
                        # wallet.walletdate = wallet.date()
                        # could check if link to svx file is valid too.. but do on-demand later
                        # But we *do* set the walletyear:
                        setwalletyear(wallet)
                        if not created:
                            print(f"\n  - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?")
                            wallet.save()
    print(f'\n  - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets')
    wallets = Wallet.objects.filter(walletyear=None)
    for w in wallets:
        w.walletyear = datetime.date(1999, 1, 1)