troggle-unchained/parsers/scans.py

import sys
import os
import subprocess
import types
import stat
import csv
import re
import datetime
import shutil, filecmp

from functools import reduce
from pathlib import Path

import settings
from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
from troggle.core.models.troggle import DataIssue
from troggle.core.utils import save_carefully, GetListDir
from troggle.core.views.scans import datewallet

'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
'''

contentsjson = "contents.json"
#indexhtml = "walletindex.html"
git = settings.GIT

# to do: create a 'low priority' field, so that any such wallet does not appear in summary reports

                    
def CheckEmptyDate(wallet):
    '''If date is not set, get it from a linked survex file. 
    Could also look at filedates for the scans in expofiles/surveyscans/ , but these can be re-set by copying.
    '''
    earliest = datetime.datetime.now().date()
    
    # This is not working, can't see why. An scans parser now taking a very long time..
    #datewallet(wallet, earliest)
    return
    
def CheckEmptyPeople(wallet):
    '''If people list is empty, copy them from the survex files: all of them
   
    To be a Troggle model change; a many:many relationship between wallets and people,
    as well as being a list in the JSON file (which is the permanent repository). We want the many:many
    relationship so that we can filter wallets based on a person.
    
    For the moment, we will just get a list..
    '''
    return

def LoadListScansFile(wallet):
    gld = [ ]
    # flatten out any directories in these wallet folders - should not be any
    for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath):
        if fisdiryf:
            gld.extend(GetListDir(ffyf))
        else:
            gld.append((fyf, ffyf, fisdiryf))
    
    c=0
    for (fyf, ffyf, fisdiryf) in gld:
        if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif|xvi)(?i)$", fyf):
            singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet)
            singlescan.save()
            c+=1
            if c>=10:
                print(".", end='')
                c = 0
        
def load_all_scans():
    '''This iterates through the scans directories (either here or on the remote server)
    and builds up the models we can access later.
    It does NOT read or validate anything in the JSON data attached to each wallet. Those checks
    are done at runtime, when a wallet is accessed, not at import time.
    
    NOTE that parsers/survex.py does NOT create a wallet if it finds an unrecognised *REF wallet.
    Instead it reports an error in DataIssues. But it does make a link in the db between the
    existing wallet (probably no JSON, just a folder containing scans) and the survex file.
    '''
    print(' - Loading Survey Scans')

    SingleScan.objects.all().delete()
    Wallet.objects.all().delete()
    print(' - deleting all Wallet and SingleScan objects')
    DataIssue.objects.filter(parser='scans').delete()

    # first do the smkhs (large kh survey scans) directory
    # this seems to be never used ?!
    #We should load all the scans, even for nonstandard names. 
    manywallets_smkhs = Wallet(fpath=os.path.join(settings.SCANS_ROOT, "../surveys/smkhs"), walletname="smkhs") 
    print("smkhs", end=' ')
    if os.path.isdir(manywallets_smkhs.fpath):
        manywallets_smkhs.save()
        LoadListScansFile(manywallets_smkhs)
    else:
        print("smkhs NOT LOADED", end=' ')

    # iterate into the surveyscans directory
    print(' - ', end=' ')
    for walletname, fpath, fisdir in GetListDir(settings.SCANS_ROOT):
        if not fisdir:
            continue
        
        # do the year folders
        if re.match(r"\d\d\d\d$", walletname):
            print(f"{walletname}", end=' ')
            for walletname, fpath, fisdir in GetListDir(fpath):
                if fisdir:
                    wallet = Wallet(fpath=fpath, walletname=walletname)
                    # this is where we should load the contents.json for people so we can report on them later
                    # this is where we should record the year explicitly
                    # line 347 of view/uploads.py and needs refactoring for loading contentsjson
                    CheckEmptyDate(wallet)
                    CheckEmptyPeople(wallet)
                    wallet.save()
                    LoadListScansFile(wallet)
             
  
        else:
            # but We *should* load all the scans, even for nonstandard names. 
            print(f'\n - IGNORE {walletname} - {fpath}')
 
    # but we also need to check if JSON exists, even if there are no uploaded scan files
    contents_path = Path(settings.DRAWINGS_DATA, "walletjson") 
    for yeardir in contents_path.iterdir(): 
        if yeardir.is_dir():
            for walletpath in yeardir.iterdir(): 
                if Path(walletpath, contentsjson).is_file():
                    walletname = walletpath.name
                    wallet, created = Wallet.objects.update_or_create(walletname=walletname)
                    if created:
                        print(f"\n{walletname} created: only JSON, no actual uploaded scan files.", end=' ')
                        CheckEmptyDate(wallet)
                        CheckEmptyPeople(wallet)
                        wallet.save()
 

    print("", flush=True)