expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-25 08:41:51 +00:00
Code Issues 19 Projects 3 Activity

Cleaner monitoring output when importing wallets and scans

Browse Source
This commit is contained in:
Philip Sargent 2022-09-24 00:17:51 +03:00
parent 6e3fdd35c1
commit f3ecdd6d87
2 changed files with 20 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/views/scans.py
View File

@ -64,7 +64,7 @@ def caveifywallet(w):
'''Gets the cave from the list of survex files,
only selects one of them though. Only used for display.
'''
print(f' - Caveify {w=}')
#print(f' - Caveify {w=}')
blocknames = []
blocks = SurvexBlock.objects.filter(scanswallet = w)
for b in blocks:
@ -208,7 +208,7 @@ def cavewallets(request, caveid):
wallets.add(z)
else:
wurl = f"/scanupload/{z.walletname.replace('#',':')}"
print(f' - Unrecognised cave name \'{zcaveid}\' in {z.walletname} (out of {len(Gcavelookup):,} cave names')
print(f' - Unrecognised cave name \'{zcaveid}\' in {z.walletname} (out of {len(Gcavelookup):,} cave names and aliases)')
message = f" ! In {z.walletname} there is an unrecognised cave name '{zcaveid}' (out of {len(Gcavelookup):,} cave names"
DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl)

86
parsers/scans.py
View File

@ -26,46 +26,6 @@ git = settings.GIT
# to do: Actually read all the JSON files and set the survex file field appropriately!
# def GetListDir(sdir):
# '''handles url or file, so we can refer to a set of scans (not drawings) on another server
# returns a list of f (file), ff (file full path), is_dir (bool)
# REPLACE all use of this with Path.rglob() !
# '''
# res = [ ]
# if type(sdir) is str and sdir[:7] == "http://":
# # s = urllib.request.urlopen(sdir)
# message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]"
# print(message)
# DataIssue.objects.create(parser='Drawings', message=message)
# sdir[:7] = ""
# for f in os.listdir(sdir):
# if f[0] != ".":
# ff = os.path.join(sdir, f)
# res.append((f, ff, os.path.isdir(ff)))
# return res
# def LoadListScansFile(wallet):
# # formerly a generic troggle utility, written by who ? Being gradually expunged and replaced by python standard library functions
# gld = [ ]
# # flatten out any directories in these wallet folders - should not be any
# for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath):
# if fisdiryf:
# gld.extend(GetListDir(ffyf))
# else:
# gld.append((fyf, ffyf, fisdiryf))
# c=0
# for (fyf, ffyf, fisdiryf) in gld:
# if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif|xvi)(?i)$", fyf):
# singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet)
# singlescan.save()
# c+=1
# if c>=10:
# print(".", end='')
# c = 0
def load_all_scans():
'''This iterates through the scans directories (either here or on the remote server)
@ -93,7 +53,7 @@ def load_all_scans():
# they are if they are /2010/2010#33
# or /1996-1999NotKHbook/
# but not if they are /2010/2010#33/therion/ : the wallet is /2010#33/ not /therion/
print(' - ', end='')
print(' ', end='')
scans_path = Path(settings.SCANS_ROOT)
seen = []
c=0
@ -110,8 +70,8 @@ def load_all_scans():
c+=1
if c % 15 == 0 :
print(".", end='')
if c % 500 == 0 :
print("\n -", end='')
if c % 750 == 0 :
print("\n ", end='')
if p.parent.parent.parent.parent == scans_path:
# print(f"too deep {p}", end='\n')
@ -141,34 +101,18 @@ def load_all_scans():
if tag not in seen:
print(f" {tag.name} ", end='')
if len(str(tag.name)) > 17:
print('\n ', end='')
seen.append(tag)
print(f'\n - found and loaded {c:,} acceptable scan files in {len(wallets):,} wallets')
# if False:
# n=0
# for topfolder, fpath, fisdir in GetListDir(settings.SCANS_ROOT):
# if not fisdir:
# continue
# # do the year folders
# # if re.match(r"\d\d\d\d$", topfolder):
# print(f"{topfolder}", end=' ')
# for walletname, fpath, fisdir in GetListDir(fpath):
# if fisdir:
# wallet = Wallet(fpath=fpath, walletname=walletname)
# # this is where we should record the year explicitly
# # line 347 of view/uploads.py and needs refactoring for loading contentsjson
# wallet.save()
# LoadListScansFile(wallet)
# # else:
# # # but We *should* load all the scans, even for nonstandard names.
# # print(f'\n - IGNORE {topfolder} - {fpath}')
# print("", flush=True)
# but we also need to check if JSON exists, even if there are no uploaded scan files.
# Here we know there is a rigid folder structure, so no need to look for sub folders
print(f"\n - Checking for wallets where only JSON exists, but there are no actual uploaded scan files:")
print(' ', end='')
wjson = 0
contents_path = Path(settings.DRAWINGS_DATA, "walletjson")
for yeardir in contents_path.iterdir():
if yeardir.is_dir():
@ -177,9 +121,15 @@ def load_all_scans():
walletname = walletpath.name
if walletname not in wallets:
print(f" - {walletname} creation attempting: only JSON, no actual uploaded scan files.", end=' ')
wjson += 1
if wjson % 10 == 0 :
print("\n ", end='')
print(f"{walletname} ", end='')
wallet, created = Wallet.objects.update_or_create(walletname=walletname)
# should now also load the json and use it ! check &ref is correct or missing too
if created:
print(f" - {walletname} created: only JSON, no actual uploaded scan files.", end=' ')
wallets[walletname] = wallet
# could now also load the json and use it. check &ref is correct or missing too..
if not created:
print(f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?")
wallet.save()
print(f'\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets')