mirror of
https://expo.survex.com/repositories/troggle/.git
synced 2025-12-17 18:17:07 +00:00
461 lines
20 KiB
Python
461 lines
20 KiB
Python
import os
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import settings
|
|
from troggle.core.models.survex import DrawingFile
|
|
from troggle.core.models.troggle import DataIssue
|
|
from troggle.core.models.wallets import Wallet
|
|
|
|
"""Searches through all the :drawings: repository looking
|
|
for tunnel and therion files
|
|
"""
|
|
|
|
todo = """
|
|
- Fix lost links to 1999 wallets now they have been renamed
|
|
|
|
- implement: findimportinsert(therionfile, imp)
|
|
Tries to link the scrap (Therion format) to the referenced therion scrap
|
|
"""
|
|
|
|
rx_xth_me = re.compile(r"xth_me_image_insert.*{.*}$", re.MULTILINE)
|
|
rx_scrap = re.compile(r"^survey (\w*).*$", re.MULTILINE)
|
|
rx_input = re.compile(r"^input ", re.MULTILINE)
|
|
rx_line = re.compile(r"^line ", re.MULTILINE)
|
|
rx_ref = re.compile(r"^#?\s?ref\s*\:?\s*([^\s\t]*)", re.MULTILINE)
|
|
|
|
rx_skpath = re.compile(r"<skpath")
|
|
rx_pcpath = re.compile(r'<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"')
|
|
rx_pctext = re.compile(r'pctext.*?\*ref&space;([^&]*)')
|
|
|
|
|
|
# Supported suffixes are stored with a leading dot and lowercase (matches Path.suffix)
|
|
# We factor out the image-like extensions which are common to both sets so they
|
|
# are defined once and then reused to build the other sets.
|
|
IMAGE_LIKE_EXTS = {".png", ".jpg", ".jpeg", ".pdf", ".gif", ".txt", ".svg"}
|
|
|
|
# Extensions that we treat as "image-like" for the purposes of reference parsing
|
|
IMAGE_EXTS = set(IMAGE_LIKE_EXTS)
|
|
|
|
# Supported extensions include image-like ones plus drawing/text-specific ones
|
|
SUPPORTED_EXTENSIONS = IMAGE_LIKE_EXTS.union({".xml", ".th", ".th2"})
|
|
|
|
|
|
def _is_supported_suffix(suffix: str) -> bool:
|
|
"""Return True if `suffix` (e.g. '.png') is a supported drawings extension."""
|
|
if not suffix:
|
|
return False
|
|
return suffix.lower() in SUPPORTED_EXTENSIONS
|
|
|
|
|
|
def _is_image_suffix(suffix: str) -> bool:
|
|
"""Return True if `suffix` looks like an image/scan-type suffix."""
|
|
if not suffix:
|
|
return False
|
|
return suffix.lower() in IMAGE_EXTS
|
|
|
|
|
|
def fetch_drawingfiles_by_paths(paths, chunk_size: int = 500):
|
|
"""Fetch DrawingFile objects for the given iterable of paths in chunks.
|
|
|
|
This avoids building a very large SQL IN(...) clause which can exceed DB
|
|
parameter limits (SQLite defaults to 999 bound variables). A default
|
|
chunk_size of 500 is conservative and works well across backends.
|
|
|
|
Returns a dict mapping dwgpath -> DrawingFile (first match if duplicates).
|
|
"""
|
|
mapping = {}
|
|
if not paths:
|
|
return mapping
|
|
|
|
# Ensure we iterate over a list to allow slicing in chunks
|
|
rel_paths = list(paths)
|
|
for i in range(0, len(rel_paths), chunk_size):
|
|
chunk = rel_paths[i : i + chunk_size]
|
|
for obj in DrawingFile.objects.filter(dwgpath__in=chunk):
|
|
# if duplicates exist, preserve the first one seen
|
|
mapping.setdefault(obj.dwgpath, obj)
|
|
|
|
return mapping
|
|
|
|
rx_wallet = re.compile(r"""
|
|
# r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg|gif|txt))$", path
|
|
# This regex is designed to extract a specific directory prefix and a filename
|
|
# from the end of a path string.
|
|
|
|
# --- Group 1: Directory or Survey Prefix ---
|
|
( # Start of Capture Group 1
|
|
\d{4}\#X?\d+\w? # Matches patterns like "2025#123", "2016#X04" or "1999#45a", NB # must be escaped in VERBOSE mode
|
|
| # OR
|
|
1995-96kh # Matches the literal string "1995-96kh"
|
|
| # OR
|
|
92-94Surveybookkh # Matches the literal string "92-94Surveybookkh"
|
|
| # OR
|
|
1991surveybook # Matches the literal string "1991surveybook"
|
|
| # OR
|
|
smkhs # Matches the literal string "smkhs"
|
|
) # End of Capture Group 1
|
|
|
|
/ # A literal forward slash separating the parts
|
|
|
|
# --- Group 2: Filename ---
|
|
( # Start of Capture Group 2
|
|
.*? # Non-greedily match the filename stem (any character)
|
|
(?: # Start of a non-capturing group for the extension
|
|
png|jpg|pdf|jpeg|gif|txt|svg
|
|
) # End of the extension group
|
|
) # End of Capture Group 2
|
|
|
|
$ # Anchor, ensuring the match is at the end of the string
|
|
""", re.VERBOSE | re.IGNORECASE)
|
|
def _set_filesize_and_check(fullpath, model_obj, parser_label, url_prefix="/dwgdataraw"):
|
|
"""Set model_obj.filesize from filesystem and create DataIssue if missing/zero.
|
|
Returns True if file exists and has size > 0, False otherwise.
|
|
"""
|
|
try:
|
|
fullpath = Path(fullpath)
|
|
size = fullpath.stat().st_size
|
|
except Exception as e:
|
|
message = f"! Unable to stat file {fullpath}: {e}"
|
|
print(message)
|
|
DataIssue.objects.create(parser=parser_label, message=message, url=f"{url_prefix}/{getattr(model_obj, 'dwgpath', '')}")
|
|
return False
|
|
|
|
model_obj.filesize = size
|
|
# Do not save here; caller should include 'filesize' in the bulk update set.
|
|
if size <= 0:
|
|
message = f"! Zero length {parser_label.lower()} file {fullpath}"
|
|
print(message)
|
|
DataIssue.objects.create(parser=parser_label, message=message, url=f"{url_prefix}/{getattr(model_obj, 'dwgpath', '')}")
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def _read_text_file(fullpath):
|
|
"""Read text file robustly, returning a str (falls back to binary decode)."""
|
|
try:
|
|
path = Path(fullpath)
|
|
with path.open("r", encoding="utf-8", errors="replace") as fh:
|
|
return fh.read()
|
|
except Exception:
|
|
try:
|
|
with path.open("rb") as fh:
|
|
return fh.read().decode("utf-8", errors="replace")
|
|
except Exception as e:
|
|
print(f"! Unable to read file {fullpath}: {e}")
|
|
return ""
|
|
|
|
def parse_tnl_file(dwgfile, path):
|
|
"""Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file
|
|
which we have already seen when we imported all the files we could find in the surveyscans direstories.
|
|
|
|
The purpose is to find cross-references between Tunnel drawing files and wallets
|
|
AND to find the names of the scanfiles in that wallet - from reading the Tunnel file not from interrogating the wallet.
|
|
Note that this means that the list of scanfiles will be as it was when the drawing was created, not as it is now. Perhaps
|
|
we should not actually do it this way ? Or at least, label the table heading.
|
|
|
|
This is used to tie drawings to the wallet, and thus the original survey data. Tunnel files
|
|
contain a centreline which is an embedded survex file.
|
|
"""
|
|
# Delegate to the unified reference processor for consistent behaviour
|
|
_process_reference(dwgfile, path, parser_label="Tunnel")
|
|
|
|
|
|
def _process_reference(dwgfile, path, parser_label="Tunnel"):
|
|
"""Unified processor to link drawing files to wallets/scans or referenced drawings.
|
|
|
|
- If `path` matches a wallet pattern (rx_wallet), link the wallet and try to find the scan file in the wallet.
|
|
- If `path` looks like an image, do nothing (images are not treated as references here).
|
|
- Otherwise, treat `path` as a possible reference to another drawing (by name) and link via `dwgcontains`.
|
|
"""
|
|
|
|
wallet, scansfile = None, None
|
|
if not path:
|
|
return None, None
|
|
|
|
if mscansdir := rx_wallet.search(path):
|
|
scanswalletl = Wallet.objects.filter(walletname=mscansdir.group(1))
|
|
if len(scanswalletl):
|
|
wallet = scanswalletl[0]
|
|
if len(scanswalletl) > 1:
|
|
message = f"! More than one scan FOLDER matches filter query. [{scanswalletl[0]}]: {mscansdir.group(1)} {mscansdir.group(2)} {dwgfile.dwgpath} {path}"
|
|
print(message)
|
|
DataIssue.objects.create(parser=parser_label, message=message)
|
|
|
|
if wallet:
|
|
scansfilel = wallet.singlescan_set.filter(name=mscansdir.group(2))
|
|
if len(scansfilel):
|
|
if len(scansfilel) > 1:
|
|
plist = [sf.ffile for sf in scansfilel]
|
|
message = f"! More than one image FILENAME matches filter query. [{scansfilel[0]}]: {mscansdir.group(1)} {mscansdir.group(2)} {dwgfile.dwgpath} {path} {plist}"
|
|
print(message)
|
|
DataIssue.objects.create(parser=parser_label, message=message)
|
|
scansfile = scansfilel[0]
|
|
|
|
if wallet:
|
|
dwgfile.dwgwallets.add(wallet)
|
|
if scansfile:
|
|
dwgfile.scans.add(scansfile)
|
|
return wallet, scansfile
|
|
|
|
# Not a wallet reference; check image extension and possibly drawing-to-drawing reference
|
|
suffix = Path(path).suffix.lower()
|
|
if _is_image_suffix(suffix):
|
|
# It's an image/scanned file type; we don't treat it as a referenced drawing
|
|
return None, None
|
|
|
|
# Not an image file: perhaps a reference to another drawing (no ext or other ext)
|
|
name = Path(path).name
|
|
rdwgfilel = DrawingFile.objects.filter(dwgname=name) # Check if it is another drawing file we have already seen
|
|
if len(rdwgfilel):
|
|
if len(rdwgfilel) > 1:
|
|
plist = [df.dwgpath for df in rdwgfilel]
|
|
message = f"- Warning {len(rdwgfilel)} files named '{name}' {plist}"
|
|
print(message)
|
|
DataIssue.objects.create(parser=parser_label, message=message, url=f"/dwgdataraw/{path}")
|
|
rdwgfile = rdwgfilel[0]
|
|
if hasattr(dwgfile, 'dwgcontains'):
|
|
dwgfile.dwgcontains.add(rdwgfile)
|
|
|
|
return None, None
|
|
|
|
|
|
def findwalletimage(therionfile, foundpath):
|
|
"""Tries to link the drawing file (Therion format) to the referenced image (scan) file"""
|
|
# Delegate to the unified reference processor for consistent behaviour
|
|
foundpath = foundpath.strip("{}")
|
|
wallet, scansfile = _process_reference(therionfile, foundpath, parser_label="Therion")
|
|
|
|
# If a wallet was found but no scan was associated from the wallet, record a DataIssue
|
|
if wallet and not scansfile:
|
|
scanfilename = Path(foundpath).name
|
|
message = f'! In {wallet.walletname} scanned file is not actually found {scanfilename} mentioned in "{therionfile.dwgpath}"'
|
|
wurl = f"/survey_scans/{wallet.walletname}/".replace("#", ":")
|
|
DataIssue.objects.create(parser="Therion", message=message, url=wurl)
|
|
|
|
|
|
def findimportinsert(therionfile, imp):
|
|
"""Tries to link the scrap (Therion format) to the referenced therion scrap"""
|
|
pass
|
|
|
|
|
|
def settherionfileinfo(filetuple):
|
|
"""Read in the drawing file contents and sets values on the dwgfile object"""
|
|
thtype, therionfile = filetuple
|
|
|
|
ff = Path(settings.DRAWINGS_DATA) / therionfile.dwgpath
|
|
modified = set()
|
|
if _set_filesize_and_check(ff, therionfile, "Therion"):
|
|
modified.add("filesize")
|
|
|
|
ttext = _read_text_file(ff)
|
|
|
|
# The equivalent for a tunnel 'path' would be a .th2 'line wall' or 'scrap'
|
|
# print(len(re.findall(r"line", ttext)))
|
|
if thtype == "th":
|
|
therionfile.npaths = len(rx_input.findall(ttext))
|
|
modified.add("npaths")
|
|
if wallet_texts := rx_ref.findall(ttext):
|
|
# print(f"#ref {therionfile.dwgname} : {wallet_text}")
|
|
if wallets := Wallet.objects.filter(walletname__in=wallet_texts): # ! Django idiom not used elsewhere. A filter using a list of strings.
|
|
for w in wallets:
|
|
therionfile.dwgwallets.add(w)
|
|
elif thtype == "th2":
|
|
therionfile.npaths = len(rx_line.findall(ttext))
|
|
modified.add("npaths")
|
|
|
|
# scan and look for survex blocks that might have been included, and image scans (as for tunnel drawings)
|
|
# which would populate dwgfile.survexfile
|
|
|
|
# in .th2 files:
|
|
# ##XTHERION## xth_me_image_insert {500 1 1.0} {1700 {}} ../../../expofiles/surveyscans/2014/01popped_elev1.jpeg 0 {}
|
|
# scrap blownout -projection plan -scale [-81.0 -42.0 216.0 -42.0 0.0 0.0 7.5438 0.0 m]
|
|
|
|
for xth_me in rx_xth_me.findall(ttext):
|
|
# WORK IN PROGRESS. Do not clutter up the DataIssues list with this
|
|
# Surely not needed for .th files ?? only .th2 ?
|
|
message = f"! Un-parsed image filename: {therionfile.dwgname} : {xth_me.split()[-3]} - {therionfile.dwgpath}"
|
|
# print(message)
|
|
# DataIssue.objects.create(parser='xTherion', message=message, url=f'/dwgdataraw/{therionfile.dwgpath}')
|
|
# ! Un-parsed image filename: 107coldest : ../../../expofiles/surveyscans/2015/2015#20/notes.jpg - therion/plan/107coldest.th2
|
|
|
|
with open("therionrefs.log", "a") as lg:
|
|
lg.write(message + "\n")
|
|
|
|
findwalletimage(therionfile, xth_me.split()[-3])
|
|
# print(f"{therionfile.dwgname} :{xth_me.split()[-3]}")
|
|
|
|
# therionfile.save()
|
|
for inp in rx_input.findall(ttext):
|
|
# if this 'input' is a .th2 file we have already seen, then we can assign this as a sub-file
|
|
# but we would need to disentangle to get the current path properly
|
|
message = f"! Un-set (?) Therion .th2 input: - {therionfile.dwgname} : {inp} - {therionfile.dwgpath}"
|
|
# print(message)
|
|
DataIssue.objects.create(parser="xTherion", message=message, url=f"/dwgdataraw/{therionfile.dwgpath}")
|
|
findimportinsert(therionfile, inp)
|
|
|
|
# Defer saving scalar fields; caller will perform bulk_update.
|
|
return modified
|
|
|
|
|
|
def settnlfileinfo(dwgfile):
|
|
"""Read in the drawing file contents and sets values on the dwgfile object
|
|
Should try to read the date too e.g. tunneldate="2010-08-16 22:51:57
|
|
then we could display on the master calendar per expo.
|
|
|
|
Tunnel files are unfortunately not fully compliant XML so we can't use any of the XML parsing tools
|
|
available. Thanks Julian.
|
|
|
|
*ref wallet identifiers may be found in at least two different places in tunnel files.
|
|
"""
|
|
ff = Path(settings.DRAWINGS_DATA) / dwgfile.dwgpath
|
|
modified = set()
|
|
if _set_filesize_and_check(ff, dwgfile, "Tunnel"):
|
|
modified.add("filesize")
|
|
|
|
ttext = _read_text_file(ff)
|
|
|
|
dwgfile.npaths = len(rx_skpath.findall(ttext))
|
|
modified.add("npaths")
|
|
|
|
# example drawing file in Tunnel format.
|
|
# <tunnelxml tunnelversion="version2009-06-21 Matienzo" tunnelproject="ireby" tunneluser="goatchurch" tunneldate="2009-06-29 23:22:17">
|
|
# <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0">
|
|
|
|
# sfsketch="surveyscans/2025/2025#41/plan_diddlypot.png"
|
|
for scanfile_path, style in rx_pcpath.findall(ttext):
|
|
parse_tnl_file(dwgfile, scanfile_path)
|
|
|
|
# <pathcodes>
|
|
# <pctext style="survey" nodeposxrel="-1.0" nodeposyrel="-1.0"> *file_begin "/home/expo/loser/caves-1623/2025-dw-01/trip1.svx" "trip1.svx" | *begin 1 | *export 1 25 | | ; Cave: 2025-dw-01 | ; Area in cave/QM: Entrance series | *title "2025-dw-01" | *date 2025.07.13 | *team "Dylan Wase" notes | *team "Daniel Gorst" dog | *instrument SAP "SAP6 Dylan" | *ref 2025#20 |
|
|
for refs in rx_pctext.findall(ttext):
|
|
try:
|
|
wallets = Wallet.objects.filter(walletname=refs)
|
|
if wallets:
|
|
for w in wallets:
|
|
dwgfile.dwgwallets.add(w)
|
|
except Exception as e:
|
|
message = f" ! wallet not found referenced from {dwgfile} -- '{refs}' ({e}) "
|
|
print(message)
|
|
DataIssue.objects.create(parser="Tunnel", message=message, url=f"/dwgdataraw/{dwgfile}")
|
|
# should also scan and look for survex blocks that might have been included, and image scans
|
|
# which would populate dwgfile.survexfile
|
|
|
|
# Defer scalar saves to bulk_update; return set of modified fields
|
|
return modified
|
|
|
|
|
|
def setdrwfileinfo(dwgfile):
|
|
"""Read in the drawing file contents and sets values on the dwgfile object,
|
|
but these are SVGs, PDFs or .txt files, so there is no useful format to search for
|
|
|
|
"""
|
|
ff = Path(settings.DRAWINGS_DATA) / dwgfile.dwgpath
|
|
# Set filesize (if available) but do not save; return modified field names
|
|
if not _set_filesize_and_check(ff, dwgfile, "drawings"):
|
|
return set()
|
|
return {"filesize"}
|
|
|
|
def load_drawings_files():
|
|
"""
|
|
Why do we have all this detection of file types/! Why not use get_mime_types ?
|
|
What is it all for ??
|
|
|
|
We import JPG, PNG and SVG files; which have already been put on the server,
|
|
but the upload form intentionally refuses to upload PNG and JPG (though it does allow SVG)
|
|
"""
|
|
all_xml = []
|
|
drawdatadir = Path(settings.DRAWINGS_DATA)
|
|
DrawingFile.objects.all().delete()
|
|
DataIssue.objects.filter(parser="drawings").delete()
|
|
DataIssue.objects.filter(parser="Therion").delete()
|
|
DataIssue.objects.filter(parser="xTherion").delete()
|
|
DataIssue.objects.filter(parser="Tunnel").delete()
|
|
if os.path.isfile("therionrefs.log"):
|
|
os.remove("therionrefs.log")
|
|
|
|
# Walk the tree with pathlib, skip hidden and backup files
|
|
files_meta = [] # list of tuples (ext, rel_path, dwgname, pathobj)
|
|
for p in drawdatadir.rglob('*'):
|
|
# Ignore anything under a .git directory
|
|
if '.git' in p.parts:
|
|
continue
|
|
if p.name.startswith('.') or p.name.endswith('~'):
|
|
continue
|
|
if p.is_dir():
|
|
continue
|
|
|
|
suffix = p.suffix.lower()
|
|
if _is_supported_suffix(suffix) or suffix == '':
|
|
rel = p.relative_to(drawdatadir).as_posix()
|
|
if suffix == '':
|
|
dwgname = p.name
|
|
ext = ''
|
|
else:
|
|
dwgname = p.stem
|
|
ext = suffix[1:]
|
|
|
|
files_meta.append((ext, rel, dwgname, p))
|
|
|
|
# Bulk create DrawingFile instances to avoid many individual DB saves
|
|
if files_meta:
|
|
objs_to_create = [DrawingFile(dwgpath=rel, dwgname=dwgname) for (_, rel, dwgname, _) in files_meta]
|
|
# Use chunks to avoid huge single queries
|
|
chunk_size = 700
|
|
for i in range(0, len(objs_to_create), chunk_size):
|
|
DrawingFile.objects.bulk_create(objs_to_create[i : i + chunk_size])
|
|
|
|
# Re-fetch created objects and map by dwgpath using a chunked fetch helper
|
|
rel_paths = [rel for (_, rel, _, _) in files_meta]
|
|
mapping = fetch_drawingfiles_by_paths(rel_paths, chunk_size=500)
|
|
|
|
# Reconstruct all_xml using the created model instances
|
|
for ext, rel, _, p in files_meta:
|
|
dwgfile = mapping.get(rel)
|
|
if dwgfile:
|
|
all_xml.append((ext, dwgfile, p))
|
|
|
|
print(f" - {len(all_xml)} Drawings files found")
|
|
|
|
# Process in a deterministic order; ensure .th2 are handled before .th
|
|
ext_priority = {'th2': 0, 'th': 1}
|
|
all_xml.sort(key=lambda t: ext_priority.get(t[0], 2))
|
|
|
|
print(f" - Drawing found, starting parsing...")
|
|
|
|
# Process files and collect modified scalar fields for bulk update
|
|
modified_map = {} # {DrawingFile instance: set(fields)}
|
|
for extension, filename, pathobj in all_xml:
|
|
modified = set()
|
|
if extension in {"pdf", "txt", "svg", "jpg", "png", ""}:
|
|
modified = setdrwfileinfo(filename) or set()
|
|
elif extension == "xml":
|
|
modified = settnlfileinfo(filename) or set()
|
|
# important to import .th2 files before .th so that we can assign them when found in .th files
|
|
elif extension == "th2":
|
|
modified = settherionfileinfo(("th2", filename)) or set()
|
|
elif extension == "th":
|
|
modified = settherionfileinfo(("th", filename)) or set()
|
|
|
|
if modified:
|
|
modified_map.setdefault(filename, set()).update(modified)
|
|
|
|
# Bulk update scalar fields grouped by identical field-sets to use bulk_update efficiently
|
|
from collections import defaultdict
|
|
|
|
groups = defaultdict(list) # {tuple(fields): [instances]}
|
|
for inst, fields in modified_map.items():
|
|
key = tuple(sorted(fields))
|
|
groups[key].append(inst)
|
|
|
|
for fields_tuple, instances in groups.items():
|
|
fields_list = list(fields_tuple)
|
|
# Use a conservative batch size
|
|
DrawingFile.objects.bulk_update(instances, fields_list, batch_size=500)
|
|
|
|
print(f" - Drawings parsed and database updated.")
|
|
|
|
# for drawfile in DrawingFile.objects.all():
|
|
# SetTunnelfileInfo(drawfile)
|