import os import re from pathlib import Path import time import settings from troggle.core.models.survex import DrawingFile from troggle.core.models.troggle import DataIssue from troggle.core.models.wallets import Wallet """Searches through all the :drawings: repository looking for tunnel and therion files """ todo = """ - Fix lost links to 1999 wallets now they have been renamed - implement: findimportinsert(therionfile, imp) Tries to link the scrap (Therion format) to the referenced therion scrap """ rx_xth_me = re.compile(r"xth_me_image_insert.*{.*}$", re.MULTILINE) rx_scrap = re.compile(r"^survey (\w*).*$", re.MULTILINE) rx_input = re.compile(r"^input ", re.MULTILINE) rx_line = re.compile(r"^line ", re.MULTILINE) rx_ref = re.compile(r"^#?\s?ref\s*\:?\s*([^\s\t]*)", re.MULTILINE) rx_skpath = re.compile(r" bool: """Return True if `suffix` (e.g. '.png') is a supported drawings extension.""" if not suffix: return False return suffix.lower() in SUPPORTED_EXTENSIONS def _is_image_suffix(suffix: str) -> bool: """Return True if `suffix` looks like an image/scan-type suffix.""" if not suffix: return False return suffix.lower() in IMAGE_EXTS def fetch_drawingfiles_by_paths(paths, chunk_size: int = 500): """Fetch DrawingFile objects for the given iterable of paths in chunks. This avoids building a very large SQL IN(...) clause which can exceed DB parameter limits (SQLite defaults to 999 bound variables). A default chunk_size of 500 is conservative and works well across backends. Returns a dict mapping dwgpath -> DrawingFile (first match if duplicates). """ mapping = {} if not paths: return mapping # Ensure we iterate over a list to allow slicing in chunks rel_paths = list(paths) for i in range(0, len(rel_paths), chunk_size): chunk = rel_paths[i : i + chunk_size] for obj in DrawingFile.objects.filter(dwgpath__in=chunk): # if duplicates exist, preserve the first one seen mapping.setdefault(obj.dwgpath, obj) return mapping rx_wallet = re.compile(r""" # This regex is designed to extract a specific directory prefix (walletname) and a filename # from the end of a path string. # --- Group 1: Directory or Survey Prefix --- # but current AND HISTORIC wallet namings, as images have been edited over the years ( # Start of Capture Group 1 \d{4}\#X?\d+\w? # Matches patterns like "2025#123", "2016#X04" or "1999#45a", NB # must be escaped in VERBOSE mode | # OR 1995-96kh # Matches the literal string "1995-96kh" | 1989LUSS # Matches the literal string "1989LUSS" | # OR 1989Surveybook # Matches the literal string "1989Surveybook" | # OR 1990NotKHsurveybook # Matches the literal string "1990NotKHsurveybook" | # OR 199?kh # | # OR 199?notkh # | # OR 199?Surveybookkh # Matches the literal string "92-94Surveybookkh" | # OR 1992-94NotKHSurveybook # Matches the literal string "92-94Surveybookkh" | # OR 92-94Surveybookkh # Matches the literal string "92-94Surveybookkh" | # OR 1991surveybook # Matches the literal string "1991surveybook" | # OR 1991surveybook # Matches the literal string "1991surveybook" | # OR smkhs # This is now expofiles/surveys/smkhs/ not in surveyscans/ at all. ) # End of Capture Group 1 / # A literal forward slash separating the parts # --- Group 2: Filename --- ( # Start of Capture Group 2 .*? # Non-greedily match the filename stem (any character) (?: # Start of a non-capturing group for the extension png|jpg|pdf|jpeg|gif|txt|svg ) # End of the extension group ) # End of Capture Group 2 $ # Anchor, ensuring the match is at the end of the string """, re.VERBOSE | re.IGNORECASE) def _set_filesize_and_check(fullpath, model_obj, parser_label, url_prefix="/dwgdataraw"): """Set model_obj.filesize from filesystem and create DataIssue if missing/zero. Returns True if file exists and has size > 0, False otherwise. """ try: fullpath = Path(fullpath) size = fullpath.stat().st_size except Exception as e: message = f"! Unable to stat file {fullpath}: {e}" print(message) DataIssue.objects.update_or_create(parser=parser_label, message=message, url=f"{url_prefix}/{getattr(model_obj, 'dwgpath', '')}") return False model_obj.filesize = size # Do not save here; caller should include 'filesize' in the bulk update set. if size <= 0: message = f"! Zero length {parser_label.lower()} file {fullpath}" print(message) DataIssue.objects.update_or_create(parser=parser_label, message=message, url=f"{url_prefix}/{getattr(model_obj, 'dwgpath', '')}") return False return True def _read_text_file(fullpath): """Read text file robustly, returning a str (falls back to binary decode).""" try: path = Path(fullpath) with path.open("r", encoding="utf-8", errors="replace") as fh: return fh.read() except Exception: try: with path.open("rb") as fh: return fh.read().decode("utf-8", errors="replace") except Exception as e: print(f"! Unable to read file {fullpath}: {e}") return "" def parse_tnl_file(dwgfile, path): """Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file which we have already seen when we imported all the files we could find in the surveyscans direstories. The purpose is to find cross-references between Tunnel drawing files and wallets AND to find the names of the scanfiles in that wallet - from reading the Tunnel file not from interrogating the wallet. Note that this means that the list of scanfiles will be as it was when the drawing was created, not as it is now. Perhaps we should not actually do it this way ? Or at least, label the table heading. This is used to tie drawings to the wallet, and thus the original survey data. Tunnel files contain a centreline which is an embedded survex file. """ # Delegate to the unified reference processor for consistent behaviour _process_reference(dwgfile, path, parser_label="Tunnel") def _handle_obsolete_wallets(old_wallet, dwgfile, path, parser_label): message = f"- Warning {old_wallet} not a currently valid wallet name. In {path}" print(message) DataIssue.objects.update_or_create(parser=parser_label, message=message, url=f"/dwgdataraw/{path}") def _process_reference(dwgfile, path, parser_label="Tunnel"): """Unified processor to link drawing files to wallets/scans or referenced drawings. - If `path` matches a wallet pattern (rx_wallet), link the wallet and try to find the scan file in the wallet. - If `path` looks like an image, do nothing (images are not treated as references here - yet). - Otherwise, treat `path` as a possible reference to another drawing (by name) and link via `dwgcontains`. """ wallet, scansfile = None, None if not path: return None, None if mscansdir := rx_wallet.search(path): scanswalletl = Wallet.objects.filter(walletname=mscansdir.group(1)) # wallet name if len(scanswalletl): wallet = scanswalletl[0] if len(scanswalletl) > 1: message = f"! More than one scan FOLDER matches filter query. [{scanswalletl[0]}]: {mscansdir.group(1)} {mscansdir.group(2)} {dwgfile.dwgpath} {path}" print(message) DataIssue.objects.update_or_create(parser=parser_label, message=message) else: # found a wallet name, but it is not one we recognise as having been imported _handle_obsolete_wallets(mscansdir.group(1),dwgfile, path, parser_label) if wallet: scansfilel = wallet.singlescan_set.filter(name=mscansdir.group(2)) # file name if len(scansfilel): if len(scansfilel) > 1: plist = [sf.ffile for sf in scansfilel] message = f"! More than one image FILENAME matches filter query. [{scansfilel[0]}]: {mscansdir.group(1)} {mscansdir.group(2)} {dwgfile.dwgpath} {path} {plist}" print(message) DataIssue.objects.update_or_create(parser=parser_label, message=message) scansfile = scansfilel[0] if wallet: dwgfile.dwgwallets.add(wallet) if scansfile: dwgfile.scans.add(scansfile) # If a wallet was found but no scan was associated from the wallet, record a DataIssue. There are a lot of these.. if wallet and not scansfile: scanfilename = Path(path).name message = f"! In '{wallet.walletname}' scanned file is not actually found '{scanfilename}' in '{path}'" wurl = f"/survey_scans/{wallet.walletname}/".replace("#", ":") DataIssue.objects.update_or_create(parser=parser_label, message=message, url=wurl) return wallet, scansfile # Not a wallet reference; check image extension and possibly drawing-to-drawing reference suffix = Path(path).suffix.lower() if _is_image_suffix(suffix): # It's an image/scanned file type; we don't treat it as a referenced drawing, though in future we should note the link return None, None # Not an image file: perhaps a reference to another drawing (no ext or other ext) name = Path(path).name rdwgfilel = DrawingFile.objects.filter(dwgname=name) # Check if it is another drawing file we have already seen if len(rdwgfilel): if len(rdwgfilel) > 1: plist = [df.dwgpath for df in rdwgfilel] message = f"- Warning {len(rdwgfilel)} files named '{name}' {plist}" print(message) DataIssue.objects.update_or_create(parser=parser_label, message=message, url=f"/dwgdataraw/{path}") rdwgfile = rdwgfilel[0] if hasattr(dwgfile, 'dwgcontains'): # implement model change in models/survex.py to use this dwgfile.dwgcontains.add(rdwgfile) return None, None def findimportinsert(therionfile, imp): """Tries to link the scrap (Therion format) to the referenced therion scrap""" pass def _assign_wallets_for_model(model_obj, wallet_names, parser_label="Tunnel"): """Assign wallets to `model_obj` by wallet name(s). wallet_names may be a single string or an iterable of names. This function will add any Wallets found via Wallet.objects.filter(walletname__in=...) to model_obj.dwgwallets and return the list of matched Wallet objects. If none are found, or an exception occurs, a DataIssue is recorded with parser set to `parser_label`. """ if not wallet_names: return [] # Normalize to list of names if isinstance(wallet_names, (str, bytes)): names = [str(wallet_names)] else: try: names = [str(n) for n in wallet_names] except Exception: names = [str(wallet_names)] try: wallets = list(Wallet.objects.filter(walletname__in=names)) if wallets: for w in wallets: model_obj.dwgwallets.add(w) return wallets # Nothing found: record a DataIssue message = f" ! wallet(s) '{names}' not found from {getattr(model_obj, 'dwgpath', model_obj)}" print(message) DataIssue.objects.update_or_create(parser=parser_label, message=message, url=f"/dwgdataraw/{getattr(model_obj, 'dwgpath', '')}") return [] except Exception as e: message = f" ! Exception while looking up wallet(s) '{names}' from {getattr(model_obj, 'dwgpath', model_obj)} -- ({e})" print(message) DataIssue.objects.update_or_create(parser=parser_label, message=message, url=f"/dwgdataraw/{getattr(model_obj, 'dwgpath', '')}") return [] def settherionfileinfo(filetuple): """Read in the drawing file contents and sets values on the dwgfile object""" thtype, therionfile = filetuple ff = Path(settings.DRAWINGS_DATA) / therionfile.dwgpath modified = set() if _set_filesize_and_check(ff, therionfile, "Therion"): modified.add("filesize") ttext = _read_text_file(ff) # The equivalent for a tunnel 'path' would be a .th2 'line wall' or 'scrap' # print(len(re.findall(r"line", ttext))) if thtype == "th": therionfile.npaths = len(rx_input.findall(ttext)) modified.add("npaths") if wallet_texts := rx_ref.findall(ttext): # Delegate wallet assignment to helper; use parser_label 'Therion' _assign_wallets_for_model(therionfile, wallet_texts, parser_label="Therion") elif thtype == "th2": therionfile.npaths = len(rx_line.findall(ttext)) modified.add("npaths") # scan and look for survex blocks that might have been included, and image scans (as for tunnel drawings) # which would populate dwgfile.survexfile # in .th2 files: # ##XTHERION## xth_me_image_insert {500 1 1.0} {1700 {}} ../../../expofiles/surveyscans/2014/01popped_elev1.jpeg 0 {} # scrap blownout -projection plan -scale [-81.0 -42.0 216.0 -42.0 0.0 0.0 7.5438 0.0 m] for xth_me in rx_xth_me.findall(ttext): # WORK IN PROGRESS. Do not clutter up the DataIssues list with this # Surely not needed for .th files ?? only .th2 ? message = f"! Un-parsed image filename: {therionfile.dwgname} : {xth_me.split()[-3]} - {therionfile.dwgpath}" # print(message) # DataIssue.objects.update_or_create(parser='xTherion', message=message, url=f'/dwgdataraw/{therionfile.dwgpath}') # ! Un-parsed image filename: 107coldest : ../../../expofiles/surveyscans/2015/2015#20/notes.jpg - therion/plan/107coldest.th2 with open("therionrefs.log", "a") as lg: lg.write(message + "\n") foundpath = xth_me.split()[-3].strip("{}") _process_reference(therionfile, foundpath, parser_label="Therion") for inp in rx_input.findall(ttext): # if this 'input' is a .th2 file we have already seen, then we can assign this as a sub-file # but we would need to disentangle to get the current path properly message = f"! Un-set (?) Therion .th2 input: - {therionfile.dwgname} : {inp} - {therionfile.dwgpath}" # print(message) DataIssue.objects.update_or_create(parser="xTherion", message=message, url=f"/dwgdataraw/{therionfile.dwgpath}") findimportinsert(therionfile, inp) # Defer saving scalar fields; caller will perform bulk_update. return modified def settnlfileinfo(dwgfile): """Read in the drawing file contents and sets values on the dwgfile object Should try to read the date too e.g. tunneldate="2010-08-16 22:51:57 then we could display on the master calendar per expo. Tunnel files are unfortunately not fully compliant XML so we can't use any of the XML parsing tools available. Thanks Julian. *ref wallet identifiers may be found in at least two different places in tunnel files. """ ff = Path(settings.DRAWINGS_DATA) / dwgfile.dwgpath modified = set() if _set_filesize_and_check(ff, dwgfile, "Tunnel"): modified.add("filesize") ttext = _read_text_file(ff) dwgfile.npaths = len(rx_skpath.findall(ttext)) modified.add("npaths") # example drawing file in Tunnel format. # # # sfsketch="surveyscans/2025/2025#41/plan_diddlypot.png" for scanfile_path, style in rx_pcpath.findall(ttext): parse_tnl_file(dwgfile, scanfile_path) # # *file_begin "/home/expo/loser/caves-1623/2025-dw-01/trip1.svx" "trip1.svx" | *begin 1 | *export 1 25 | | ; Cave: 2025-dw-01 | ; Area in cave/QM: Entrance series | *title "2025-dw-01" | *date 2025.07.13 | *team "Dylan Wase" notes | *team "Daniel Gorst" dog | *instrument SAP "SAP6 Dylan" | *ref 2025#20 | for refs in rx_pctext.findall(ttext): if refs: # Delegate wallet lookup/assignment to helper for consistent handling _assign_wallets_for_model(dwgfile, refs, parser_label="Tunnel") # should also scan and look for survex blocks that might have been included, and image scans # which would populate dwgfile.survexfile # Defer scalar saves to bulk_update; return set of modified fields return modified def setdrwfileinfo(dwgfile): """Read in the drawing file contents and sets values on the dwgfile object, but these are SVGs, PDFs or .txt files, so there is no useful format to search for """ ff = Path(settings.DRAWINGS_DATA) / dwgfile.dwgpath # Set filesize (if available) but do not save; return modified field names if not _set_filesize_and_check(ff, dwgfile, "drawings"): return set() return {"filesize"} def load_drawings_files(): """ Why do we have all this detection of file types/! Why not use get_mime_types ? What is it all for ?? We import JPG, PNG and SVG files; which have already been put on the server, but the upload form intentionally refuses to upload PNG and JPG (though it does allow SVG) """ # Track elapsed time start_time = time.perf_counter() all_xml = [] drawdatadir = Path(settings.DRAWINGS_DATA) DrawingFile.objects.all().delete() DataIssue.objects.filter(parser="drawings").delete() DataIssue.objects.filter(parser="Therion").delete() DataIssue.objects.filter(parser="xTherion").delete() DataIssue.objects.filter(parser="Tunnel").delete() if os.path.isfile("therionrefs.log"): os.remove("therionrefs.log") # Walk the tree with pathlib, skip hidden and backup files files_meta = [] # list of tuples (ext, rel_path, dwgname, pathobj) for p in drawdatadir.rglob('*'): # Ignore anything under a .git directory if '.git' in p.parts: continue if p.name.startswith('.') or p.name.endswith('~'): continue if p.is_dir(): continue suffix = p.suffix.lower() if _is_supported_suffix(suffix) or suffix == '': rel = p.relative_to(drawdatadir).as_posix() if suffix == '': dwgname = p.name ext = '' else: dwgname = p.stem ext = suffix[1:] files_meta.append((ext, rel, dwgname, p)) # Bulk create DrawingFile instances to avoid many individual DB saves if files_meta: objs_to_create = [DrawingFile(dwgpath=rel, dwgname=dwgname) for (_, rel, dwgname, _) in files_meta] # Use chunks to avoid huge single queries chunk_size = 700 for i in range(0, len(objs_to_create), chunk_size): DrawingFile.objects.bulk_create(objs_to_create[i : i + chunk_size]) # Re-fetch created objects and map by dwgpath using a chunked fetch helper rel_paths = [rel for (_, rel, _, _) in files_meta] mapping = fetch_drawingfiles_by_paths(rel_paths, chunk_size=500) # Reconstruct all_xml using the created model instances for ext, rel, _, p in files_meta: dwgfile = mapping.get(rel) if dwgfile: all_xml.append((ext, dwgfile, p)) elapsed = time.perf_counter() - start_time print(f" - {len(all_xml)} Drawings files found ({elapsed:.2f}s)") # Process in a deterministic order; ensure .th2 are handled before .th ext_priority = {'th2': 0, 'th': 1} all_xml.sort(key=lambda t: ext_priority.get(t[0], 2)) # Process files and collect modified scalar fields for bulk update modified_map = {} # {DrawingFile instance: set(fields)} for extension, filename, pathobj in all_xml: modified = set() if extension in {"pdf", "txt", "svg", "jpg", "png", ""}: modified = setdrwfileinfo(filename) or set() elif extension == "xml": modified = settnlfileinfo(filename) or set() # important to import .th2 files before .th so that we can assign them when found in .th files elif extension == "th2": modified = settherionfileinfo(("th2", filename)) or set() elif extension == "th": modified = settherionfileinfo(("th", filename)) or set() if modified: modified_map.setdefault(filename, set()).update(modified) elapsed = time.perf_counter() - start_time print(f" - Drawings parsed ({elapsed:.2f}s)") # Bulk update scalar fields grouped by identical field-sets to use bulk_update efficiently from collections import defaultdict groups = defaultdict(list) # {tuple(fields): [instances]} for inst, fields in modified_map.items(): key = tuple(sorted(fields)) groups[key].append(inst) for fields_tuple, instances in groups.items(): fields_list = list(fields_tuple) # Use a conservative batch size DrawingFile.objects.bulk_update(instances, fields_list, batch_size=500) elapsed = time.perf_counter() - start_time print(f" - Database updated ({elapsed:.2f}s)") # for drawfile in DrawingFile.objects.all(): # SetTunnelfileInfo(drawfile)