forked from expo/troggle
333 lines
15 KiB
Python
333 lines
15 KiB
Python
import os
|
|
import re
|
|
import stat
|
|
from pathlib import Path
|
|
|
|
import settings
|
|
from troggle.core.models.survex import DrawingFile
|
|
from troggle.core.models.troggle import DataIssue
|
|
from troggle.core.models.wallets import Wallet
|
|
|
|
"""Searches through all the :drawings: repository looking
|
|
for tunnel and therion files
|
|
"""
|
|
|
|
todo = """
|
|
- Rename functions more consistently between tunnel and therion variants
|
|
|
|
- Refactor to use pathlib instead of whacky resetting of loop variable inside loop
|
|
to scan sub-folders.
|
|
|
|
- Recode rx_valid_ext to use pathlib suffix() function
|
|
|
|
- Recode load_drawings_files() to use a list of suffices - not the huge if-else monstrosity
|
|
"""
|
|
|
|
rx_valid_ext = re.compile(r"(?i)\.(?:png|jpg|pdf|jpeg|gif|txt)$")
|
|
|
|
|
|
def find_dwg_file(dwgfile, path):
|
|
"""Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file
|
|
which we have already seen when we imported all the files we could find in the surveyscans direstories.
|
|
|
|
The purpose is to find cross-references between Tunnel drawing files. But this is not reported anywhere yet ?
|
|
|
|
What is all this really for ?! Is this data used anywhere ??
|
|
"""
|
|
wallet, scansfile = None, None
|
|
mscansdir = re.search(
|
|
r"(\d\d\d\d#X?\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg|pdf|jpeg|gif|txt))$", path
|
|
)
|
|
if mscansdir:
|
|
scanswalletl = Wallet.objects.filter(walletname=mscansdir.group(1))
|
|
# This should be changed to properly detect if a list of folders is returned and do something sensible, not just pick the first.
|
|
if len(scanswalletl):
|
|
wallet = scanswalletl[0]
|
|
if len(scanswalletl) > 1:
|
|
message = f"! More than one scan FOLDER matches filter query. [{scansfilel[0]}]: {mscansdir.group(1)} {mscansdir.group(2)} {dwgfile.dwgpath} {path}"
|
|
print(message)
|
|
DataIssue.objects.create(parser="Tunnel", message=message)
|
|
|
|
if wallet:
|
|
scansfilel = wallet.singlescan_set.filter(name=mscansdir.group(2))
|
|
if len(scansfilel):
|
|
if len(scansfilel) > 1:
|
|
plist = []
|
|
for sf in scansfilel:
|
|
plist.append(sf.ffile)
|
|
message = f"! More than one image FILENAME matches filter query. [{scansfilel[0]}]: {mscansdir.group(1)} {mscansdir.group(2)} {dwgfile.dwgpath} {path} {plist}"
|
|
print(message)
|
|
DataIssue.objects.create(parser="Tunnel", message=message)
|
|
scansfile = scansfilel[0]
|
|
|
|
if wallet:
|
|
dwgfile.dwgwallets.add(wallet)
|
|
if scansfile:
|
|
dwgfile.scans.add(scansfile)
|
|
|
|
elif path and not rx_valid_ext.search(
|
|
path
|
|
): # ie not recognised as a path where wallets live and not an image file type
|
|
name = os.path.split(path)[1]
|
|
rdwgfilel = DrawingFile.objects.filter(dwgname=name) # Check if it is another drawing file we have already seen
|
|
if len(rdwgfilel):
|
|
if len(rdwgfilel) > 1:
|
|
plist = []
|
|
for df in rdwgfilel:
|
|
plist.append(df.dwgpath)
|
|
message = f"- Warning {len(rdwgfilel)} files named '{name}' {plist}" # should not be a problem?
|
|
print(message)
|
|
DataIssue.objects.create(parser="Tunnel", message=message, url=f"/dwgdataraw/{path}")
|
|
rdwgfile = rdwgfilel[0]
|
|
dwgfile.dwgcontains.add(rdwgfile)
|
|
|
|
dwgfile.save()
|
|
|
|
|
|
def findwalletimage(therionfile, foundpath):
|
|
"""Tries to link the drawing file (Therion format) to the referenced image (scan) file"""
|
|
foundpath = foundpath.strip("{}")
|
|
mscansdir = re.search(r"(\d\d\d\d#\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)", foundpath)
|
|
if mscansdir:
|
|
scanswalletl = Wallet.objects.filter(walletname=mscansdir.group(1))
|
|
# This should be changed to properly detect if a list of folders is returned and do something sensible, not just pick the first.
|
|
if len(scanswalletl):
|
|
wallet = scanswalletl[0]
|
|
if len(scanswalletl) > 1:
|
|
message = "! More than one scan FOLDER matches filter query. [{}]: {} {} {}".format(
|
|
therionfile, mscansdir.group(1), foundpath
|
|
)
|
|
print(message)
|
|
DataIssue.objects.create(parser="Therion", message=message)
|
|
if wallet:
|
|
therionfile.dwgwallets.add(wallet)
|
|
|
|
scanfilename = Path(foundpath).name
|
|
scansfilel = wallet.singlescan_set.filter(name=scanfilename, wallet=wallet)
|
|
if len(scansfilel):
|
|
# message = f'! {len(scansfilel)} {scansfilel} = {scanfilename} found in the wallet specified {wallet.walletname}'
|
|
# print(message)
|
|
if len(scansfilel) > 1:
|
|
plist = []
|
|
for sf in scansfilel:
|
|
plist.append(sf.ffile)
|
|
message = f"! More than one image FILENAME matches filter query. [{scansfilel[0]}]: {mscansdir.group(1)} {mscansdir.group(2)} {dwgfile.dwgpath} {path} {plist}"
|
|
print(message)
|
|
DataIssue.objects.create(parser="Therion", message=message)
|
|
scansfile = scansfilel[0]
|
|
therionfile.scans.add(scansfile)
|
|
else:
|
|
message = f'! Scanned file {scanfilename} mentioned in "{therionfile.dwgpath}" is not actually found in {wallet.walletname}'
|
|
wurl = f"/survey_scans/{wallet.walletname}/".replace("#", ":")
|
|
# print(message)
|
|
DataIssue.objects.create(parser="Therion", message=message, url=wurl)
|
|
|
|
|
|
def findimportinsert(therionfile, imp):
|
|
"""Tries to link the scrap (Therion format) to the referenced therion scrap"""
|
|
pass
|
|
|
|
|
|
rx_xth_me = re.compile(r"xth_me_image_insert.*{.*}$", re.MULTILINE)
|
|
rx_scrap = re.compile(r"^survey (\w*).*$", re.MULTILINE)
|
|
rx_input = re.compile(r"^input (\w*).*$", re.MULTILINE)
|
|
|
|
|
|
def settherionfileinfo(filetuple):
|
|
"""Read in the drawing file contents and sets values on the dwgfile object"""
|
|
thtype, therionfile = filetuple
|
|
|
|
ff = os.path.join(settings.DRAWINGS_DATA, therionfile.dwgpath)
|
|
therionfile.filesize = os.stat(ff)[stat.ST_SIZE]
|
|
if therionfile.filesize <= 0:
|
|
message = f"! Zero length therion file {ff}"
|
|
print(message)
|
|
DataIssue.objects.create(parser="Therion", message=message, url=f"/dwgdataraw/{therionfile.dwgpath}")
|
|
return
|
|
fin = open(ff, "r")
|
|
ttext = fin.read()
|
|
fin.close()
|
|
|
|
# The equivalent for a tunnel 'path' would be a .th2 'line wall' or 'scrap'
|
|
# print(len(re.findall(r"line", ttext)))
|
|
if thtype == "th":
|
|
therionfile.npaths = len(re.findall(r"^input ", ttext, re.MULTILINE))
|
|
elif thtype == "th2":
|
|
therionfile.npaths = len(re.findall(r"^line ", ttext, re.MULTILINE))
|
|
therionfile.save()
|
|
|
|
# scan and look for survex blocks that might have been included, and image scans (as for tunnel drawings)
|
|
# which would populate dwgfile.survexfile
|
|
|
|
# in .th2 files:
|
|
# ##XTHERION## xth_me_image_insert {500 1 1.0} {1700 {}} ../../../expofiles/surveyscans/2014/01popped_elev1.jpeg 0 {}
|
|
# scrap blownout -projection plan -scale [-81.0 -42.0 216.0 -42.0 0.0 0.0 7.5438 0.0 m]
|
|
|
|
for xth_me in rx_xth_me.findall(ttext):
|
|
# WORK IN PROGRESS. Do not clutter up the DataIssues list with this
|
|
message = f"! Un-parsed image filename: {therionfile.dwgname} : {xth_me.split()[-3]} - {therionfile.dwgpath}"
|
|
# print(message)
|
|
# DataIssue.objects.create(parser='xTherion', message=message, url=f'/dwgdataraw/{therionfile.dwgpath}')
|
|
# ! Un-parsed image filename: 107coldest : ../../../expofiles/surveyscans/2015/2015#20/notes.jpg - therion/plan/107coldest.th2
|
|
|
|
with open("therionrefs.log", "a") as lg:
|
|
lg.write(message + "\n")
|
|
|
|
findwalletimage(therionfile, xth_me.split()[-3])
|
|
|
|
for inp in rx_input.findall(ttext):
|
|
# if this 'input' is a .th2 file we have already seen, then we can assign this as a sub-file
|
|
# but we would need to disentangle to get the current path properly
|
|
message = f"! Un-set (?) Therion .th2 input: - {therionfile.dwgname} : {inp} - {therionfile.dwgpath}"
|
|
# print(message)
|
|
DataIssue.objects.create(parser="xTherion", message=message, url=f"/dwgdataraw/{therionfile.dwgpath}")
|
|
findimportinsert(therionfile, inp)
|
|
|
|
therionfile.save()
|
|
|
|
|
|
rx_skpath = re.compile(rb"<skpath")
|
|
rx_pcpath = re.compile(rb'<pcarea area_signal="frame".*?sfsketch="([^"]*)" sfstyle="([^"]*)"')
|
|
|
|
|
|
def settnlfileinfo(dwgfile):
|
|
"""Read in the drawing file contents and sets values on the dwgfile object
|
|
Should try to read the date too e.g. tunneldate="2010-08-16 22:51:57
|
|
then we could display on the master calendar per expo.
|
|
"""
|
|
ff = os.path.join(settings.DRAWINGS_DATA, dwgfile.dwgpath)
|
|
dwgfile.filesize = os.stat(ff)[stat.ST_SIZE]
|
|
if dwgfile.filesize <= 0:
|
|
message = f"! Zero length tunnel file {ff}"
|
|
print(message)
|
|
DataIssue.objects.create(parser="Tunnel", message=message, url=f"/dwgdataraw/{dwgfile.dwgpath}")
|
|
return
|
|
fin = open(ff, "rb")
|
|
ttext = fin.read()
|
|
fin.close()
|
|
|
|
dwgfile.npaths = len(rx_skpath.findall(ttext))
|
|
dwgfile.save()
|
|
|
|
# example drawing file in Tunnel format.
|
|
# <tunnelxml tunnelversion="version2009-06-21 Matienzo" tunnelproject="ireby" tunneluser="goatchurch" tunneldate="2009-06-29 23:22:17">
|
|
# <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0">
|
|
|
|
for path, style in rx_pcpath.findall(ttext):
|
|
find_dwg_file(dwgfile, path.decode())
|
|
|
|
# should also scan and look for survex blocks that might have been included, and image scans
|
|
# which would populate dwgfile.survexfile
|
|
|
|
dwgfile.save()
|
|
|
|
|
|
def setdrwfileinfo(dwgfile):
|
|
"""Read in the drawing file contents and sets values on the dwgfile object,
|
|
but these are SVGs, PDFs or .txt files, so there is no useful format to search for
|
|
This function is a placeholder in case we thnk of a way to do something
|
|
to recognise generic survex filenames.
|
|
"""
|
|
ff = Path(settings.DRAWINGS_DATA) / dwgfile.dwgpath
|
|
dwgfile.filesize = ff.stat().st_size
|
|
if dwgfile.filesize <= 0:
|
|
message = f"! Zero length drawing file {ff}"
|
|
print(message)
|
|
DataIssue.objects.create(parser="drawings", message=message, url=f"/dwgdataraw/{dwgfile.dwgpath}")
|
|
return
|
|
|
|
|
|
def load_drawings_files():
|
|
"""Breadth first search of drawings directory looking for sub-directories and *.xml filesize
|
|
This is brain-damaged very early code. Should be replaced with proper use of pathlib.
|
|
|
|
Why do we have all this detection of file types/! Why not use get_mime_types ?
|
|
What is it all for ??
|
|
|
|
We import JPG, PNG and SVG files; which have already been put on the server,
|
|
but the upload form intentionally refuses to upload PNG and JPG (though it does allow SVG)
|
|
"""
|
|
all_xml = []
|
|
drawdatadir = settings.DRAWINGS_DATA
|
|
DrawingFile.objects.all().delete()
|
|
DataIssue.objects.filter(parser="drawings").delete()
|
|
DataIssue.objects.filter(parser="Therion").delete()
|
|
DataIssue.objects.filter(parser="xTherion").delete()
|
|
DataIssue.objects.filter(parser="Tunnel").delete()
|
|
if os.path.isfile("therionrefs.log"):
|
|
os.remove("therionrefs.log")
|
|
|
|
drawingsdirs = [""]
|
|
while drawingsdirs:
|
|
drawdir = drawingsdirs.pop()
|
|
for f in os.listdir(os.path.join(drawdatadir, drawdir)):
|
|
if f[0] == "." or f[-1] == "~":
|
|
continue
|
|
lf = os.path.join(drawdir, f)
|
|
ff = os.path.join(drawdatadir, lf)
|
|
if os.path.isdir(ff):
|
|
drawingsdirs.append(
|
|
lf
|
|
) # lunatic! adding to list in middle of list while loop! Replace with pathlib functions.
|
|
elif Path(f).suffix.lower() == ".txt":
|
|
# Always creates new
|
|
dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1])
|
|
dwgfile.save()
|
|
all_xml.append(("txt", dwgfile))
|
|
elif Path(f).suffix.lower() == ".xml":
|
|
# Always creates new
|
|
dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1])
|
|
dwgfile.save()
|
|
all_xml.append(("xml", dwgfile))
|
|
elif Path(f).suffix.lower() == ".th":
|
|
# Always creates new
|
|
dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1])
|
|
dwgfile.save()
|
|
all_xml.append(("th", dwgfile))
|
|
elif Path(f).suffix.lower() == ".th2":
|
|
# Always creates new
|
|
dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1])
|
|
dwgfile.save()
|
|
all_xml.append(("th2", dwgfile))
|
|
elif Path(f).suffix.lower() == ".pdf":
|
|
# Always creates new
|
|
dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1])
|
|
dwgfile.save()
|
|
all_xml.append(("pdf", dwgfile))
|
|
elif Path(f).suffix.lower() == ".png":
|
|
# Always creates new
|
|
dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1])
|
|
dwgfile.save()
|
|
all_xml.append(("png", dwgfile))
|
|
elif Path(f).suffix.lower() == ".svg":
|
|
# Always creates new
|
|
dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1])
|
|
dwgfile.save()
|
|
all_xml.append(("svg", dwgfile))
|
|
elif Path(f).suffix.lower() == ".jpg":
|
|
# Always creates new
|
|
dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f[:-4])[1])
|
|
dwgfile.save()
|
|
all_xml.append(("jpg", dwgfile))
|
|
elif Path(f).suffix == "":
|
|
# therion file
|
|
dwgfile = DrawingFile(dwgpath=lf, dwgname=os.path.split(f)[1])
|
|
dwgfile.save()
|
|
all_xml.append(("", dwgfile))
|
|
|
|
print(f" - {len(all_xml)} Drawings files found")
|
|
|
|
for d in all_xml:
|
|
if d[0] in ["pdf", "txt", "svg", "jpg", "png", ""]:
|
|
setdrwfileinfo(d[1])
|
|
if d[0] == "xml":
|
|
settnlfileinfo(d[1])
|
|
# important to import .th2 files before .th so that we can assign them when found in .th files
|
|
if d[0] == "th2":
|
|
settherionfileinfo(d)
|
|
if d[0] == "th":
|
|
settherionfileinfo(d)
|
|
|
|
# for drawfile in DrawingFile.objects.all():
|
|
# SetTunnelfileInfo(drawfile)
|