From 1638f97d0c785832ad9c6456f109971d5171c5e7 Mon Sep 17 00:00:00 2001
From: Philip Sargent <philip.sargent@klebos.com>
Date: Tue, 4 May 2021 20:57:16 +0100
Subject: [PATCH] moved functions between files
---
core/utils.py | 17 ++++++++
core/views/drawings.py | 72 +++++++++++++++++---------------
parsers/drawings.py | 94 ++----------------------------------------
parsers/scans.py | 20 +--------
4 files changed, 60 insertions(+), 143 deletions(-)
diff --git a/core/utils.py b/core/utils.py
index ff5f35c..fe5bd07 100644
--- a/core/utils.py
+++ b/core/utils.py
@@ -63,6 +63,23 @@ def chaosmonkey(n):
# print("CHAOS strikes !", file=sys.stderr)
return True
+# handles url or file, so we can refer to a set of scans (not drawings) on another server
+def GetListDir(sdir):
+ res = [ ]
+ if sdir[:7] == "http://":
+ # s = urllib.request.urlopen(sdir)
+ message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]"
+ print(message)
+ DataIssue.objects.create(parser='Drawings', message=message)
+ sdir[:7] = ""
+
+ for f in os.listdir(sdir):
+ if f[0] != ".":
+ ff = os.path.join(sdir, f)
+ res.append((f, ff, os.path.isdir(ff)))
+ return res
+
+
def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
"""Looks up instance using lookupAttribs and carries out the following:
diff --git a/core/views/drawings.py b/core/views/drawings.py
index f0b684a..9b08dd5 100644
--- a/core/views/drawings.py
+++ b/core/views/drawings.py
@@ -15,10 +15,16 @@ from troggle.core.views.expo import getmimetype
as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked
by looking inside the file before being served.
+
+'''
+
+todo='''The git stuff on upload!
+
need to check if invalid query string is invalid, or produces multiple replies
and render a user-friendly error page.
'''
+
def dwgallfiles(request):
'''Report on all the drawing files in the system. These were loaded by parsing the entire directory tree
'''
@@ -50,49 +56,49 @@ def dwgfilesingle(request, path):
return HttpResponse(content="Unable to understand the encoding for this file: not UTF-8 nor iso-8859-1, or some other read error happened.")
-def dwgfileupload(request, path):
- '''Use bits of this to REGISTEr a recently uploaded dwg file which used dwgupload
- '''
- try:
- dwgfile = DrawingFile.objects.get(dwgpath=urlunquote(path)) # need to check if inavlid query string and produce friendly error
- except:
- message = f'Drawing file error or not found \'{path}\' .'
- return render(request, 'errors/generic.html', {'message': message})
- tfile = Path(settings.DRAWINGS_DATA, dwgfile.dwgpath)
+# def dwgfileupload(request, path):
+ # '''Use bits of this to REGISTEr a recently uploaded dwg file which used dwgupload
+ # '''
+ # try:
+ # dwgfile = DrawingFile.objects.get(dwgpath=urlunquote(path)) # need to check if inavlid query string and produce friendly error
+ # except:
+ # message = f'Drawing file error or not found \'{path}\' .'
+ # return render(request, 'errors/generic.html', {'message': message})
+ # tfile = Path(settings.DRAWINGS_DATA, dwgfile.dwgpath)
- project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"]
- print(project, user, tunnelversion)
+ # project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"]
+ # print(project, user, tunnelversion)
- if not (len(list(request.FILES.values())) == 1): # "only one file to upload"
- return HttpResponse(content="Error: more than one file selected for upload", content_type="text/plain")
+ # if not (len(list(request.FILES.values())) == 1): # "only one file to upload"
+ # return HttpResponse(content="Error: more than one file selected for upload", content_type="text/plain")
- uploadedfile = list(request.FILES.values())[0]
+ # uploadedfile = list(request.FILES.values())[0]
- if uploadedfile.field_name != "sketch":
- return HttpResponse(content="Error: non-sketch file uploaded", content_type="text/plain")
- if uploadedfile.content_type != "text/plain":
- return HttpResponse(content="Error: non-plain content type", content_type="text/plain")
+ # if uploadedfile.field_name != "sketch":
+ # return HttpResponse(content="Error: non-sketch file uploaded", content_type="text/plain")
+ # if uploadedfile.content_type != "text/plain":
+ # return HttpResponse(content="Error: non-plain content type", content_type="text/plain")
- # could use this to add new files
- if os.path.split(path)[1] != uploadedfile.name:
- return HttpResponse(content="Error: name disagrees", content_type="text/plain")
+ # # could use this to add new files
+ # if os.path.split(path)[1] != uploadedfile.name:
+ # return HttpResponse(content="Error: name disagrees", content_type="text/plain")
- orgsize = dwgfile.filesize # = os.stat(tfile)[stat.ST_SIZE]
+ # orgsize = dwgfile.filesize # = os.stat(tfile)[stat.ST_SIZE]
- ttext = uploadedfile.read()
+ # ttext = uploadedfile.read()
- # could check that the user and projects agree here
+ # # could check that the user and projects agree here
- fout = open(tfile, "w")
- fout.write(ttext)
- fout.close()
+ # fout = open(tfile, "w")
+ # fout.write(ttext)
+ # fout.close()
- # redo its settings of
- parsers.surveys.SetTunnelfileInfo(dwgfile) # commented out
- dwgfile.save()
+ # # redo its settings of
+ # parsers.surveys.SetTunnelfileInfo(dwgfile) # commented out
+ # dwgfile.save()
- uploadedfile.close()
- message = "File size %d overwritten with size %d" % (orgsize, dwgfile.filesize)
- return HttpResponse(content=message, content_type="text/plain")
+ # uploadedfile.close()
+ # message = "File size %d overwritten with size %d" % (orgsize, dwgfile.filesize)
+ # return HttpResponse(content=message, content_type="text/plain")
diff --git a/parsers/drawings.py b/parsers/drawings.py
index a313a83..f3aca47 100644
--- a/parsers/drawings.py
+++ b/parsers/drawings.py
@@ -13,14 +13,14 @@ from functools import reduce
import settings
from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
from troggle.core.models.troggle import DataIssue
-from troggle.core.utils import save_carefully
+from troggle.core.utils import save_carefully, GetListDir
'''Searches through all the :drawings: repository looking
for tunnel and therion files
-
-Searches through all the survey scans directories in expofiles, looking for images to be referenced.
'''
+todo='''
+'''
def get_or_create_placeholder(year):
""" All surveys must be related to a logbookentry. We don't have a way to
@@ -34,94 +34,6 @@ def get_or_create_placeholder(year):
placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs)
return placeholder_logbook_entry
-def listdir(*directories):
- try:
- return os.listdir(os.path.join(settings.SURVEYS, *directories))
- except:
- import urllib.request, urllib.parse, urllib.error
- url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories))
- folders = urllib.request.urlopen(url.replace("#", "%23")).readlines()
- return [folder.rstrip(r"/") for folder in folders]
-
-
-# handles url or file, so we can refer to a set of scans (not drawings) on another server
-def GetListDir(sdir):
- res = [ ]
- if sdir[:7] == "http://":
- # s = urllib.request.urlopen(sdir)
- message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]"
- print(message)
- DataIssue.objects.create(parser='Drawings', message=message)
- sdir[:7] = ""
-
- for f in os.listdir(sdir):
- if f[0] != ".":
- ff = os.path.join(sdir, f)
- res.append((f, ff, os.path.isdir(ff)))
- return res
-
-
-def LoadListScansFile(wallet):
- gld = [ ]
- # flatten out any directories in these wallet folders - should not be any
- for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath):
- if fisdiryf:
- gld.extend(GetListDir(ffyf))
- else:
- gld.append((fyf, ffyf, fisdiryf))
-
- c=0
- for (fyf, ffyf, fisdiryf) in gld:
- if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif)(?i)$", fyf):
- singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet)
- singlescan.save()
- c+=1
- if c>=10:
- print(".", end='')
- c = 0
-
-
-# this iterates through the scans directories (either here or on the remote server)
-# and builds up the models we can access later
-def load_all_scans():
-
- print(' - Loading Survey Scans')
-
- SingleScan.objects.all().delete()
- Wallet.objects.all().delete()
- print(' - deleting all scansFolder and scansSingle objects')
-
- # first do the smkhs (large kh survey scans) directory
- manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs")
- print("smkhs", end=' ')
- if os.path.isdir(manywallets_smkhs.fpath):
- manywallets_smkhs.save()
- LoadListScansFile(manywallets_smkhs)
-
-
- # iterate into the surveyscans directory
- print(' - ', end=' ')
- for f, ff, fisdir in GetListDir(settings.SURVEY_SCANS):
- if not fisdir:
- continue
-
- # do the year folders
- if re.match(r"\d\d\d\d$", f):
- print("%s" % f, end=' ')
- for fy, ffy, fisdiry in GetListDir(ff):
- if fisdiry:
- wallet = Wallet(fpath=ffy, walletname=fy)
- wallet.save()
- LoadListScansFile(wallet)
-
- # do the
- elif f != "thumbs":
- wallet = Wallet(fpath=ff, walletname=f)
- wallet.save()
- LoadListScansFile(wallet)
-
- print("", flush=True)
-
def find_tunnel_file(dwgfile, path):
'''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file
which we have already seen when we imported all the files we could find in the surveyscans direstories
diff --git a/parsers/scans.py b/parsers/scans.py
index e43ddc4..9c0d28a 100644
--- a/parsers/scans.py
+++ b/parsers/scans.py
@@ -12,7 +12,7 @@ from functools import reduce
import settings
from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
from troggle.core.models.troggle import DataIssue
-from troggle.core.utils import save_carefully
+from troggle.core.utils import save_carefully, GetListDir
'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
'''
@@ -39,24 +39,6 @@ def listdir(*directories):
folders = urllib.request.urlopen(url.replace("#", "%23")).readlines()
return [folder.rstrip(r"/") for folder in folders]
-
-# handles url or file, so we can refer to a set of scans (not drawings) on another server
-def GetListDir(sdir):
- res = [ ]
- if sdir[:7] == "http://":
- # s = urllib.request.urlopen(sdir)
- message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]"
- print(message)
- DataIssue.objects.create(parser='Drawings', message=message)
- sdir[:7] = ""
-
- for f in os.listdir(sdir):
- if f[0] != ".":
- ff = os.path.join(sdir, f)
- res.append((f, ff, os.path.isdir(ff)))
- return res
-
-
def LoadListScansFile(wallet):
gld = [ ]
# flatten out any directories in these wallet folders - should not be any