moved functions between files

core/utils.py

@@ -63,6 +63,23 @@ def chaosmonkey(n):
     # print("CHAOS strikes !", file=sys.stderr)
     return True
 
+# handles url or file, so we can refer to a set of scans (not drawings) on another server
+def GetListDir(sdir):
+    res = [ ]
+    if sdir[:7] == "http://":
+        # s = urllib.request.urlopen(sdir)
+        message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]"         
+        print(message)
+        DataIssue.objects.create(parser='Drawings', message=message)
+        sdir[:7] = ""
+
+    for f in os.listdir(sdir):
+        if f[0] != ".":
+            ff = os.path.join(sdir, f)
+            res.append((f, ff, os.path.isdir(ff)))
+    return res
+
+
 
 def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
     """Looks up instance using lookupAttribs and carries out the following:

core/views/drawings.py

@@ -15,10 +15,16 @@ from troggle.core.views.expo import getmimetype
 as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked 
 by looking inside the file before being served.
 
+
+'''
+
+todo='''The git stuff on upload!
+
 need to check if invalid query string is invalid, or produces multiple replies
 and render a user-friendly error page.
 '''
 
+
 def dwgallfiles(request):
     '''Report on all the drawing files in the system. These were loaded by parsing the entire directory tree
     '''
@@ -50,49 +56,49 @@ def dwgfilesingle(request, path):
         return HttpResponse(content="Unable to understand the encoding for this file: not UTF-8 nor iso-8859-1, or some other read error happened.") 
     
     
-def dwgfileupload(request, path):
+# def dwgfileupload(request, path):
-    '''Use bits of this to REGISTEr a recently uploaded dwg file which used dwgupload
+    # '''Use bits of this to REGISTEr a recently uploaded dwg file which used dwgupload
-    '''
+    # '''
-    try:
+    # try:
-        dwgfile = DrawingFile.objects.get(dwgpath=urlunquote(path)) # need to check if inavlid query string and produce friendly error
+        # dwgfile = DrawingFile.objects.get(dwgpath=urlunquote(path)) # need to check if inavlid query string and produce friendly error
-    except:
+    # except:
-        message = f'Drawing file error or not found \'{path}\' .'
+        # message = f'Drawing file error or not found \'{path}\' .'
-        return render(request, 'errors/generic.html', {'message': message})   
+        # return render(request, 'errors/generic.html', {'message': message})   
-    tfile = Path(settings.DRAWINGS_DATA, dwgfile.dwgpath)
+    # tfile = Path(settings.DRAWINGS_DATA, dwgfile.dwgpath)
     
-    project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"]
+    # project, user, password, tunnelversion = request.POST["tunnelproject"], request.POST["tunneluser"], request.POST["tunnelpassword"], request.POST["tunnelversion"]
-    print(project, user, tunnelversion)
+    # print(project, user, tunnelversion)
     
     
-    if not (len(list(request.FILES.values())) == 1): # "only one file to upload"
+    # if not (len(list(request.FILES.values())) == 1): # "only one file to upload"
-         return HttpResponse(content="Error: more than one file selected for upload", content_type="text/plain")
+         # return HttpResponse(content="Error: more than one file selected for upload", content_type="text/plain")
        
-    uploadedfile = list(request.FILES.values())[0]
+    # uploadedfile = list(request.FILES.values())[0]
         
-    if uploadedfile.field_name != "sketch":
+    # if uploadedfile.field_name != "sketch":
-        return HttpResponse(content="Error: non-sketch file uploaded", content_type="text/plain")
+        # return HttpResponse(content="Error: non-sketch file uploaded", content_type="text/plain")
-    if uploadedfile.content_type != "text/plain":
+    # if uploadedfile.content_type != "text/plain":
-        return HttpResponse(content="Error: non-plain content type", content_type="text/plain")
+        # return HttpResponse(content="Error: non-plain content type", content_type="text/plain")
         
-    # could use this to add new files
+    # # could use this to add new files
-    if os.path.split(path)[1] != uploadedfile.name:  
+    # if os.path.split(path)[1] != uploadedfile.name:  
-        return HttpResponse(content="Error: name disagrees", content_type="text/plain")
+        # return HttpResponse(content="Error: name disagrees", content_type="text/plain")
     
-    orgsize = dwgfile.filesize   # = os.stat(tfile)[stat.ST_SIZE]
+    # orgsize = dwgfile.filesize   # = os.stat(tfile)[stat.ST_SIZE]
         
-    ttext = uploadedfile.read()
+    # ttext = uploadedfile.read()
     
-    # could check that the user and projects agree here
+    # # could check that the user and projects agree here
     
-    fout = open(tfile, "w")
+    # fout = open(tfile, "w")
-    fout.write(ttext)
+    # fout.write(ttext)
-    fout.close()
+    # fout.close()
     
-    # redo its settings of 
+    # # redo its settings of 
-    parsers.surveys.SetTunnelfileInfo(dwgfile) # commented out
+    # parsers.surveys.SetTunnelfileInfo(dwgfile) # commented out
-    dwgfile.save()
+    # dwgfile.save()
     
-    uploadedfile.close()
+    # uploadedfile.close()
-    message = "File size %d overwritten with size %d" % (orgsize, dwgfile.filesize)
+    # message = "File size %d overwritten with size %d" % (orgsize, dwgfile.filesize)
-    return HttpResponse(content=message, content_type="text/plain")
+    # return HttpResponse(content=message, content_type="text/plain")
 

parsers/drawings.py

@@ -13,14 +13,14 @@ from functools import reduce
 import settings
 from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
 from troggle.core.models.troggle import DataIssue
-from troggle.core.utils import save_carefully
+from troggle.core.utils import save_carefully, GetListDir
 
 '''Searches through all the :drawings: repository looking
 for tunnel and therion files
-    
-Searches through all the survey scans directories in expofiles, looking for images to be referenced.
 '''
 
+todo='''
+'''
 
 def get_or_create_placeholder(year):
     """ All surveys must be related to a logbookentry. We don't have a way to
@@ -34,94 +34,6 @@ def get_or_create_placeholder(year):
     placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs)
     return placeholder_logbook_entry
 
-def listdir(*directories):
-    try:
-        return os.listdir(os.path.join(settings.SURVEYS, *directories))
-    except:
-        import urllib.request, urllib.parse, urllib.error
-        url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories))
-        folders = urllib.request.urlopen(url.replace("#", "%23")).readlines()
-        return [folder.rstrip(r"/") for folder in folders]
-
-
-# handles url or file, so we can refer to a set of scans (not drawings) on another server
-def GetListDir(sdir):
-    res = [ ]
-    if sdir[:7] == "http://":
-        # s = urllib.request.urlopen(sdir)
-        message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]"         
-        print(message)
-        DataIssue.objects.create(parser='Drawings', message=message)
-        sdir[:7] = ""
-
-    for f in os.listdir(sdir):
-        if f[0] != ".":
-            ff = os.path.join(sdir, f)
-            res.append((f, ff, os.path.isdir(ff)))
-    return res
-
-
-def LoadListScansFile(wallet):
-    gld = [ ]
-    # flatten out any directories in these wallet folders - should not be any
-    for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath):
-        if fisdiryf:
-            gld.extend(GetListDir(ffyf))
-        else:
-            gld.append((fyf, ffyf, fisdiryf))
-    
-    c=0
-    for (fyf, ffyf, fisdiryf) in gld:
-        if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif)(?i)$", fyf):
-            singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet)
-            singlescan.save()
-            c+=1
-            if c>=10:
-                print(".", end='')
-                c = 0
-
-        
-# this iterates through the scans directories (either here or on the remote server)
-# and builds up the models we can access later
-def load_all_scans():
-
-    print(' - Loading Survey Scans')
-
-    SingleScan.objects.all().delete()
-    Wallet.objects.all().delete()
-    print(' - deleting all scansFolder and scansSingle objects')
-
-    # first do the smkhs (large kh survey scans) directory
-    manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs") 
-    print("smkhs", end=' ')
-    if os.path.isdir(manywallets_smkhs.fpath):
-        manywallets_smkhs.save()
-        LoadListScansFile(manywallets_smkhs)
-        
-    
-    # iterate into the surveyscans directory
-    print(' - ', end=' ')
-    for f, ff, fisdir in GetListDir(settings.SURVEY_SCANS):
-        if not fisdir:
-            continue
-        
-        # do the year folders
-        if re.match(r"\d\d\d\d$", f):
-            print("%s" % f, end=' ')
-            for fy, ffy, fisdiry in GetListDir(ff):
-                if fisdiry:
-                    wallet = Wallet(fpath=ffy, walletname=fy)
-                    wallet.save()
-                    LoadListScansFile(wallet)
-        
-        # do the 
-        elif f != "thumbs":
-            wallet = Wallet(fpath=ff, walletname=f)
-            wallet.save()
-            LoadListScansFile(wallet)
-
-    print("", flush=True)
-
 def find_tunnel_file(dwgfile, path):
     '''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file
     which we have already seen when we imported all the files we could find in the surveyscans direstories

parsers/scans.py

@@ -12,7 +12,7 @@ from functools import reduce
 import settings
 from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
 from troggle.core.models.troggle import DataIssue
-from troggle.core.utils import save_carefully
+from troggle.core.utils import save_carefully, GetListDir
 
 '''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
 '''
@@ -39,24 +39,6 @@ def listdir(*directories):
         folders = urllib.request.urlopen(url.replace("#", "%23")).readlines()
         return [folder.rstrip(r"/") for folder in folders]
 
-
-# handles url or file, so we can refer to a set of scans (not drawings) on another server
-def GetListDir(sdir):
-    res = [ ]
-    if sdir[:7] == "http://":
-        # s = urllib.request.urlopen(sdir)
-        message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]"         
-        print(message)
-        DataIssue.objects.create(parser='Drawings', message=message)
-        sdir[:7] = ""
-
-    for f in os.listdir(sdir):
-        if f[0] != ".":
-            ff = os.path.join(sdir, f)
-            res.append((f, ff, os.path.isdir(ff)))
-    return res
-
-
 def LoadListScansFile(wallet):
     gld = [ ]
     # flatten out any directories in these wallet folders - should not be any