split surveys->scans + drawings

core/TESTS/test_imports.py

@@ -42,7 +42,7 @@ class SimpleTest(SimpleTestCase):
         from troggle.parsers.people import GetPersonExpeditionNameLookup
         from troggle.core.views.other import troggle404, frontpage
         from troggle.core.views.caves import ent, cavepage
-        from troggle.core.views import surveys, other, caves, statistics, survex
+        from troggle.core.views import scans, drawings, other, caves, statistics, survex
     def test_import_parsers_QMs(self):
         from troggle.core.models.caves import QM, Cave, LogbookEntry
     def test_import_parsers_people(self):
@@ -61,7 +61,7 @@ class SimpleTest(SimpleTestCase):
         from troggle.core.models.troggle import Expedition
         from troggle.core.models.caves import CaveSlug, Cave, CaveAndEntrance, QM, EntranceSlug, Entrance, Area, SurvexStation
         from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, EntranceForm, EntranceLetterForm
-        from troggle.core.views.login import login_required_if_public
+        from troggle.core.views.auth import login_required_if_public
         from django.contrib.auth.decorators import login_required
         from django.conf import settings
     def test_import_parsers_mix(self):
@@ -70,7 +70,8 @@ class SimpleTest(SimpleTestCase):
         import troggle.logbooksdump
         import troggle.parsers.caves
         import troggle.parsers.people
-        import troggle.parsers.surveys
+        import troggle.parsers.drawings
+        import troggle.parsers.scans
         import troggle.parsers.logbooks
         import troggle.parsers.QMs
         import troggle.parsers.survex 
@@ -90,7 +91,7 @@ class SimpleTest(SimpleTestCase):
         from django.views.generic.base import RedirectView 
         from django.views.generic.edit import UpdateView
         from django.views.generic.list import ListView
-        from troggle.core.views import surveys, other, caves, statistics, survex
+        from troggle.core.views import other, caves, statistics, survex
         from troggle.core.views.auth import expologin, expologout
         from troggle.core.views.caves import ent, cavepage
         from troggle.core.views.expo import expofiles_redirect, expofilessingle, expopage, editexpopage, mediapage, map, mapfile

core/views/drawings.py

@@ -2,51 +2,23 @@ import os, stat
 import re
 from pathlib import Path
 from urllib.parse import urljoin, unquote as urlunquote
-from urllib.request import urlopen
 
 from django.conf import settings
 from django.shortcuts import render
-from django.http import HttpResponse, Http404
+from django.http import HttpResponse
 
-from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock, DrawingFile
+from troggle.core.models.survex import DrawingFile
 from troggle.core.views.expo import getmimetype
-import parsers.surveys
+#import parsers.surveys
 
 '''Some of these views serve files as binary blobs, and simply set the mime type based on the file extension,
 as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked 
 by looking inside the file before being served.
 
-need to check if inavlid query string is invalid, or produces multiple replies
+need to check if invalid query string is invalid, or produces multiple replies
 and render a user-friendly error page.
 '''
 
-def singlewallet(request, path):
-    #print [ s.walletname  for s in Wallet.objects.all() ]
-    try:
-        wallet = Wallet.objects.get(walletname=urlunquote(path))  
-        return render(request, 'wallet.html', { 'wallet':wallet, 'settings': settings })
-    except:
-        message = f'Scan folder error or not found \'{path}\' .'
-        return render(request, 'errors/generic.html', {'message': message})   
-
-def scansingle(request, path, file):
-    '''sends a single binary file to the user for display - browser decides how using mimetype
-    '''
-    try:
-        wallet = Wallet.objects.get(walletname=urlunquote(path)) 
-        singlescan = SingleScan.objects.get(wallet=wallet, name=file)
-        # print(" - scansingle {}:{}:{}:".format(path, file, getmimetype(file)))
-        return HttpResponse(content=open(singlescan.ffile,"rb"), content_type=getmimetype(file)) # any type of image
-    except:
-        message = f'Scan folder or scan item error or not found \'{path}\'  and \'{file}\'.'
-        return render(request, 'errors/generic.html', {'message': message})   
-
-
-def allwallets(request):
-    manywallets = Wallet.objects.all()
-    return render(request, 'manywallets.html', { 'manywallets':manywallets, 'settings': settings })
-    
-    
 def dwgdata(request):
     '''Report on all the drawing files in the system. These were loaded by parsing the entire directory tree
     '''
@@ -124,6 +96,3 @@ def dwgfileupload(request, path):
     message = "File size %d overwritten with size %d" % (orgsize, dwgfile.filesize)
     return HttpResponse(content=message, content_type="text/plain")
 
-    
-    
-

core/views/scans.py

@@ -0,0 +1,47 @@
+import os, stat
+import re
+from pathlib import Path
+from urllib.parse import urljoin, unquote as urlunquote
+from urllib.request import urlopen
+
+from django.conf import settings
+from django.shortcuts import render
+from django.http import HttpResponse
+
+from troggle.core.models.survex import Wallet, SingleScan
+from troggle.core.views.expo import getmimetype
+#import parsers.surveys
+
+'''one of these views serves files as binary blobs, and simply set the mime type based on the file extension,
+as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked 
+by looking inside the file before being served.
+
+need to check if inavlid query string is invalid, or produces multiple replies
+and render a user-friendly error page.
+'''
+
+def singlewallet(request, path):
+    #print [ s.walletname  for s in Wallet.objects.all() ]
+    try:
+        wallet = Wallet.objects.get(walletname=urlunquote(path))  
+        return render(request, 'wallet.html', { 'wallet':wallet, 'settings': settings })
+    except:
+        message = f'Scan folder error or not found \'{path}\' .'
+        return render(request, 'errors/generic.html', {'message': message})   
+
+def scansingle(request, path, file):
+    '''sends a single binary file to the user for display - browser decides how using mimetype
+    '''
+    try:
+        wallet = Wallet.objects.get(walletname=urlunquote(path)) 
+        singlescan = SingleScan.objects.get(wallet=wallet, name=file)
+        # print(" - scansingle {}:{}:{}:".format(path, file, getmimetype(file)))
+        return HttpResponse(content=open(singlescan.ffile,"rb"), content_type=getmimetype(file)) # any type of image
+    except:
+        message = f'Scan folder or scan item error or not found \'{path}\'  and \'{file}\'.'
+        return render(request, 'errors/generic.html', {'message': message})   
+
+
+def allwallets(request):
+    manywallets = Wallet.objects.all()
+    return render(request, 'manywallets.html', { 'manywallets':manywallets, 'settings': settings })

parsers/imports.py

@@ -11,9 +11,10 @@ from django.db import transaction
 import troggle.settings
 import troggle.parsers.caves
 import troggle.parsers.people
-import troggle.parsers.surveys
+import troggle.parsers.drawings
 import troggle.parsers.logbooks
 import troggle.parsers.QMs
+import troggle.parsers.scans 
 
 '''Master data import.
 Used only by databaseReset.py and online controlpanel.
@@ -33,7 +34,7 @@ def import_people():
 def import_surveyscans():
     print("-- Importing Survey Scans")
     with transaction.atomic():
-        troggle.parsers.surveys.load_all_scans()
+        troggle.parsers.scans.load_all_scans()
 
 def import_logbooks():
     print("-- Importing Logbooks")
@@ -67,6 +68,6 @@ def import_loadpos():
 def import_drawingsfiles():
     print("-- Importing Drawings files")
     with transaction.atomic():
-        troggle.parsers.surveys.load_drawings_files()
+        troggle.parsers.drawings.load_drawings_files()
 
 

parsers/scans.py

@@ -0,0 +1,119 @@
+import sys
+import os
+import types
+import stat
+import csv
+import re
+import datetime
+
+from PIL import Image
+from functools import reduce
+
+import settings
+from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
+from troggle.core.models.troggle import DataIssue
+from troggle.core.utils import save_carefully
+
+'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
+'''
+
+
+def get_or_create_placeholder(year):
+    """ All surveys must be related to a logbookentry. We don't have a way to
+        automatically figure out which survey went with which logbookentry,
+        so we create a survey placeholder logbook entry for each year. This
+        function always returns such a placeholder, and creates it if it doesn't
+        exist yet.
+    """
+    lookupAttribs={'date__year':int(year),  'title':"placeholder for surveys",}
+    nonLookupAttribs={'text':"surveys temporarily attached to this should be re-attached to their actual trips", 'date':datetime.date(int(year),1,1)}
+    placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs)
+    return placeholder_logbook_entry
+
+def listdir(*directories):
+    try:
+        return os.listdir(os.path.join(settings.SURVEYS, *directories))
+    except:
+        import urllib.request, urllib.parse, urllib.error
+        url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories))
+        folders = urllib.request.urlopen(url.replace("#", "%23")).readlines()
+        return [folder.rstrip(r"/") for folder in folders]
+
+
+# handles url or file, so we can refer to a set of scans (not drawings) on another server
+def GetListDir(sdir):
+    res = [ ]
+    if sdir[:7] == "http://":
+        # s = urllib.request.urlopen(sdir)
+        message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]"         
+        print(message)
+        DataIssue.objects.create(parser='Drawings', message=message)
+        sdir[:7] = ""
+
+    for f in os.listdir(sdir):
+        if f[0] != ".":
+            ff = os.path.join(sdir, f)
+            res.append((f, ff, os.path.isdir(ff)))
+    return res
+
+
+def LoadListScansFile(wallet):
+    gld = [ ]
+    # flatten out any directories in these wallet folders - should not be any
+    for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath):
+        if fisdiryf:
+            gld.extend(GetListDir(ffyf))
+        else:
+            gld.append((fyf, ffyf, fisdiryf))
+    
+    c=0
+    for (fyf, ffyf, fisdiryf) in gld:
+        if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif)(?i)$", fyf):
+            singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet)
+            singlescan.save()
+            c+=1
+            if c>=10:
+                print(".", end='')
+                c = 0
+
+        
+# this iterates through the scans directories (either here or on the remote server)
+# and builds up the models we can access later
+def load_all_scans():
+
+    print(' - Loading Survey Scans')
+
+    SingleScan.objects.all().delete()
+    Wallet.objects.all().delete()
+    print(' - deleting all scansFolder and scansSingle objects')
+
+    # first do the smkhs (large kh survey scans) directory
+    manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs") 
+    print("smkhs", end=' ')
+    if os.path.isdir(manywallets_smkhs.fpath):
+        manywallets_smkhs.save()
+        LoadListScansFile(manywallets_smkhs)
+        
+    
+    # iterate into the surveyscans directory
+    print(' - ', end=' ')
+    for f, ff, fisdir in GetListDir(settings.SURVEY_SCANS):
+        if not fisdir:
+            continue
+        
+        # do the year folders
+        if re.match(r"\d\d\d\d$", f):
+            print("%s" % f, end=' ')
+            for fy, ffy, fisdiry in GetListDir(ff):
+                if fisdiry:
+                    wallet = Wallet(fpath=ffy, walletname=fy)
+                    wallet.save()
+                    LoadListScansFile(wallet)
+        
+        # do the 
+        elif f != "thumbs":
+            wallet = Wallet(fpath=ff, walletname=f)
+            wallet.save()
+            LoadListScansFile(wallet)
+
+    print("", flush=True)

parsers/survex.py

@@ -109,7 +109,7 @@ class LoadingSurvex():
     rx_names   = re.compile(r'(?i)names')
     rx_flagsnot= re.compile(r"not\s")
     rx_linelen = re.compile(r"[\d\-+.]+$")
-    instruments = "(waiting_patiently|slacker|Useless|nagging|unknown|Inst|instrument|rig|rigger|rigging|helper|something| compass|comp|clino|Notes|sketch|book|Tape|Dog|Pics|photo|drawing|Helper|GPS|Disto|Distox|Distox2|topodroid|point|Consultant|nail|polish|varnish|bitch|monkey)"
+    instruments = "(waiting_patiently|slacker|Useless|nagging|unknown|Inst|instrument|rig|rigger|rigging|helper|something| compass|comp|clino|Notes|sketch|book|Tape|Dog|Pics|photo|drawing|Helper|GPS|Disto|Distox|Distox2|topodroid|point|Consultant|nail|polish|varnish|bitch|monkey|PowerDrill|drill)"
     rx_teammem = re.compile(r"(?i)"+instruments+"?(?:es|s)?\s+(.*)"+instruments+"?(?:es|s)?$")
     rx_person  = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$")
     rx_qm      = re.compile(r'(?i)^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')

urls.py

@@ -9,7 +9,8 @@ from django.contrib import auth
 from django.urls import reverse, resolve
 
 from troggle.core.views import caves, statistics, survex
-from troggle.core.views.surveys import scansingle, singlewallet, allwallets, dwgdata, dwgfilesingle, dwgfileupload
+from troggle.core.views.scans import scansingle, singlewallet, allwallets
+from troggle.core.views.drawings import dwgdata, dwgfilesingle, dwgfileupload
 from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, scanupload 
 from troggle.core.views.other import exportlogbook
 from troggle.core.views.caves import ent, cavepage