split surveys->scans + drawings

parsers/imports.py

@@ -11,9 +11,10 @@ from django.db import transaction
 import troggle.settings
 import troggle.parsers.caves
 import troggle.parsers.people
-import troggle.parsers.surveys
+import troggle.parsers.drawings
 import troggle.parsers.logbooks
 import troggle.parsers.QMs
+import troggle.parsers.scans 
 
 '''Master data import.
 Used only by databaseReset.py and online controlpanel.
@@ -33,7 +34,7 @@ def import_people():
 def import_surveyscans():
     print("-- Importing Survey Scans")
     with transaction.atomic():
-        troggle.parsers.surveys.load_all_scans()
+        troggle.parsers.scans.load_all_scans()
 
 def import_logbooks():
     print("-- Importing Logbooks")
@@ -67,6 +68,6 @@ def import_loadpos():
 def import_drawingsfiles():
     print("-- Importing Drawings files")
     with transaction.atomic():
-        troggle.parsers.surveys.load_drawings_files()
+        troggle.parsers.drawings.load_drawings_files()
 
 

parsers/scans.py

@@ -0,0 +1,119 @@
+import sys
+import os
+import types
+import stat
+import csv
+import re
+import datetime
+
+from PIL import Image
+from functools import reduce
+
+import settings
+from troggle.core.models.survex import SingleScan, Wallet, DrawingFile
+from troggle.core.models.troggle import DataIssue
+from troggle.core.utils import save_carefully
+
+'''Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced.
+'''
+
+
+def get_or_create_placeholder(year):
+    """ All surveys must be related to a logbookentry. We don't have a way to
+        automatically figure out which survey went with which logbookentry,
+        so we create a survey placeholder logbook entry for each year. This
+        function always returns such a placeholder, and creates it if it doesn't
+        exist yet.
+    """
+    lookupAttribs={'date__year':int(year),  'title':"placeholder for surveys",}
+    nonLookupAttribs={'text':"surveys temporarily attached to this should be re-attached to their actual trips", 'date':datetime.date(int(year),1,1)}
+    placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs)
+    return placeholder_logbook_entry
+
+def listdir(*directories):
+    try:
+        return os.listdir(os.path.join(settings.SURVEYS, *directories))
+    except:
+        import urllib.request, urllib.parse, urllib.error
+        url = settings.SURVEYS + reduce(lambda x, y: x + "/" + y, ["listdir"] + list(directories))
+        folders = urllib.request.urlopen(url.replace("#", "%23")).readlines()
+        return [folder.rstrip(r"/") for folder in folders]
+
+
+# handles url or file, so we can refer to a set of scans (not drawings) on another server
+def GetListDir(sdir):
+    res = [ ]
+    if sdir[:7] == "http://":
+        # s = urllib.request.urlopen(sdir)
+        message = f"! Requesting loading from http:// NOT IMPLEMENTED. [{sdir}]"         
+        print(message)
+        DataIssue.objects.create(parser='Drawings', message=message)
+        sdir[:7] = ""
+
+    for f in os.listdir(sdir):
+        if f[0] != ".":
+            ff = os.path.join(sdir, f)
+            res.append((f, ff, os.path.isdir(ff)))
+    return res
+
+
+def LoadListScansFile(wallet):
+    gld = [ ]
+    # flatten out any directories in these wallet folders - should not be any
+    for (fyf, ffyf, fisdiryf) in GetListDir(wallet.fpath):
+        if fisdiryf:
+            gld.extend(GetListDir(ffyf))
+        else:
+            gld.append((fyf, ffyf, fisdiryf))
+    
+    c=0
+    for (fyf, ffyf, fisdiryf) in gld:
+        if re.search(r"\.(?:png|jpg|jpeg|pdf|svg|gif)(?i)$", fyf):
+            singlescan = SingleScan(ffile=ffyf, name=fyf, wallet=wallet)
+            singlescan.save()
+            c+=1
+            if c>=10:
+                print(".", end='')
+                c = 0
+
+        
+# this iterates through the scans directories (either here or on the remote server)
+# and builds up the models we can access later
+def load_all_scans():
+
+    print(' - Loading Survey Scans')
+
+    SingleScan.objects.all().delete()
+    Wallet.objects.all().delete()
+    print(' - deleting all scansFolder and scansSingle objects')
+
+    # first do the smkhs (large kh survey scans) directory
+    manywallets_smkhs = Wallet(fpath=os.path.join(settings.SURVEY_SCANS, "../surveys/smkhs"), walletname="smkhs") 
+    print("smkhs", end=' ')
+    if os.path.isdir(manywallets_smkhs.fpath):
+        manywallets_smkhs.save()
+        LoadListScansFile(manywallets_smkhs)
+        
+    
+    # iterate into the surveyscans directory
+    print(' - ', end=' ')
+    for f, ff, fisdir in GetListDir(settings.SURVEY_SCANS):
+        if not fisdir:
+            continue
+        
+        # do the year folders
+        if re.match(r"\d\d\d\d$", f):
+            print("%s" % f, end=' ')
+            for fy, ffy, fisdiry in GetListDir(ff):
+                if fisdiry:
+                    wallet = Wallet(fpath=ffy, walletname=fy)
+                    wallet.save()
+                    LoadListScansFile(wallet)
+        
+        # do the 
+        elif f != "thumbs":
+            wallet = Wallet(fpath=ff, walletname=f)
+            wallet.save()
+            LoadListScansFile(wallet)
+
+    print("", flush=True)

parsers/survex.py

@@ -109,7 +109,7 @@ class LoadingSurvex():
     rx_names   = re.compile(r'(?i)names')
     rx_flagsnot= re.compile(r"not\s")
     rx_linelen = re.compile(r"[\d\-+.]+$")
-    instruments = "(waiting_patiently|slacker|Useless|nagging|unknown|Inst|instrument|rig|rigger|rigging|helper|something| compass|comp|clino|Notes|sketch|book|Tape|Dog|Pics|photo|drawing|Helper|GPS|Disto|Distox|Distox2|topodroid|point|Consultant|nail|polish|varnish|bitch|monkey)"
+    instruments = "(waiting_patiently|slacker|Useless|nagging|unknown|Inst|instrument|rig|rigger|rigging|helper|something| compass|comp|clino|Notes|sketch|book|Tape|Dog|Pics|photo|drawing|Helper|GPS|Disto|Distox|Distox2|topodroid|point|Consultant|nail|polish|varnish|bitch|monkey|PowerDrill|drill)"
     rx_teammem = re.compile(r"(?i)"+instruments+"?(?:es|s)?\s+(.*)"+instruments+"?(?:es|s)?$")
     rx_person  = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$")
     rx_qm      = re.compile(r'(?i)^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$')