dwg upload and django admin extra search

core/TESTS/test_imports.py

@@ -40,9 +40,11 @@ class SimpleTest(SimpleTestCase):
         import troggle.core.models.survex as models_survex
         import troggle.core.models.caves as models_caves
         from troggle.parsers.people import GetPersonExpeditionNameLookup
-        from troggle.core.views.other import troggle404, frontpage
+        from troggle.core.views.other import frontpage
         from troggle.core.views.caves import ent, cavepage
-        from troggle.core.views import scans, drawings, other, caves, statistics, survex
+        from troggle.core.views import scans, drawings, other, caves, statistics, survex, uploads
+    def test_import_views_uploads(self):
+        from troggle.core.views.uploads import dwgupload, scanupload
     def test_import_parsers_QMs(self):
         from troggle.core.models.caves import QM, Cave, LogbookEntry
     def test_import_parsers_people(self):
@@ -97,7 +99,7 @@ class SimpleTest(SimpleTestCase):
         from troggle.core.views.logbooks import expedition, personexpedition, Expeditions_tsvListView, Expeditions_jsonListView
         from troggle.core.views.logbooks import get_logbook_entries, logbookentry, logbookSearch
         from troggle.core.views.logbooks import notablepersons, person, get_people
-        from troggle.core.views.other import troggle404, frontpage
+        from troggle.core.views.other import controlpanel
         from troggle.core.views.prospect import prospecting
         from troggle.core.views.prospect import prospecting_image
         from troggle.core.views.statistics import pathsreport, stats, dataissues

core/TESTS/tests_logins.py

@@ -86,20 +86,49 @@ class PostTests(TestCase):
         self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE')
         logged_in = c.login(username=u.username, password='secretword') 
 
-        with open('README.txt','r') as testf:
+        with open('core/fixtures/test_upload_file.txt','r') as testf:
-            response = self.client.post('/scanupload/2021:00', data={'title': '2021#00', 'name': 'README.txt', 'scanfiles': testf })
+            response = self.client.post('/scanupload/2020:00', data={'name': 'test_upload_file.txt', 'uploadfiles': testf })
             content = response.content.decode()
             self.assertEqual(response.status_code, 200)
             self.assertEqual(response.status_code, HTTPStatus.OK)
-            # with open('test_up.html', 'w') as f: 
+            # with open('testresponse.html', 'w') as f: 
                 # f.write(content) 
-            t    = re.search('<em>README', content)
+            for ph in [ r'test_upload_',
-            self.assertIsNotNone(t, 'Logged in but failed to see "<em>\'README"' )
+                    r'&larr; 2020#00 &rarr;',
-            t    = re.search(' saved as', content)
+                    r'Upload more?']:
-            self.assertIsNotNone(t, 'Logged in but failed to see "File(s) saved as"' )
+                phmatch    = re.search(ph, content)
-            t    = re.search('/expofiles/surveyscans/2021/2021%2300/README', content)
+                self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")
-            self.assertIsNotNone(t, 'Logged in but failed to see "/expofiles/..."' )
 
+    def test_dwg_upload(self):
+        '''Test file upload. Need to login first.
+        First upload is refused as it is a TXT file
+        Second upload is an image and suceeds.
+        '''
+        c = self.client
+        from django.contrib.auth.models import User
+        u = User.objects.get(username='expotest')
+         
+        self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE')
+        logged_in = c.login(username=u.username, password='secretword') 
+
+        with open('core/fixtures/test_upload_file.txt','r') as testf:
+            response = self.client.post('/dwgupload/uploads', data={'name': 'test_upload_file.txt', 'uploadfiles': testf })
+            content = response.content.decode()
+            self.assertEqual(response.status_code, 200)
+            t    = re.search('Files refused:', content)
+            self.assertIsNotNone(t, 'Logged in but failed to see "Files refused:"' )
+
+        with open('core/fixtures/test_upload_nosuffix','r') as testf:
+            response = self.client.post('/dwgupload/uploads', data={'name': 'test_upload_nosuffix', 'uploadfiles': testf })
+            content = response.content.decode()
+            with open('testresponse.html', 'w') as f: 
+                f.write(content) 
+            self.assertEqual(response.status_code, 200)
+            for ph in [ r'Upload more',
+                    r' saved as ',
+                    r'Clicking on a filename only']:
+                phmatch    = re.search(ph, content)
+                self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")
 
 
 class ComplexLoginTests(TestCase):

core/admin.py

@@ -7,7 +7,8 @@ from django.core import serializers
 from troggle.core.views.other import exportlogbook
 from troggle.core.models.troggle import Person, PersonExpedition, Expedition, DataIssue
 from troggle.core.models.caves import Cave, Area, Entrance, CaveAndEntrance, LogbookEntry, PersonTrip, QM
-from troggle.core.models.survex import SurvexBlock, SurvexPersonRole, SurvexStation, Wallet, SingleScan
+from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole, SurvexStation, SurvexDirectory
+from troggle.core.models.survex import Wallet, SingleScan, DrawingFile
 
 '''This code significantly adds to the capabilities of the Django Management control panel for Troggle data.
 In particular, it enables JSON export of any data with 'export_as_json'
@@ -36,9 +37,9 @@ class SurvexBlockAdmin(TroggleModelAdmin):
     inlines = (RoleInline,)
 
 
-class SurveyAdmin(TroggleModelAdmin):
+# class SurveyAdmin(TroggleModelAdmin):
-    #inlines = (ScannedImageInline,)
+    # #inlines = (ScannedImageInline,)
-    search_fields = ('expedition__year','wallet_number')    
+    # search_fields = ('expedition__year','wallet_number')    
 
 
 class QMsFoundInline(admin.TabularInline):
@@ -105,20 +106,38 @@ class CaveAdmin(TroggleModelAdmin):
 class EntranceAdmin(TroggleModelAdmin):
     search_fields = ('caveandentrance__cave__kataster_number',)
 
+class SurvexStationAdmin(TroggleModelAdmin):
+    search_fields = ('name',)
+
+class SurvexFileAdmin(TroggleModelAdmin):
+    search_fields = ('path',)
+
+class SurvexDirectoryAdmin(TroggleModelAdmin):
+    search_fields = ('path', 'survexdirectory',)
+
+class DrawingFileAdmin(TroggleModelAdmin):
+    search_fields = ('dwgname',)
+
+class WalletAdmin(TroggleModelAdmin):
+    search_fields = ('fpath',)
+
 
 admin.site.register(Cave, CaveAdmin)
 admin.site.register(Area)
 admin.site.register(CaveAndEntrance)
 admin.site.register(Entrance, EntranceAdmin)
 admin.site.register(SurvexBlock, SurvexBlockAdmin)
+admin.site.register(DrawingFile, DrawingFileAdmin)
 admin.site.register(Expedition)
 admin.site.register(Person,PersonAdmin)
 admin.site.register(SurvexPersonRole)
+admin.site.register(SurvexDirectory, SurvexDirectoryAdmin)
+admin.site.register(SurvexFile, SurvexFileAdmin)
+admin.site.register(SurvexStation, SurvexStationAdmin)
 admin.site.register(PersonExpedition,PersonExpeditionAdmin)
 admin.site.register(LogbookEntry, LogbookEntryAdmin)
 admin.site.register(QM, QMAdmin)
-admin.site.register(SurvexStation)
+admin.site.register(Wallet, WalletAdmin)
-admin.site.register(Wallet)
 admin.site.register(SingleScan)
 admin.site.register(DataIssue)
 

core/fixtures/test_upload_file.txt

@@ -0,0 +1,5 @@
+This file is uploaded by the integration test suite as part of the tests.
+
+It, and any other with similar names, e.g test_upload_GPev9qN.txt can be safely deleted,
+EXCEPT for the original copy which lives in troggle/core/fixtures/
+

core/fixtures/test_upload_nosuffix

@@ -0,0 +1,7 @@
+This file is uploaded by the integration test suite as part of the tests.
+
+This has no suffix so it is pretending to be a Therion config file.
+
+It, and any other with similar names, e.g test_upload_GPev9qN.txt can be safely deleted,
+EXCEPT for the original copy which lives in troggle/core/fixtures/
+

core/models/caves.py

@@ -432,7 +432,7 @@ class LogbookEntry(TroggleModel):
         # #return super(LogbookEntry, self).__init__(*args, **kwargs) # works in py3.5
         # #return TroggleModel.__init__(*args, **kwargs) # fails in py3.5, runtime fail in 3.8
 
-    def cave(self): # Why didn't he just make this a foreign key to Cave ? Replaces __egtattrribute__ sillyness.
+    def cave(self): # Why didn't he just make this a foreign key to Cave ? Replaces __getattrribute__ sillyness.
         c = CaveSlug.objects.get(slug=self.cave_slug, primary=True).cave
         return c
 
@@ -540,13 +540,13 @@ class PersonTrip(TroggleModel):
     def __str__(self):
         return f'{self.personexpedition} ({self.logbook_entry.date})'
 
-scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL)
+# scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL)
-def get_scan_path(instance, filename):
+# def get_scan_path(instance, filename):
-    year=instance.survey.expedition.year
+    # year=instance.survey.expedition.year
-    number=str(instance.survey.wallet_number)
+    # number=str(instance.survey.wallet_number)
-    if str(instance.survey.wallet_letter) != "None":
+    # if str(instance.survey.wallet_letter) != "None":
-        number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01
+        # number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01
-    return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg')
+    # return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg')
 
 Gcavelookup = None
 Gcave_count = None

core/models/survex.py

@@ -15,10 +15,10 @@ class SurvexDirectory(models.Model):
     
     class Meta:
         ordering = ('id',)
+        verbose_name_plural = "Survex directories"
 
-    # Don't change from the default as that breaks troggle webpages and internal referencing!
+    def __str__(self):
-    # def __str__(self):
+        return "[SurvexDirectory:"+str(self.path)  + "-" + str(self.primarysurvexfile.path) + "-" + str(self.cave)+"]" 
-        # return "[SurvexDirectory:"+str(self.path)  + "-" + str(self.primarysurvexfile.path) + "-" + str(self.cave)+"]" 
 
 
 class SurvexFile(models.Model):
@@ -53,7 +53,9 @@ class SurvexFile(models.Model):
             survexdirectory.save()
             self.survexdirectory = survexdirectory
         self.save()
-
+        
+    def __str__(self):
+        return self.path
 
 class SurvexStationLookUpManager(models.Manager):
     def lookup(self, name):
@@ -122,11 +124,11 @@ class SurvexBlock(models.Model):
     class Meta:
         ordering = ('id',)
 
-    # Don't change from the original as that breaks troggle webpages and internal referencing!
+    def __str__(self):
-    # def __str__(self):
+        return "[SurvexBlock:"+ str(self.name) + "-path:" + \
-        # return "[SurvexBlock:"+ str(self.name) + "-path:" + \
+            str(self.survexpath) + "-cave:" + \
-            # str(self.survexpath) + "-cave:" + \
+            str(self.cave) + "]" 
-            # str(self.cave) + "]" 
+            
     def __str__(self):
         return self.name and str(self.name) or 'no name'
 
@@ -134,40 +136,19 @@ class SurvexBlock(models.Model):
         return True
 
     def GetPersonroles(self):
-        '''To do: excise the 'role' bit of this while retaining personrole
+        '''
-        which is used in some later logic
-        
         But apparently never used !?
         '''
         res = [ ]
         for personrole in self.survexpersonrole_set.order_by('personexpedition'):
-            # if res and res[-1]['person'] == personrole.personexpedition.person:
-                # res[-1]['roles'] += ", " + str(personrole.nrole)
-            # else:
-                # res.append({'person':personrole.personexpedition.person, 'expeditionyear':personrole.personexpedition.expedition.year, 'roles':str(personrole.nrole)})
             res.append({'person':personrole.personexpedition.person, 'expeditionyear':personrole.personexpedition.expedition.year})
         return res
    
     def DayIndex(self):
         return list(self.expeditionday.survexblock_set.all()).index(self)
-#
-# member of a SurvexBlock
-#
-# ROLE_CHOICES = (
-        # ('insts','Instruments'),
-        # ('dog','Other'),
-        # ('notes','Notes'),
-        # ('pics','Pictures'),
-        # ('tape','Tape measure'),
-        # ('useless','Useless'),
-        # ('helper','Helper'),
-        # ('disto','Disto'),
-        # ('consultant','Consultant'),
-        # )
 
 class SurvexPersonRole(models.Model):
     survexblock         = models.ForeignKey('SurvexBlock',on_delete=models.CASCADE)
-#    nrole               = models.CharField(choices=ROLE_CHOICES, max_length=200, blank=True, null=True)
         # increasing levels of precision
     personname          = models.CharField(max_length=100)
     person              = models.ForeignKey('Person', blank=True, null=True,on_delete=models.SET_NULL)
@@ -176,7 +157,7 @@ class SurvexPersonRole(models.Model):
     expeditionday       = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.SET_NULL)
     
     def __str__(self):
-        return str(self.person) + " - " + str(self.survexblock) + " - " + str(self.nrole)
+        return str(self.person) + " - " + str(self.survexblock) 
 
 class Wallet(models.Model):
     fpath               = models.CharField(max_length=200)
@@ -189,7 +170,7 @@ class Wallet(models.Model):
         return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)}))
 
     def __str__(self):
-        return str(self.walletname) + " (Survey Scans Folder)"
+        return str(self.walletname) + " (Wallet)"
 
 class SingleScan(models.Model):
     ffile    = models.CharField(max_length=200)
@@ -216,4 +197,8 @@ class DrawingFile(models.Model):
     survexfiles      = models.ManyToManyField("SurvexFile")  # direct link to SVX files - not populated yet
 
     class Meta:
-        ordering = ('dwgpath',)
+        ordering = ('dwgpath',)
+
+    def __str__(self):
+        return "Drawing File: " + str(self.dwgname) + " (" + str(self.filesize) + " bytes)" 
+        

core/views/other.py

@@ -1,4 +1,5 @@
 import re, os
+import subprocess
 from pathlib import Path
 
 from django import forms
@@ -16,6 +17,7 @@ from troggle.parsers.imports import import_logbooks, import_QMs, import_drawings
 # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
 from troggle.core.models.troggle import Expedition, Person, PersonExpedition
 from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
+from troggle.core.models.survex import DrawingFile
 from .auth import login_required_if_public
 
 '''Utility functions and code to serve the control panel and individual user's 
@@ -32,9 +34,7 @@ todo = '''
     so this is only a tool for a first pass, to be followed by extensive hand-editing!
     When we have done all the old logbooks, delete this function and the two templates.
 
--Do GIT stuff for upload forms
 
--Move upload forms to proper file locations
 '''
 
 def todos(request, module): 
@@ -200,150 +200,3 @@ def exportlogbook(request,year=None,extension=None):
 
         return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':[completed]})
 
-
-class FilesForm(forms.Form):  # not a model-form, just a form-form
-    uploadfiles = forms.FileField() 
-
-@login_required_if_public
-def scanupload(request, wallet=None):
-    '''Upload scanned image files into a wallet on /expofiles
-    This does NOT use a Django model linked to a Django form. Just a simple Django form.
-    '''
-    filesaved = False
-    actual_saved = []
-    # print(f'! - FORM scanupload - start {wallet}')
-    if wallet is None:
-        wallet = "2021#01" # improve this later
-    if not re.match('(19|20)\d\d:\d\d', wallet):
-        wallet = "2021:01" # improve this later
-    
-    year = wallet[:4]
-    if int(year) < 1977: 
-        year = "1977"
-    if int(year) > 2050: 
-        year = "2050"
-    nexty = f'{int(year)+1}'
-    prevy = f'{int(year)-1}'
-
-    wnumber = wallet[5:]
-    next = f'{int(wnumber)+1:02d}'
-    prev = f'{int(wnumber)-1:02d}'
-    
-    if int(wnumber) == 0:
-        prev = f'{int(wnumber):02d}'
-    
-    context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty}
-    
-    wallet = wallet.replace(':','#')
-    dirpath = Path(settings.SURVEY_SCANS, year, wallet)
-    
-    form = FilesForm()
-    
-    if request.method == 'POST':
-        form = FilesForm(request.POST,request.FILES)
-        if form.is_valid():
-            f = request.FILES["uploadfiles"]
-            multiple = request.FILES.getlist('uploadfiles')
-            fs = FileSystemStorage(os.path.join(settings.SURVEY_SCANS, year, wallet))
-           
-            actual_saved = []
-            if multiple:
-                for f in multiple:
-                    actual_saved.append( fs.save(f.name, content=f) )
-                    # print(f'! - FORM scanupload multiple {actual_saved}')
-            filesaved = True
-    
-    files = []
-    dirs = []
-    # print(f'! - FORM scanupload - start {wallet} {dirpath}')
-    try:
-        for f in dirpath.iterdir():
-            if f.is_dir():
-                dirs.append(f.name)
-            if f.is_file():
-                if f.name != 'contents.json' and f.name != 'walletindex.html':
-                    files.append(f.name)
-    except FileNotFoundError:
-        files.append('(no wallet yet - would be created)')
-    if len(files) ==0 :
-        files.append('(no image files in wallet)')
-    else:
-        files = sorted(files)
-        
-    if dirs:
-        dirs = sorted(dirs)
-
-    return render(request, 'scanuploadform.html', 
-        {'form': form, 'wallet': wallet, **context, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved})
-
-@login_required_if_public
-def dwgupload(request, folder=None):
-    '''Upload DRAWING files (tunnel or therion) into the upload folder in :drawings:
-    This does NOT use a Django model linked to a Django form. Just a simple Django form.
-    '''
-    def dwgvalid(name):
-        if name in [ '.gitignore', '.hgignore', ]:
-            return False
-        if Path(name).suffix in ['.xml', '.th', '.th2', '', '.svg', '.jpg']:
-            return True
-        return False
-        
-    filesaved = False
-    actual_saved = []
-    doesnotexist = ''
-    #print(f'! - FORM dwgupload - start "{folder}"')
-    if folder is None:
-        folder = "" # improve this later
-        dirpath = Path(settings.DRAWINGS_DATA)
-        urlfile = '/dwgdataraw'
-        urldir  = '/dwgupload'
-    else:
-        dirpath = Path(settings.DRAWINGS_DATA, folder)
-        urlfile = Path('/dwgdataraw/') / folder
-        urldir  = Path('/dwgupload/') / folder
-    
-    form = FilesForm()
-    
-    if request.method == 'POST':
-        form = FilesForm(request.POST,request.FILES)
-        if form.is_valid():
-            f = request.FILES["uploadfiles"]
-            multiple = request.FILES.getlist('uploadfiles')
-            fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder))
-           
-            actual_saved = []
-            if multiple:
-                for f in multiple:
-                    if dwgvalid(f.name):
-                        actual_saved.append( fs.save(f.name, content=f) )
-            # print(f'! - FORM dwgupload multiple {actual_saved}')
-            filesaved = True
-            
-            # DO GIT STUFF & load into Database too
-                # parsers.surveys.SetTunnelfileInfo(dwgfile) # commented out
-    # dwgfile.save()
-
-    
-    files = []
-    dirs = []
-    #print(f'! - FORM dwgupload - start {folder} \n"{dirpath}" \n"{dirpath.parent}" \n"{dirpath.exists()}"')
-    try:
-        for f in dirpath.iterdir():
-            if f.is_dir():
-                if f.name not in ['.git' ]:
-                    dirs.append(f.name)
-                    continue
-            if f.is_file():
-                if dwgvalid(f.name): 
-                    files.append(f.name)
-                    continue
-    except FileNotFoundError:
-        doesnotexist = True
-    if files:
-        files = sorted(files)
-        
-    if dirs:
-        dirs = sorted(dirs)
-
-    return render(request, 'dwguploadform.html', 
-        {'form': form, 'doesnotexist': doesnotexist, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved})

core/views/uploads.py

@@ -0,0 +1,188 @@
+import re, os
+import subprocess
+from pathlib import Path
+
+from django import forms
+
+from django.conf import settings
+from django.urls import reverse
+from django.db.models import Q
+from django.http import HttpResponse, HttpResponseRedirect
+from django.shortcuts import render
+from django.template import Context, loader
+from django.core.files.storage import FileSystemStorage, default_storage
+
+from troggle.parsers.imports import import_caves, import_people, import_surveyscans
+from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
+# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
+from troggle.core.models.troggle import Expedition, Person, PersonExpedition
+from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
+from troggle.core.models.survex import DrawingFile
+from .auth import login_required_if_public
+
+'''File upload 'views' 
+'''
+
+todo = '''
+-Move upload forms to proper file locations
+'''
+
+class FilesForm(forms.Form):  # not a model-form, just a form-form
+    uploadfiles = forms.FileField() 
+
+@login_required_if_public
+def scanupload(request, wallet=None):
+    '''Upload scanned image files into a wallet on /expofiles
+    This does NOT use a Django model linked to a Django form. Just a simple Django form.
+    '''
+    filesaved = False
+    actual_saved = []
+    # print(f'! - FORM scanupload - start {wallet}')
+    if wallet is None:
+        wallet = "2021#01" # improve this later
+    if not re.match('(19|20)\d\d:\d\d', wallet):
+        wallet = "2021:01" # improve this later
+    
+    year = wallet[:4]
+    if int(year) < 1977: 
+        year = "1977"
+    if int(year) > 2050: 
+        year = "2050"
+    nexty = f'{int(year)+1}'
+    prevy = f'{int(year)-1}'
+
+    wnumber = wallet[5:]
+    next = f'{int(wnumber)+1:02d}'
+    prev = f'{int(wnumber)-1:02d}'
+    
+    if int(wnumber) == 0:
+        prev = f'{int(wnumber):02d}'
+    
+    context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty}
+    
+    wallet = wallet.replace(':','#')
+    dirpath = Path(settings.SURVEY_SCANS, year, wallet)
+    
+    form = FilesForm()
+    
+    if request.method == 'POST':
+        form = FilesForm(request.POST,request.FILES)
+        if form.is_valid():
+            f = request.FILES["uploadfiles"]
+            multiple = request.FILES.getlist('uploadfiles')
+            fs = FileSystemStorage(os.path.join(settings.SURVEY_SCANS, year, wallet))
+           
+            actual_saved = []
+            if multiple:
+                for f in multiple:
+                    actual_saved.append( fs.save(f.name, content=f) )
+                    # print(f'! - FORM scanupload multiple {actual_saved}')
+            filesaved = True
+    
+    files = []
+    dirs = []
+    # print(f'! - FORM scanupload - start {wallet} {dirpath}')
+    try:
+        for f in dirpath.iterdir():
+            if f.is_dir():
+                dirs.append(f.name)
+            if f.is_file():
+                if f.name != 'contents.json' and f.name != 'walletindex.html':
+                    files.append(f.name)
+    except FileNotFoundError:
+        files.append('(no wallet yet - would be created)')
+    if len(files) ==0 :
+        files.append('(no image files in wallet)')
+    else:
+        files = sorted(files)
+        
+    if dirs:
+        dirs = sorted(dirs)
+
+    return render(request, 'scanuploadform.html', 
+        {'form': form, 'wallet': wallet, **context, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved})
+
+@login_required_if_public
+def dwgupload(request, folder=None):
+    '''Upload DRAWING files (tunnel or therion) into the upload folder in :drawings:
+    This does NOT use a Django model linked to a Django form. Just a simple Django form.
+    
+    We use get_or_create instead of simply creating a new object in case someone uploads the same file
+    several times in one session, and expects them to be overwritten in the database. Although
+    the actual file will be duplicated in the filesystem with different random name ammendation.
+    '''
+    def dwgvalid(name):
+        if name in [ '.gitignore', '.hgignore', ]:
+            return False
+        if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.jpg', '.pdf', 'jpeg']:
+            return True
+        return False
+        
+    filesaved = False
+    actual_saved = []
+    refused = []
+    doesnotexist = ''
+    #print(f'! - FORM dwgupload - start "{folder}"')
+    if folder is None:
+        folder = "" # improve this later
+        dirpath = Path(settings.DRAWINGS_DATA)
+        urlfile = '/dwgdataraw'
+        urldir  = '/dwgupload'
+    else:
+        dirpath = Path(settings.DRAWINGS_DATA, folder)
+        urlfile = Path('/dwgdataraw/') / folder
+        urldir  = Path('/dwgupload/') / folder
+    
+    form = FilesForm()
+    
+    if request.method == 'POST':
+        form = FilesForm(request.POST,request.FILES)
+        if form.is_valid():
+            f = request.FILES["uploadfiles"]
+            multiple = request.FILES.getlist('uploadfiles')
+            fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder))
+           
+            actual_saved = []
+            refused = []
+            git = settings.GIT
+            if multiple:
+                for f in multiple:
+                    if dwgvalid(f.name):
+                        saved_filename = fs.save(f.name, content=f)
+                        actual_saved.append(saved_filename)
+                        subprocess.call([git, "add", saved_filename], cwd=dirpath)
+                        # dwgfile = DrawingFile(dwgpath=f.name, dwgname=Path(f.name).stem, filesize=f.size)
+                        dwgfile, created = DrawingFile.objects.get_or_create(dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size)  
+                        # if not created:
+                            # print(f'FAILED to create {saved_filename} in {dirpath}')
+                        # else:
+                            # print(f'{dwgfile}')
+                        dwgfile.save()
+                    else:
+                        refused.append(f.name)
+            # print(f'! - FORM dwgupload multiple {actual_saved}')
+            filesaved = True
+            subprocess.call([git, "commit", "-m", 'dwgupload'], cwd=dirpath)
+    files = []
+    dirs = []
+    #print(f'! - FORM dwgupload - start {folder} \n"{dirpath}" \n"{dirpath.parent}" \n"{dirpath.exists()}"')
+    try:
+        for f in dirpath.iterdir():
+            if f.is_dir():
+                if f.name not in ['.git' ]:
+                    dirs.append(f.name)
+                    continue
+            if f.is_file():
+                if dwgvalid(f.name): 
+                    files.append(f.name)
+                    continue
+    except FileNotFoundError:
+        doesnotexist = True
+    if files:
+        files = sorted(files)
+        
+    if dirs:
+        dirs = sorted(dirs)
+
+    return render(request, 'dwguploadform.html', 
+        {'form': form, 'doesnotexist': doesnotexist, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved, 'refused': refused})

parsers/drawings.py

@@ -34,7 +34,7 @@ def get_or_create_placeholder(year):
     placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs)
     return placeholder_logbook_entry
 
-def find_tunnel_file(dwgfile, path):
+def find_dwg_file(dwgfile, path):
     '''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file
     which we have already seen when we imported all the files we could find in the surveyscans direstories
     '''
@@ -164,7 +164,7 @@ def setdwgfileinfo(dwgfile):
     # <pcarea area_signal="frame" sfscaledown="12.282584" sfrotatedeg="-90.76982" sfxtrans="11.676667377221136" sfytrans="-15.677173422877454" sfsketch="204description/scans/plan(38).png" sfstyle="" nodeconnzsetrelative="0.0">
     
     for path, style in rx_pcpath.findall(ttext):
-        find_tunnel_file(dwgfile, path.decode())
+        find_dwg_file(dwgfile, path.decode())
     
     # should also scan and look for survex blocks that might have been included, and image scans
     # which would populate dwgfile.survexfile

parsers/survex.py

@@ -207,7 +207,6 @@ class LoadingSurvex():
                     if (personexpedition, tm) not in teammembers:
                         teammembers.append((personexpedition, tm))
                         personrole = SurvexPersonRole(survexblock=survexblock, personexpedition=personexpedition, personname=tm)
-#                       personrole = SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm)
                         personrole.save()
                         personrole.expeditionday = survexblock.expeditionday
                         if personexpedition:

templates/dwguploadform.html

@@ -22,41 +22,48 @@
 <p>Only drawings and drawing config files can be uploaded.
 </div>
 <div style = "max-width:70%; margin-left:20%; text-align: left" >
+    {% if refused %}
+        <p>
+        <b>Files refused: </b><br>
+        {% for f in refused %}
+        <em>{{f}}</em> <br>
+        {% endfor %}
+        </p>
+    {% endif %}
     {% if filesaved %}
         <p>
-        <b>Drawing(s) saved as <br>
+        <b>Drawing(s) saved as </b><br>
         {% for f in actual_saved %}
         <em>{{f}}</em> <br>
         {% endfor %}
-        <br><br>Upload more?</b>
+        <br>Upload more?</b>
         </p>
-        <br>
     {% endif %}
-
+    <hr><br>
-{% if doesnotexist %}
+    {% if doesnotexist %}
-<p>No folder of this name.<br>
+        <p>No folder of this name.<br>
-It would be created if you upload a file.
+        It would be created if you upload a file.
-{% else %}
+    {% else %}
-    <strong style="font-size: 110%;">Files:</strong><br>
+        <strong style="font-size: 110%;">Files:</strong><br>
-    {% for f in files %}
+        {% for f in files %}
-    <a href="{{urlfile|urlencode}}/{{f|urlencode}}">{{f}}</a><br />
+        <a href="{{urlfile|urlencode}}/{{f|urlencode}}">{{f}}</a><br />
-    {% empty %}
+        {% empty %}
-    <p>&lt;No files here&gt;
+        <p>&lt;No files here&gt;
-    {% endfor %}
+        {% endfor %}
 
 
-    <p><strong style="font-size: 110%;">Directories:</strong><br>
+        <p><strong style="font-size: 110%;">Directories:</strong><br>
-    {% if folder %}
+        {% if folder %}
-    <a href="{{urldir}}/..">[up]</a><br />
+        <a href="{{urldir}}/..">[up]</a><br />
+        {% endif %}
+        {% for f in dirs %}
+        <a href="{{urldir}}/{{f}}">/{{f}}/</a><br />
+        {% empty %}
+        <p>&lt;No subdirectories&gt;
+        {% endfor %}
+        <p>Clicking on a filename only works if the drawing file has been imported into the system as part of a bulk-import
+        as we are matching it against a file recorded in the database.
     {% endif %}
-    {% for f in dirs %}
-    <a href="{{urldir}}/{{f}}">/{{f}}/</a><br />
-    {% empty %}
-    <p>&lt;No subdirectories&gt;
-    {% endfor %}
-    <p>Clicking on a filename only works if the drawing file has been imported into the system as part of a bulk-import
-    as we are matching it against a file recorded in the database.
-{% endif %}
 </div> 
 
 {% endblock %}

urls.py

@@ -12,8 +12,8 @@ from troggle.core.views import caves, statistics, survex
 from troggle.core.views.scans import scansingle, singlewallet, allwallets
 from troggle.core.views.drawings import dwgallfiles, dwgfilesingle
 #from troggle.core.views.drawings import dwgfileupload
-from troggle.core.views.other import dwgupload
+from troggle.core.views.uploads import dwgupload, scanupload
-from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, scanupload 
+from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage 
 from troggle.core.views.other import exportlogbook
 from troggle.core.views.caves import ent, cavepage
 from troggle.core.views.logbooks import get_logbook_entries, logbookentry, logbookSearch