From d374779c473a65fca4740d83942bae24c10f683d Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Wed, 5 May 2021 00:35:10 +0100 Subject: [PATCH] dwg upload and django admin extra search --- core/TESTS/test_imports.py | 8 +- core/TESTS/tests_logins.py | 47 ++++++-- core/admin.py | 31 ++++- core/fixtures/test_upload_file.txt | 5 + core/fixtures/test_upload_nosuffix | 7 ++ core/models/caves.py | 16 +-- core/models/survex.py | 53 +++----- core/views/other.py | 151 +---------------------- core/views/uploads.py | 188 +++++++++++++++++++++++++++++ parsers/drawings.py | 4 +- parsers/survex.py | 1 - templates/dwguploadform.html | 57 +++++---- urls.py | 4 +- 13 files changed, 333 insertions(+), 239 deletions(-) create mode 100644 core/fixtures/test_upload_file.txt create mode 100644 core/fixtures/test_upload_nosuffix create mode 100644 core/views/uploads.py diff --git a/core/TESTS/test_imports.py b/core/TESTS/test_imports.py index 9edd5c5..7445d25 100644 --- a/core/TESTS/test_imports.py +++ b/core/TESTS/test_imports.py @@ -40,9 +40,11 @@ class SimpleTest(SimpleTestCase): import troggle.core.models.survex as models_survex import troggle.core.models.caves as models_caves from troggle.parsers.people import GetPersonExpeditionNameLookup - from troggle.core.views.other import troggle404, frontpage + from troggle.core.views.other import frontpage from troggle.core.views.caves import ent, cavepage - from troggle.core.views import scans, drawings, other, caves, statistics, survex + from troggle.core.views import scans, drawings, other, caves, statistics, survex, uploads + def test_import_views_uploads(self): + from troggle.core.views.uploads import dwgupload, scanupload def test_import_parsers_QMs(self): from troggle.core.models.caves import QM, Cave, LogbookEntry def test_import_parsers_people(self): @@ -97,7 +99,7 @@ class SimpleTest(SimpleTestCase): from troggle.core.views.logbooks import expedition, personexpedition, Expeditions_tsvListView, Expeditions_jsonListView from troggle.core.views.logbooks import get_logbook_entries, logbookentry, logbookSearch from troggle.core.views.logbooks import notablepersons, person, get_people - from troggle.core.views.other import troggle404, frontpage + from troggle.core.views.other import controlpanel from troggle.core.views.prospect import prospecting from troggle.core.views.prospect import prospecting_image from troggle.core.views.statistics import pathsreport, stats, dataissues diff --git a/core/TESTS/tests_logins.py b/core/TESTS/tests_logins.py index 9148207..4c021aa 100644 --- a/core/TESTS/tests_logins.py +++ b/core/TESTS/tests_logins.py @@ -86,20 +86,49 @@ class PostTests(TestCase): self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') logged_in = c.login(username=u.username, password='secretword') - with open('README.txt','r') as testf: - response = self.client.post('/scanupload/2021:00', data={'title': '2021#00', 'name': 'README.txt', 'scanfiles': testf }) + with open('core/fixtures/test_upload_file.txt','r') as testf: + response = self.client.post('/scanupload/2020:00', data={'name': 'test_upload_file.txt', 'uploadfiles': testf }) content = response.content.decode() self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, HTTPStatus.OK) - # with open('test_up.html', 'w') as f: + # with open('testresponse.html', 'w') as f: # f.write(content) - t = re.search('README', content) - self.assertIsNotNone(t, 'Logged in but failed to see "\'README"' ) - t = re.search(' saved as', content) - self.assertIsNotNone(t, 'Logged in but failed to see "File(s) saved as"' ) - t = re.search('/expofiles/surveyscans/2021/2021%2300/README', content) - self.assertIsNotNone(t, 'Logged in but failed to see "/expofiles/..."' ) + for ph in [ r'test_upload_', + r'← 2020#00 →', + r'Upload more?']: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") + def test_dwg_upload(self): + '''Test file upload. Need to login first. + First upload is refused as it is a TXT file + Second upload is an image and suceeds. + ''' + c = self.client + from django.contrib.auth.models import User + u = User.objects.get(username='expotest') + + self.assertTrue(u.is_active, 'User \'' + u.username + '\' is INACTIVE') + logged_in = c.login(username=u.username, password='secretword') + + with open('core/fixtures/test_upload_file.txt','r') as testf: + response = self.client.post('/dwgupload/uploads', data={'name': 'test_upload_file.txt', 'uploadfiles': testf }) + content = response.content.decode() + self.assertEqual(response.status_code, 200) + t = re.search('Files refused:', content) + self.assertIsNotNone(t, 'Logged in but failed to see "Files refused:"' ) + + with open('core/fixtures/test_upload_nosuffix','r') as testf: + response = self.client.post('/dwgupload/uploads', data={'name': 'test_upload_nosuffix', 'uploadfiles': testf }) + content = response.content.decode() + with open('testresponse.html', 'w') as f: + f.write(content) + self.assertEqual(response.status_code, 200) + for ph in [ r'Upload more', + r' saved as ', + r'Clicking on a filename only']: + phmatch = re.search(ph, content) + self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'") class ComplexLoginTests(TestCase): diff --git a/core/admin.py b/core/admin.py index dbf34b8..c11d866 100644 --- a/core/admin.py +++ b/core/admin.py @@ -7,7 +7,8 @@ from django.core import serializers from troggle.core.views.other import exportlogbook from troggle.core.models.troggle import Person, PersonExpedition, Expedition, DataIssue from troggle.core.models.caves import Cave, Area, Entrance, CaveAndEntrance, LogbookEntry, PersonTrip, QM -from troggle.core.models.survex import SurvexBlock, SurvexPersonRole, SurvexStation, Wallet, SingleScan +from troggle.core.models.survex import SurvexBlock, SurvexFile, SurvexPersonRole, SurvexStation, SurvexDirectory +from troggle.core.models.survex import Wallet, SingleScan, DrawingFile '''This code significantly adds to the capabilities of the Django Management control panel for Troggle data. In particular, it enables JSON export of any data with 'export_as_json' @@ -36,9 +37,9 @@ class SurvexBlockAdmin(TroggleModelAdmin): inlines = (RoleInline,) -class SurveyAdmin(TroggleModelAdmin): - #inlines = (ScannedImageInline,) - search_fields = ('expedition__year','wallet_number') +# class SurveyAdmin(TroggleModelAdmin): + # #inlines = (ScannedImageInline,) + # search_fields = ('expedition__year','wallet_number') class QMsFoundInline(admin.TabularInline): @@ -105,20 +106,38 @@ class CaveAdmin(TroggleModelAdmin): class EntranceAdmin(TroggleModelAdmin): search_fields = ('caveandentrance__cave__kataster_number',) +class SurvexStationAdmin(TroggleModelAdmin): + search_fields = ('name',) + +class SurvexFileAdmin(TroggleModelAdmin): + search_fields = ('path',) + +class SurvexDirectoryAdmin(TroggleModelAdmin): + search_fields = ('path', 'survexdirectory',) + +class DrawingFileAdmin(TroggleModelAdmin): + search_fields = ('dwgname',) + +class WalletAdmin(TroggleModelAdmin): + search_fields = ('fpath',) + admin.site.register(Cave, CaveAdmin) admin.site.register(Area) admin.site.register(CaveAndEntrance) admin.site.register(Entrance, EntranceAdmin) admin.site.register(SurvexBlock, SurvexBlockAdmin) +admin.site.register(DrawingFile, DrawingFileAdmin) admin.site.register(Expedition) admin.site.register(Person,PersonAdmin) admin.site.register(SurvexPersonRole) +admin.site.register(SurvexDirectory, SurvexDirectoryAdmin) +admin.site.register(SurvexFile, SurvexFileAdmin) +admin.site.register(SurvexStation, SurvexStationAdmin) admin.site.register(PersonExpedition,PersonExpeditionAdmin) admin.site.register(LogbookEntry, LogbookEntryAdmin) admin.site.register(QM, QMAdmin) -admin.site.register(SurvexStation) -admin.site.register(Wallet) +admin.site.register(Wallet, WalletAdmin) admin.site.register(SingleScan) admin.site.register(DataIssue) diff --git a/core/fixtures/test_upload_file.txt b/core/fixtures/test_upload_file.txt new file mode 100644 index 0000000..9d955a5 --- /dev/null +++ b/core/fixtures/test_upload_file.txt @@ -0,0 +1,5 @@ +This file is uploaded by the integration test suite as part of the tests. + +It, and any other with similar names, e.g test_upload_GPev9qN.txt can be safely deleted, +EXCEPT for the original copy which lives in troggle/core/fixtures/ + diff --git a/core/fixtures/test_upload_nosuffix b/core/fixtures/test_upload_nosuffix new file mode 100644 index 0000000..53b75be --- /dev/null +++ b/core/fixtures/test_upload_nosuffix @@ -0,0 +1,7 @@ +This file is uploaded by the integration test suite as part of the tests. + +This has no suffix so it is pretending to be a Therion config file. + +It, and any other with similar names, e.g test_upload_GPev9qN.txt can be safely deleted, +EXCEPT for the original copy which lives in troggle/core/fixtures/ + diff --git a/core/models/caves.py b/core/models/caves.py index 259854d..7bbe21b 100644 --- a/core/models/caves.py +++ b/core/models/caves.py @@ -432,7 +432,7 @@ class LogbookEntry(TroggleModel): # #return super(LogbookEntry, self).__init__(*args, **kwargs) # works in py3.5 # #return TroggleModel.__init__(*args, **kwargs) # fails in py3.5, runtime fail in 3.8 - def cave(self): # Why didn't he just make this a foreign key to Cave ? Replaces __egtattrribute__ sillyness. + def cave(self): # Why didn't he just make this a foreign key to Cave ? Replaces __getattrribute__ sillyness. c = CaveSlug.objects.get(slug=self.cave_slug, primary=True).cave return c @@ -540,13 +540,13 @@ class PersonTrip(TroggleModel): def __str__(self): return f'{self.personexpedition} ({self.logbook_entry.date})' -scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL) -def get_scan_path(instance, filename): - year=instance.survey.expedition.year - number=str(instance.survey.wallet_number) - if str(instance.survey.wallet_letter) != "None": - number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01 - return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg') +# scansFileStorage = FileSystemStorage(location=settings.SURVEY_SCANS, base_url=settings.SURVEYS_URL) +# def get_scan_path(instance, filename): + # year=instance.survey.expedition.year + # number=str(instance.survey.wallet_number) + # if str(instance.survey.wallet_letter) != "None": + # number=str(instance.survey.wallet_letter) + number #two strings formatting because convention is 2009#01 or 2009#X01 + # return os.path.join('./',year,year+r'#'+number,str(instance.contents)+str(instance.number_in_wallet)+r'.jpg') Gcavelookup = None Gcave_count = None diff --git a/core/models/survex.py b/core/models/survex.py index 50173f4..1273a8b 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -15,10 +15,10 @@ class SurvexDirectory(models.Model): class Meta: ordering = ('id',) + verbose_name_plural = "Survex directories" - # Don't change from the default as that breaks troggle webpages and internal referencing! - # def __str__(self): - # return "[SurvexDirectory:"+str(self.path) + "-" + str(self.primarysurvexfile.path) + "-" + str(self.cave)+"]" + def __str__(self): + return "[SurvexDirectory:"+str(self.path) + "-" + str(self.primarysurvexfile.path) + "-" + str(self.cave)+"]" class SurvexFile(models.Model): @@ -53,7 +53,9 @@ class SurvexFile(models.Model): survexdirectory.save() self.survexdirectory = survexdirectory self.save() - + + def __str__(self): + return self.path class SurvexStationLookUpManager(models.Manager): def lookup(self, name): @@ -122,11 +124,11 @@ class SurvexBlock(models.Model): class Meta: ordering = ('id',) - # Don't change from the original as that breaks troggle webpages and internal referencing! - # def __str__(self): - # return "[SurvexBlock:"+ str(self.name) + "-path:" + \ - # str(self.survexpath) + "-cave:" + \ - # str(self.cave) + "]" + def __str__(self): + return "[SurvexBlock:"+ str(self.name) + "-path:" + \ + str(self.survexpath) + "-cave:" + \ + str(self.cave) + "]" + def __str__(self): return self.name and str(self.name) or 'no name' @@ -134,40 +136,19 @@ class SurvexBlock(models.Model): return True def GetPersonroles(self): - '''To do: excise the 'role' bit of this while retaining personrole - which is used in some later logic - + ''' But apparently never used !? ''' res = [ ] for personrole in self.survexpersonrole_set.order_by('personexpedition'): - # if res and res[-1]['person'] == personrole.personexpedition.person: - # res[-1]['roles'] += ", " + str(personrole.nrole) - # else: - # res.append({'person':personrole.personexpedition.person, 'expeditionyear':personrole.personexpedition.expedition.year, 'roles':str(personrole.nrole)}) res.append({'person':personrole.personexpedition.person, 'expeditionyear':personrole.personexpedition.expedition.year}) return res def DayIndex(self): return list(self.expeditionday.survexblock_set.all()).index(self) -# -# member of a SurvexBlock -# -# ROLE_CHOICES = ( - # ('insts','Instruments'), - # ('dog','Other'), - # ('notes','Notes'), - # ('pics','Pictures'), - # ('tape','Tape measure'), - # ('useless','Useless'), - # ('helper','Helper'), - # ('disto','Disto'), - # ('consultant','Consultant'), - # ) class SurvexPersonRole(models.Model): survexblock = models.ForeignKey('SurvexBlock',on_delete=models.CASCADE) -# nrole = models.CharField(choices=ROLE_CHOICES, max_length=200, blank=True, null=True) # increasing levels of precision personname = models.CharField(max_length=100) person = models.ForeignKey('Person', blank=True, null=True,on_delete=models.SET_NULL) @@ -176,7 +157,7 @@ class SurvexPersonRole(models.Model): expeditionday = models.ForeignKey("ExpeditionDay", null=True,on_delete=models.SET_NULL) def __str__(self): - return str(self.person) + " - " + str(self.survexblock) + " - " + str(self.nrole) + return str(self.person) + " - " + str(self.survexblock) class Wallet(models.Model): fpath = models.CharField(max_length=200) @@ -189,7 +170,7 @@ class Wallet(models.Model): return urljoin(settings.URL_ROOT, reverse('singlewallet', kwargs={"path":re.sub("#", "%23", self.walletname)})) def __str__(self): - return str(self.walletname) + " (Survey Scans Folder)" + return str(self.walletname) + " (Wallet)" class SingleScan(models.Model): ffile = models.CharField(max_length=200) @@ -216,4 +197,8 @@ class DrawingFile(models.Model): survexfiles = models.ManyToManyField("SurvexFile") # direct link to SVX files - not populated yet class Meta: - ordering = ('dwgpath',) \ No newline at end of file + ordering = ('dwgpath',) + + def __str__(self): + return "Drawing File: " + str(self.dwgname) + " (" + str(self.filesize) + " bytes)" + \ No newline at end of file diff --git a/core/views/other.py b/core/views/other.py index 8ac7f34..0fc43a6 100644 --- a/core/views/other.py +++ b/core/views/other.py @@ -1,4 +1,5 @@ import re, os +import subprocess from pathlib import Path from django import forms @@ -16,6 +17,7 @@ from troggle.parsers.imports import import_logbooks, import_QMs, import_drawings # from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip +from troggle.core.models.survex import DrawingFile from .auth import login_required_if_public '''Utility functions and code to serve the control panel and individual user's @@ -32,9 +34,7 @@ todo = ''' so this is only a tool for a first pass, to be followed by extensive hand-editing! When we have done all the old logbooks, delete this function and the two templates. --Do GIT stuff for upload forms --Move upload forms to proper file locations ''' def todos(request, module): @@ -200,150 +200,3 @@ def exportlogbook(request,year=None,extension=None): return render(request,'controlPanel.html', {'expeditions':Expedition.objects.all(),'jobs_completed':[completed]}) - -class FilesForm(forms.Form): # not a model-form, just a form-form - uploadfiles = forms.FileField() - -@login_required_if_public -def scanupload(request, wallet=None): - '''Upload scanned image files into a wallet on /expofiles - This does NOT use a Django model linked to a Django form. Just a simple Django form. - ''' - filesaved = False - actual_saved = [] - # print(f'! - FORM scanupload - start {wallet}') - if wallet is None: - wallet = "2021#01" # improve this later - if not re.match('(19|20)\d\d:\d\d', wallet): - wallet = "2021:01" # improve this later - - year = wallet[:4] - if int(year) < 1977: - year = "1977" - if int(year) > 2050: - year = "2050" - nexty = f'{int(year)+1}' - prevy = f'{int(year)-1}' - - wnumber = wallet[5:] - next = f'{int(wnumber)+1:02d}' - prev = f'{int(wnumber)-1:02d}' - - if int(wnumber) == 0: - prev = f'{int(wnumber):02d}' - - context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty} - - wallet = wallet.replace(':','#') - dirpath = Path(settings.SURVEY_SCANS, year, wallet) - - form = FilesForm() - - if request.method == 'POST': - form = FilesForm(request.POST,request.FILES) - if form.is_valid(): - f = request.FILES["uploadfiles"] - multiple = request.FILES.getlist('uploadfiles') - fs = FileSystemStorage(os.path.join(settings.SURVEY_SCANS, year, wallet)) - - actual_saved = [] - if multiple: - for f in multiple: - actual_saved.append( fs.save(f.name, content=f) ) - # print(f'! - FORM scanupload multiple {actual_saved}') - filesaved = True - - files = [] - dirs = [] - # print(f'! - FORM scanupload - start {wallet} {dirpath}') - try: - for f in dirpath.iterdir(): - if f.is_dir(): - dirs.append(f.name) - if f.is_file(): - if f.name != 'contents.json' and f.name != 'walletindex.html': - files.append(f.name) - except FileNotFoundError: - files.append('(no wallet yet - would be created)') - if len(files) ==0 : - files.append('(no image files in wallet)') - else: - files = sorted(files) - - if dirs: - dirs = sorted(dirs) - - return render(request, 'scanuploadform.html', - {'form': form, 'wallet': wallet, **context, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved}) - -@login_required_if_public -def dwgupload(request, folder=None): - '''Upload DRAWING files (tunnel or therion) into the upload folder in :drawings: - This does NOT use a Django model linked to a Django form. Just a simple Django form. - ''' - def dwgvalid(name): - if name in [ '.gitignore', '.hgignore', ]: - return False - if Path(name).suffix in ['.xml', '.th', '.th2', '', '.svg', '.jpg']: - return True - return False - - filesaved = False - actual_saved = [] - doesnotexist = '' - #print(f'! - FORM dwgupload - start "{folder}"') - if folder is None: - folder = "" # improve this later - dirpath = Path(settings.DRAWINGS_DATA) - urlfile = '/dwgdataraw' - urldir = '/dwgupload' - else: - dirpath = Path(settings.DRAWINGS_DATA, folder) - urlfile = Path('/dwgdataraw/') / folder - urldir = Path('/dwgupload/') / folder - - form = FilesForm() - - if request.method == 'POST': - form = FilesForm(request.POST,request.FILES) - if form.is_valid(): - f = request.FILES["uploadfiles"] - multiple = request.FILES.getlist('uploadfiles') - fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder)) - - actual_saved = [] - if multiple: - for f in multiple: - if dwgvalid(f.name): - actual_saved.append( fs.save(f.name, content=f) ) - # print(f'! - FORM dwgupload multiple {actual_saved}') - filesaved = True - - # DO GIT STUFF & load into Database too - # parsers.surveys.SetTunnelfileInfo(dwgfile) # commented out - # dwgfile.save() - - - files = [] - dirs = [] - #print(f'! - FORM dwgupload - start {folder} \n"{dirpath}" \n"{dirpath.parent}" \n"{dirpath.exists()}"') - try: - for f in dirpath.iterdir(): - if f.is_dir(): - if f.name not in ['.git' ]: - dirs.append(f.name) - continue - if f.is_file(): - if dwgvalid(f.name): - files.append(f.name) - continue - except FileNotFoundError: - doesnotexist = True - if files: - files = sorted(files) - - if dirs: - dirs = sorted(dirs) - - return render(request, 'dwguploadform.html', - {'form': form, 'doesnotexist': doesnotexist, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved}) diff --git a/core/views/uploads.py b/core/views/uploads.py new file mode 100644 index 0000000..6273ec4 --- /dev/null +++ b/core/views/uploads.py @@ -0,0 +1,188 @@ +import re, os +import subprocess +from pathlib import Path + +from django import forms + +from django.conf import settings +from django.urls import reverse +from django.db.models import Q +from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import render +from django.template import Context, loader +from django.core.files.storage import FileSystemStorage, default_storage + +from troggle.parsers.imports import import_caves, import_people, import_surveyscans +from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex +# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time* +from troggle.core.models.troggle import Expedition, Person, PersonExpedition +from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip +from troggle.core.models.survex import DrawingFile +from .auth import login_required_if_public + +'''File upload 'views' +''' + +todo = ''' +-Move upload forms to proper file locations +''' + +class FilesForm(forms.Form): # not a model-form, just a form-form + uploadfiles = forms.FileField() + +@login_required_if_public +def scanupload(request, wallet=None): + '''Upload scanned image files into a wallet on /expofiles + This does NOT use a Django model linked to a Django form. Just a simple Django form. + ''' + filesaved = False + actual_saved = [] + # print(f'! - FORM scanupload - start {wallet}') + if wallet is None: + wallet = "2021#01" # improve this later + if not re.match('(19|20)\d\d:\d\d', wallet): + wallet = "2021:01" # improve this later + + year = wallet[:4] + if int(year) < 1977: + year = "1977" + if int(year) > 2050: + year = "2050" + nexty = f'{int(year)+1}' + prevy = f'{int(year)-1}' + + wnumber = wallet[5:] + next = f'{int(wnumber)+1:02d}' + prev = f'{int(wnumber)-1:02d}' + + if int(wnumber) == 0: + prev = f'{int(wnumber):02d}' + + context = {'year': year, 'prev': prev, 'next': next, 'prevy': prevy, 'nexty': nexty} + + wallet = wallet.replace(':','#') + dirpath = Path(settings.SURVEY_SCANS, year, wallet) + + form = FilesForm() + + if request.method == 'POST': + form = FilesForm(request.POST,request.FILES) + if form.is_valid(): + f = request.FILES["uploadfiles"] + multiple = request.FILES.getlist('uploadfiles') + fs = FileSystemStorage(os.path.join(settings.SURVEY_SCANS, year, wallet)) + + actual_saved = [] + if multiple: + for f in multiple: + actual_saved.append( fs.save(f.name, content=f) ) + # print(f'! - FORM scanupload multiple {actual_saved}') + filesaved = True + + files = [] + dirs = [] + # print(f'! - FORM scanupload - start {wallet} {dirpath}') + try: + for f in dirpath.iterdir(): + if f.is_dir(): + dirs.append(f.name) + if f.is_file(): + if f.name != 'contents.json' and f.name != 'walletindex.html': + files.append(f.name) + except FileNotFoundError: + files.append('(no wallet yet - would be created)') + if len(files) ==0 : + files.append('(no image files in wallet)') + else: + files = sorted(files) + + if dirs: + dirs = sorted(dirs) + + return render(request, 'scanuploadform.html', + {'form': form, 'wallet': wallet, **context, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved}) + +@login_required_if_public +def dwgupload(request, folder=None): + '''Upload DRAWING files (tunnel or therion) into the upload folder in :drawings: + This does NOT use a Django model linked to a Django form. Just a simple Django form. + + We use get_or_create instead of simply creating a new object in case someone uploads the same file + several times in one session, and expects them to be overwritten in the database. Although + the actual file will be duplicated in the filesystem with different random name ammendation. + ''' + def dwgvalid(name): + if name in [ '.gitignore', '.hgignore', ]: + return False + if Path(name).suffix.lower() in ['.xml', '.th', '.th2', '', '.svg', '.jpg', '.pdf', 'jpeg']: + return True + return False + + filesaved = False + actual_saved = [] + refused = [] + doesnotexist = '' + #print(f'! - FORM dwgupload - start "{folder}"') + if folder is None: + folder = "" # improve this later + dirpath = Path(settings.DRAWINGS_DATA) + urlfile = '/dwgdataraw' + urldir = '/dwgupload' + else: + dirpath = Path(settings.DRAWINGS_DATA, folder) + urlfile = Path('/dwgdataraw/') / folder + urldir = Path('/dwgupload/') / folder + + form = FilesForm() + + if request.method == 'POST': + form = FilesForm(request.POST,request.FILES) + if form.is_valid(): + f = request.FILES["uploadfiles"] + multiple = request.FILES.getlist('uploadfiles') + fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder)) + + actual_saved = [] + refused = [] + git = settings.GIT + if multiple: + for f in multiple: + if dwgvalid(f.name): + saved_filename = fs.save(f.name, content=f) + actual_saved.append(saved_filename) + subprocess.call([git, "add", saved_filename], cwd=dirpath) + # dwgfile = DrawingFile(dwgpath=f.name, dwgname=Path(f.name).stem, filesize=f.size) + dwgfile, created = DrawingFile.objects.get_or_create(dwgpath=saved_filename, dwgname=Path(f.name).stem, filesize=f.size) + # if not created: + # print(f'FAILED to create {saved_filename} in {dirpath}') + # else: + # print(f'{dwgfile}') + dwgfile.save() + else: + refused.append(f.name) + # print(f'! - FORM dwgupload multiple {actual_saved}') + filesaved = True + subprocess.call([git, "commit", "-m", 'dwgupload'], cwd=dirpath) + files = [] + dirs = [] + #print(f'! - FORM dwgupload - start {folder} \n"{dirpath}" \n"{dirpath.parent}" \n"{dirpath.exists()}"') + try: + for f in dirpath.iterdir(): + if f.is_dir(): + if f.name not in ['.git' ]: + dirs.append(f.name) + continue + if f.is_file(): + if dwgvalid(f.name): + files.append(f.name) + continue + except FileNotFoundError: + doesnotexist = True + if files: + files = sorted(files) + + if dirs: + dirs = sorted(dirs) + + return render(request, 'dwguploadform.html', + {'form': form, 'doesnotexist': doesnotexist, 'urlfile': urlfile, 'urldir': urldir,'folder': folder, 'files': files, 'dirs': dirs, 'filesaved': filesaved, 'actual_saved': actual_saved, 'refused': refused}) diff --git a/parsers/drawings.py b/parsers/drawings.py index f3aca47..8c45aae 100644 --- a/parsers/drawings.py +++ b/parsers/drawings.py @@ -34,7 +34,7 @@ def get_or_create_placeholder(year): placeholder_logbook_entry, newly_created = save_carefully(LogbookEntry, lookupAttribs, nonLookupAttribs) return placeholder_logbook_entry -def find_tunnel_file(dwgfile, path): +def find_dwg_file(dwgfile, path): '''Is given a line of text 'path' which may or may not contain a recognisable name of a scanned file which we have already seen when we imported all the files we could find in the surveyscans direstories ''' @@ -164,7 +164,7 @@ def setdwgfileinfo(dwgfile): # for path, style in rx_pcpath.findall(ttext): - find_tunnel_file(dwgfile, path.decode()) + find_dwg_file(dwgfile, path.decode()) # should also scan and look for survex blocks that might have been included, and image scans # which would populate dwgfile.survexfile diff --git a/parsers/survex.py b/parsers/survex.py index f329ace..569d091 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -207,7 +207,6 @@ class LoadingSurvex(): if (personexpedition, tm) not in teammembers: teammembers.append((personexpedition, tm)) personrole = SurvexPersonRole(survexblock=survexblock, personexpedition=personexpedition, personname=tm) -# personrole = SurvexPersonRole(survexblock=survexblock, nrole=mteammember.group(1).lower(), personexpedition=personexpedition, personname=tm) personrole.save() personrole.expeditionday = survexblock.expeditionday if personexpedition: diff --git a/templates/dwguploadform.html b/templates/dwguploadform.html index ac65dab..caeb66f 100644 --- a/templates/dwguploadform.html +++ b/templates/dwguploadform.html @@ -22,41 +22,48 @@

Only drawings and drawing config files can be uploaded.

+ {% if refused %} +

+ Files refused:
+ {% for f in refused %} + {{f}}
+ {% endfor %} +

+ {% endif %} {% if filesaved %}

- Drawing(s) saved as
+ Drawing(s) saved as
{% for f in actual_saved %} {{f}}
{% endfor %} -

Upload more? +
Upload more?

-
{% endif %} - -{% if doesnotexist %} -

No folder of this name.
-It would be created if you upload a file. -{% else %} - Files:
- {% for f in files %} - {{f}}
- {% empty %} -

<No files here> - {% endfor %} +


+ {% if doesnotexist %} +

No folder of this name.
+ It would be created if you upload a file. + {% else %} + Files:
+ {% for f in files %} + {{f}}
+ {% empty %} +

<No files here> + {% endfor %} -

Directories:
- {% if folder %} - [up]
+

Directories:
+ {% if folder %} + [up]
+ {% endif %} + {% for f in dirs %} + /{{f}}/
+ {% empty %} +

<No subdirectories> + {% endfor %} +

Clicking on a filename only works if the drawing file has been imported into the system as part of a bulk-import + as we are matching it against a file recorded in the database. {% endif %} - {% for f in dirs %} - /{{f}}/
- {% empty %} -

<No subdirectories> - {% endfor %} -

Clicking on a filename only works if the drawing file has been imported into the system as part of a bulk-import - as we are matching it against a file recorded in the database. -{% endif %}

{% endblock %} diff --git a/urls.py b/urls.py index c3eabc6..c83bfdb 100644 --- a/urls.py +++ b/urls.py @@ -12,8 +12,8 @@ from troggle.core.views import caves, statistics, survex from troggle.core.views.scans import scansingle, singlewallet, allwallets from troggle.core.views.drawings import dwgallfiles, dwgfilesingle #from troggle.core.views.drawings import dwgfileupload -from troggle.core.views.other import dwgupload -from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage, scanupload +from troggle.core.views.uploads import dwgupload, scanupload +from troggle.core.views.other import troggle404, frontpage, todos, controlpanel, frontpage from troggle.core.views.other import exportlogbook from troggle.core.views.caves import ent, cavepage from troggle.core.views.logbooks import get_logbook_entries, logbookentry, logbookSearch