diff --git a/core/TESTS/test_imports.py b/core/TESTS/test_imports.py index 7936c95..38d89ae 100644 --- a/core/TESTS/test_imports.py +++ b/core/TESTS/test_imports.py @@ -60,7 +60,7 @@ class SimpleTest(SimpleTestCase): import troggle.core.views.expo from troggle.core.models.troggle import Expedition from troggle.core.models.caves import CaveSlug, Cave, CaveAndEntrance, QM, EntranceSlug, Entrance, Area, SurvexStation - from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm + from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, EntranceForm, EntranceLetterForm from troggle.core.views.login import login_required_if_public from django.contrib.auth.decorators import login_required from django.conf import settings diff --git a/core/forms.py b/core/forms.py index 907f6ce..42ed0e0 100644 --- a/core/forms.py +++ b/core/forms.py @@ -54,10 +54,10 @@ class CaveForm(ModelForm): self._errors["url"] = self.error_class(["This field cannot start with a /."]) return self.cleaned_data -class VersionControlCommentForm(forms.Form): - '''Was appended to all forms. Not used currently - ''' - description_of_change = forms.CharField(required = True, widget=forms.Textarea(attrs={'rows':2})) +# class VersionControlCommentForm(forms.Form): + # '''Was appended to all forms. Not used currently + # ''' + # description_of_change = forms.CharField(required = True, widget=forms.Textarea(attrs={'rows':2})) class EntranceForm(ModelForm): diff --git a/core/models/caves.py b/core/models/caves.py index c76b8b5..a990650 100644 --- a/core/models/caves.py +++ b/core/models/caves.py @@ -5,6 +5,7 @@ import re import json from subprocess import call from collections import defaultdict +from pathlib import Path from urllib.parse import urljoin @@ -22,6 +23,20 @@ from django.template import Context, loader from troggle.core.models.troggle import TroggleModel, Person, Expedition, DataIssue from troggle.core.models.survex import SurvexStation +'''The model declarations for Areas, Caves and Entrances. Also LogBookENtry, QM, PersonTrip +''' + +todo='''- Move utility function into utils.py + +- Find out why we have separate objects CaveSlug and EntranceSlug and why + these are not just a single field on the Model. Do we ever need more + than one slug per cave or entrance? Surely that would break everything?? + +- Move PersonTrip to be with Person and Expedition elsewhere + +- Restore constraint: unique_together = (("area", "kataster_number"), ("area", "unofficial_number")) +''' + class Area(TroggleModel): short_name = models.CharField(max_length=100) name = models.CharField(max_length=200, blank=True, null=True) @@ -344,6 +359,9 @@ class Entrance(TroggleModel): self.save() return self.cached_primary_slug + def get_file_path(self): + return Path(settings.ENTRANCEDESCRIPTIONS, self.filename) + def writeDataFile(self): try: f = open(os.path.join(settings.ENTRANCEDESCRIPTIONS, self.filename), "w") diff --git a/core/views/caves.py b/core/views/caves.py index efccd8f..9d77d33 100644 --- a/core/views/caves.py +++ b/core/views/caves.py @@ -18,7 +18,8 @@ import troggle.settings as settings from troggle.core.views import expo from troggle.core.models.troggle import Expedition, DataIssue from troggle.core.models.caves import CaveSlug, Cave, CaveAndEntrance, QM, EntranceSlug, Entrance, Area, SurvexStation, GetCaveLookup -from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm +from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, EntranceForm, EntranceLetterForm +#from troggle.core.forms import VersionControlCommentForm from .login import login_required_if_public '''Manages the complex procedures to assemble a cave description out of the compnoents @@ -323,7 +324,7 @@ def edit_cave(request, slug=None): if request.POST: form = CaveForm(request.POST, instance=cave) ceFormSet = CaveAndEntranceFormSet(request.POST) - versionControlForm = VersionControlCommentForm(request.POST) + #versionControlForm = VersionControlCommentForm(request.POST) if form.is_valid() and ceFormSet.is_valid(): #print(f'! POST is valid. {cave}') cave = form.save(commit = False) @@ -397,8 +398,6 @@ def edit_entrance(request, caveslug=None, slug=None): slugname = cave.slug() + entletter.cleaned_data["entrance_letter"] else: slugname = cave.slug() - # Converting a PENDING cave to a real cave by saving this form - slugname = slugname.replace('-PENDING-', '-') entrance.cached_primary_slug = slugname entrance.filename = slugname + ".html" entrance.save() diff --git a/core/views/other.py b/core/views/other.py index 45d781c..c864593 100644 --- a/core/views/other.py +++ b/core/views/other.py @@ -32,7 +32,7 @@ todo = ''' - But how does this interact with troggle/logbooksdump.py ? -- deleted nefile() - check on deleted UploadFileForm using the editfile.html template which is about re-submitting +- deleted newfile() - check on deleted UploadFileForm using the editfile.html template which is about re-submitting a LBE aka TripReport ''' @@ -44,12 +44,16 @@ def todos(request, module): from troggle.core.TESTS.tests import todo as tests from troggle.core.views.logbooks import todo as viewlogbooks from troggle.parsers.logbooks import todo as parserslogbooks + from troggle.parsers.survex import todo as parserssurvex + from troggle.core.models.caves import todo as modelcaves from troggle.core.forms import todo as forms from troggle.core.templatetags.wiki_markup import todo as wiki tododict = {'views/other': todo, 'tests': tests, 'views/logbooks': viewlogbooks, 'parsers/logbooks': parserslogbooks, + 'parsers/survex': parserssurvex, + 'core/models/caves': modelcaves, 'core/forms': forms, 'core/templatetags/wiki_markup': wiki} return render(request,'core/todos.html', {'tododict': tododict}) diff --git a/parsers/caves.py b/parsers/caves.py index 8ccaa9d..ce2eaa7 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -23,9 +23,104 @@ entrances_xslug = {} caves_xslug = {} areas_xslug = {} +def do_pending_cave(k, url, area_1623): + ''' + default for a PENDING cave, should be overwritten in the db later if a real cave of the same name exists + in expoweb/cave_data/1623-"k".html + ''' + default_note = f"_Survex file found in loser repo but no description in expoweb


\n" + default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then " + default_note += f"
\n - search in the Expo for that year e.g. {k[0:4]} to find a " + default_note += f"relevant logbook entry, then
\n - " + default_note += f"click on 'Edit this cave' and copy the information you find in the survex file and the logbook" + default_note += f"
\n - " + default_note += f"When you Submit it will create a file file in expoweb/cave_data/ " + default_note += f"
\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. " + default_note += f"and then Submit to save it. NB your entrance info will not be visible after a reboot of the server until a programmer has edited parser/caves.py (python code) to remove the cave form the [pending] list. But it won't be lost." + + slug = "1623-" + k + + cave = Cave( + unofficial_number = k, + underground_description = "Pending cave write-up - creating as empty object. No XML file available yet.", + survex_file = "caves-1623/" + k + "/" + k +".svx", + url = url, + notes = default_note) + if cave: + cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key. + cave.area.add(area_1623[0]) + cave.save() + message = f" ! {k:12} {cave.underground_description}" + DataIssue.objects.create(parser='caves', message=message, url=url) + print(message) + + try: # Now create a cave slug ID + cs = CaveSlug.objects.update_or_create(cave = cave, + slug = slug, primary = False) + except: + message = f" ! {k:11s} PENDING cave SLUG create failure" + DataIssue.objects.create(parser='caves', message=message) + print(message) + else: + message = f' ! {k:11s} PENDING cave create failure' + DataIssue.objects.create(parser='caves', message=message) + print(message) + + ent = Entrance( + name = k, + entrance_description = "Dummy entrance: auto-created when registering a new cave." + + "This file WILL NOT BE LOADED while the cave is in the pending[] list in parsers/caves.py", + marking = '?') + if ent: + ent.save() # must save to have id before foreign keys work. + try: # Now create a entrance slug ID + es = EntranceSlug.objects.update_or_create(entrance = ent, + slug = slug, primary = False) + except: + message = f" ! {k:11s} PENDING entrance create failure" + DataIssue.objects.create(parser='caves', message=message) + print(message) + + # Now we will actually write this default entrance slugfile. Yes it's naughty, but an extra + # entrance file is less hassle than a missing one! And people always forget to save the entrance file + # when they are saving the edited cave file. + + #This WILL then casue an error when the parser then tries to import this file and we get an import failure because + # the entrance exists in the db. + ent.cached_primary_slug = slug + ent.filename = slug + ".html" + ent.save() + if not ent.get_file_path().is_file(): + # don't overwrite if by some chnace it exists + # but the pre-existing file won't be parsed later until the cave is removed from the pending list + ent.writeDataFile() + else: + message = f" ! {k:11s} PENDING cave SLUG '{slug}' create failure" + DataIssue.objects.create(parser='caves', message=message) + print(message) + + try: + ceinsts = CaveAndEntrance.objects.update_or_create(cave = cave, entrance_letter = "", entrance = ent) + for ceinst in ceinsts: + if str(ceinst) == str(cave): # magic runes... why is the next value a Bool? + ceinst.cave = cave + ceinst.save() + break + except: + message = f" ! {k:11s} PENDING entrance + cave UNION create failure '{cave}' [{ent}]" + DataIssue.objects.create(parser='caves', message=message) + print(message) + + def readcaves(): '''Reads the xml-format HTML files in the EXPOWEB repo, not from the loser repo. ''' + # For those caves which do not have cave_data/1623-xxx.html XML files even though they exist and have surveys + pending = ["2007-06", "2009-02", + "2010-06", "2010-07", "2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06", + "2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888", + "2018-pf-01", "2018-pf-02"] + with transaction.atomic(): print(" - Deleting Caves and Entrances") Cave.objects.all().delete() @@ -33,7 +128,6 @@ def readcaves(): # Clear the cave data issues and the caves as we are reloading DataIssue.objects.filter(parser='caves').delete() DataIssue.objects.filter(parser='entrances').delete() - area_1623 = Area.objects.update_or_create(short_name = "1623", parent = None) # This seems to return a tuple, not a single object! i.e. (, True) @@ -44,64 +138,26 @@ def readcaves(): print (" - Setting pending caves") # Do this first, so that these empty entries are overwritten as they get properly created. - # For those caves which do not have cave_data/1623-xxx.html XML files even though they exist and have surveys - pending = ["2007-06", "2009-01", "2009-02", - "2010-06", "2010-07", "2012-ns-01", "2012-ns-02", "2010-04", "2012-ns-05", "2012-ns-06", - "2012-ns-07", "2012-ns-08", "2012-ns-12", "2012-ns-14", "2012-ns-15", "2014-bl888", - "2018-pf-01", "2018-pf-02", "haldenloch"] for k in pending: - url = "1623/" + k + url = "1623/" + k # Note we are not appending the .htm as we are modern folks now. try: - # default for a PENDING cave, ooverwritten if a real cave exists - cave = Cave( - unofficial_number = k, - # official_name = "", - underground_description = "Pending cave write-up - creating as empty object. No XML file available yet.", - survex_file = "caves-1623/" + k + "/" + k +".svx", - url = url, - notes=f"_Survex file found in loser repo but no description in expoweb


\n"+ - f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then " + - f"
\n - search in the Expo for that year e.g. {k[0:4]} to find a relevant logbook entry, then \n - " + - f"click on 'Edit this cave' and copy the information you find in the survex file and the logbook"+ - f"
\n - " + - f"When you Submit it will create a file file in expoweb/cave_data/ " + - f"
\n - but you have not finished yet. You MUST go and create the entrance: click on New Entrance. Then this will no longer be 'Pending' once the flag has been removed from the code") - if cave: - cave.save() # must save to have id before foreign keys work. This is also a ManyToMany key. - #print(f' ! - READ CAVES: cave {k} {cave}') - cave.area.add(area_1623[0]) - cave.save() - message = f" ! {k:12} {cave.underground_description}" - DataIssue.objects.create(parser='caves', message=message, url=url) - print(message) - - try: # Now create a cave slug ID - cs = CaveSlug.objects.update_or_create(cave = cave, - slug = "1623-" + k, - primary = False) - except: - message = " ! {:11s} {} PENDING cave slug create failure".format(k) - DataIssue.objects.create(parser='caves', message=message) - print(message) - else: - message = f' ! {k:11s} PENDING cave slug create failure' - DataIssue.objects.create(parser='caves', message=message) - print(message) - - + do_pending_cave(k, url, area_1623) except: - message = " ! Error. Cannot create pending cave, pending-id:{}".format(k) + message = " ! Error. Cannot create pending cave and entrance, pending-id:{}".format(k) DataIssue.objects.create(parser='caves', message=message) print(message) raise with transaction.atomic(): - print(" - Reading Entrances from entrance descriptions xml files") print(" - settings.CAVEDESCRIPTIONS: ", settings.CAVEDESCRIPTIONS) + print(" - Reading Entrances from entrance descriptions xml files") for filename in next(os.walk(settings.ENTRANCEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files if filename.endswith('.html'): - readentrance(filename) + if Path(filename).stem[5:] in pending: + print(f'Skipping pending entrance dummy file <{filename}>') + else: + readentrance(filename) print(" - Reading Caves from cave descriptions xml files") for filename in next(os.walk(settings.CAVEDESCRIPTIONS))[2]: #Should be a better way of getting a list of files @@ -182,13 +238,14 @@ def readentrance(filename): primary = primary) except: # need to cope with duplicates - print(" ! FAILED to get only one ENTRANCE when updating using: "+filename) + message = f" ! FAILED to get precisely one ENTRANCE when updating using: cave_entrance/{filename}" + DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/') kents = EntranceSlug.objects.all().filter(entrance = e, slug = slug, primary = primary) for k in kents: message = " ! - DUPLICATE in db. entrance:"+ str(k.entrance) + ", slug:" + str(k.slug()) - DataIssue.objects.create(parser='caves', message=message) + DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/') print(message) for k in kents: if k.slug() != None: @@ -300,7 +357,7 @@ def readcave(filename): primary = primary) caves_xslug[slug] = cs except Exception as ex: - message = " ! Cave update/create failure : %s, skipping file %s\nException: %s" % (slug, context, ex.__class__) + message = " ! Cave update/create failure : %s, skipping file cave_data/%s with exception\nException: %s" % (slug, context, ex.__class__) DataIssue.objects.create(parser='caves', message=message) print(message) diff --git a/parsers/survex.py b/parsers/survex.py index a073686..1f0f036 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -23,6 +23,10 @@ from troggle.core.models.survex import SurvexPersonRole, ScansFolder, SurvexDire It does also NOT scan the Loser repo for all the svx files - though it should ! ''' +todo = '''Also walk the entire tree in the :loser: repo looking for unconnected survex files +- add them to the system so that they can be reported-on +- produce a parser report and create a troggle report page (some are OK, e.g. futility series replaced by ARGE survey in 115) +''' survexblockroot = None ROOTBLOCK = "rootblock" METRESINFEET = 3.28084 @@ -658,7 +662,7 @@ class LoadingSurvex(): #print("\n"+message) #print("\n"+message,file=sys.stderr) return - message = " ! {} is not a cave. (while creating '{}' sfile & sdirectory)".format(headpath, includelabel) + message = f" ! {headpath} is not a fully-registered cave. (while creating '{includelabel}' sfile & sdirectory in survex parsing)" print("\n"+message) print("\n"+message,file=sys.stderr) DataIssue.objects.create(parser='survex', message=message)