From f1b206ad34ab06772d648ced3ae0995538e26dd6 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Tue, 22 Mar 2022 02:22:15 +0000 Subject: [PATCH] fixing bugs after wookey session --- core/forms.py | 3 ++- core/views/caves.py | 27 ++------------------------ core/views/expo.py | 9 ++++++--- core/views/survex.py | 2 +- parsers/caves.py | 4 ++-- parsers/survex.py | 45 +++++++++++++++++++++++--------------------- templates/cave.html | 4 ++-- urls.py | 4 ++-- 8 files changed, 41 insertions(+), 57 deletions(-) diff --git a/core/forms.py b/core/forms.py index 08f7e6c..cd40822 100644 --- a/core/forms.py +++ b/core/forms.py @@ -11,7 +11,8 @@ from django.contrib.admin.widgets import AdminDateWidget from troggle.core.models.troggle import Person, PersonExpedition, Expedition from troggle.core.models.caves import Cave, LogbookEntry, QM, Entrance, CaveAndEntrance -'''These are all the Forms used by troggle +'''These are all the class-based Forms used by troggle. +There are other, simpler, upload forms in view/uploads.py Some are not used and need renovating or destroying. ''' diff --git a/core/views/caves.py b/core/views/caves.py index 9e40c51..325c593 100644 --- a/core/views/caves.py +++ b/core/views/caves.py @@ -267,6 +267,7 @@ def cavepage(request, karea, subpath): except Cave.DoesNotExist: # probably a link to text or an image e.g. 1623/161/l/rl89a.htm i.e. an expoweb page + # cannot assume that this is a simple cave page, for a cave we don't know. return expo.expopage(request, str(kpath)) except Cave.MultipleObjectsReturned: caves = Cave.objects.filter(url = kpath) @@ -276,20 +277,6 @@ def cavepage(request, karea, subpath): message = f'Failed to find cave: {kpath}' return render(request,'errors/generic.html', {'message': message}) -def cave(request, cave_id='', offical_name=''): - '''Displays a cave description page - accesssed by a fairly random id which might be anything - ''' - try: - cave=getCave(cave_id) - except MultipleObjectsReturned: - caves = Cave.objects.filter(kataster_number=cave_id) - return render(request, 'svxcaveseveral.html', {'settings': settings, "caves":caves }) # not the right template, needs a specific one - except ObjectDoesNotExist: - return render(request, 'errors/svxcavesingle404.html', {'settings': settings, "cave":cave_id }) - except: - return render(request, 'errors/svxcavesingle404.html', {'settings': settings }) - return rendercave(request, cave, cave.slug(), cave_id=cave_id) def caveEntrance(request, slug): @@ -303,23 +290,13 @@ def caveEntrance(request, slug): else: return render(request,'cave_entrances.html', {'cave': cave}) -# def caveDescription(request, slug): - # try: - # cave = Cave.objects.get(caveslug__slug = slug) - # except: - # return render(request,'errors/badslug.html', {'badslug': slug}) - - # if cave.non_public and settings.PUBLIC_SITE and not request.user.is_authenticated: - # return render(request,'nonpublic.html', {'instance': cave}) - # else: - # return render(request,'cave_uground_description.html', {'cave': cave}) - @login_required_if_public def edit_cave(request, slug=None): '''This is the form that edits all the cave data and writes out an XML file in the :expoweb: repo folder The format for the file being saved is in templates/dataformat/cave.xml It does save the data into into the database directly, not by parsing the file. + It does NOT yet commit to the git repo ''' message = "" if slug is not None: diff --git a/core/views/expo.py b/core/views/expo.py index c6ef01b..b6644f3 100644 --- a/core/views/expo.py +++ b/core/views/expo.py @@ -125,9 +125,12 @@ def expowebpage(request, expowebpath, path): print(f' - 404 error in expowebpage() {path}') return render(request, 'pagenotfound.html', {'path': path}, status="404") - with open(os.path.normpath(expowebpath / path), "r") as o: - html = o.read() - + try: + with open(os.path.normpath(expowebpath / path), "r") as o: + html = o.read() + except: + return HttpResponse(default_head + html + '

UTF-8 Parsing Failue:
Page could not be parsed using UTF-8:
failure detected in expowebpage in views.expo.py

Please edit this :expoweb: page to replace dubious umlauts with HTML entitiies in the expected code. ]*)>(.*)<\s*/head\s*>(.*)<\s*body([^>]*)>(.*)<\s*/body\s*>(.*)', html, re.DOTALL + re.IGNORECASE) if m: preheader, headerattrs, head, postheader, bodyattrs, body, postbody = m.groups() diff --git a/core/views/survex.py b/core/views/survex.py index c0f63ef..21603bc 100644 --- a/core/views/survex.py +++ b/core/views/survex.py @@ -109,7 +109,7 @@ class SvxForm(forms.Form): filename = forms.CharField(widget=forms.TextInput(attrs={"readonly":True})) datetime = forms.DateTimeField(widget=forms.TextInput(attrs={"readonly":True})) outputtype = forms.CharField(widget=forms.TextInput(attrs={"readonly":True})) - code = forms.CharField(widget=forms.Textarea(attrs={"cols":150, "rows":36})) + code = forms.CharField(widget=forms.Textarea(attrs={"cols":140, "rows":36})) template = False diff --git a/parsers/caves.py b/parsers/caves.py index 314d331..21f2b3f 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -417,13 +417,13 @@ def readcave(filename): if survex_file[0]: if not (Path(SURVEX_DATA) / survex_file[0]).is_file(): message = f' ! {slug:12} survex filename does not exist :LOSER:"{survex_file[0]}" in "{filename}"' - DataIssue.objects.create(parser='caves', message=message, url=f'/cave/{slug}/edit/') + DataIssue.objects.create(parser='caves', message=message, url=f'/{slug}_cave_edit/') print(message) if description_file[0]: # if not an empty string message = f' - {slug:12} complex description filename "{description_file[0]}" inside "{CAVEDESCRIPTIONS}/{filename}"' - DataIssue.objects.create(parser='caves ok', message=message, url=f'/cave/{slug}/edit/') + DataIssue.objects.create(parser='caves ok', message=message, url=f'{slug}/edit/') print(message) if not (Path(EXPOWEB) / description_file[0]).is_file(): diff --git a/parsers/survex.py b/parsers/survex.py index 127bb70..1f93a4e 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -114,7 +114,7 @@ class LoadingSurvex(): rx_names = re.compile(r'(?i)names') rx_flagsnot= re.compile(r"not\s") rx_linelen = re.compile(r"[\d\-+.]+$") - instruments = "(waiting_patiently|slacker|Useless|nagging|unknown|Inst|instrument|rig|rigger|rigging|helper|something| compass|comp|clino|Notes|sketch|book|Tape|Dog|Pics|photo|drawing|Helper|GPS|Disto|Distox|Distox2|topodroid|point|Consultant|nail|polish|varnish|bitch|monkey|PowerDrill|drill)" + instruments = "(waiting_patiently|slacker|Useless|nagging|unknown|Inst|instrument|rig|rigger|rigging|helper|something| compass|comp|clino|Notes|sketch|book|Tape|Dog|Pics|photo|drawing|Helper|GPS|Disto|Distox|Distox2|topodroid|point|Consultant|nail|polish|nail_polish_bitch|nail_polish_monkey|varnish|nail_polish|nail_varnish|bitch|monkey|PowerDrill|drill)" rx_teammem = re.compile(r"(?i)"+instruments+"?(?:es|s)?\s+(.*)"+instruments+"?(?:es|s)?$") rx_person = re.compile(r"(?i) and | / |, | & | \+ |^both$|^none$") rx_qm = re.compile(r'(?i)^\s*QM(\d)\s+?([a-dA-DxX])\s+([\w\-]+)\.(\d+)\s+(([\w\-]+)\.(\d+)|\-)\s+(.+)$') @@ -448,27 +448,27 @@ class LoadingSurvex(): def LoadSurvexRef(self, survexblock, args): #print(self.insp+ "*REF ---- '"+ args +"'") - + url=f'/survexfile/{survexblock.survexfile.path}' # *REF but also ; Ref years from 1960 to 2039 refline = self.rx_ref_text.match(args) if refline: # a textual reference such as "1996-1999 Not-KH survey book pp 92-95" - # print(self.insp+ "*REF quoted text so ignored:"+ args) + print(self.insp+ "*REF quoted text so ignored:"+ args) return if len(args)< 4: message = " ! Empty or BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path) print((self.insp+message)) - DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message, url=url) return argsgps = self.rx_argsref.match(args) if argsgps: yr, letterx, wallet = argsgps.groups() else: - message = " ! BAD *REF statement '{}' in '{}'".format(args, survexblock.survexfile.path) + message = " ! Wallet *REF '{}' in '{}' ".format(args, survexblock.survexfile.path) print(self.insp+message) - DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message, url=url) return if not letterx: @@ -480,31 +480,31 @@ class LoadingSurvex(): if not (int(yr)>1960 and int(yr)<2039): message = " ! Wallet year out of bounds {yr} '{refscan}' {survexblock.survexfile.path}" print((self.insp+message)) - DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message, url=url) refscan = "%s#%s%s" % (yr, letterx, wallet) try: if int(wallet)>100: message = " ! Wallet *REF {} - too big in '{}'".format(refscan, survexblock.survexfile.path) print((self.insp+message)) - DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message, url=url) except: - message = " ! Wallet *REF {} - not numeric in '{}'".format(refscan, survexblock.survexfile.path) + message = " ! Wallet *REF {} - not numeric in '{}' -- parsing continues".format(refscan, survexblock.survexfile.path) print((self.insp+message)) - DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message, url=url) manywallets = Wallet.objects.filter(walletname=refscan) if manywallets: survexblock.scanswallet = manywallets[0] # this is a ForeignKey field print(manywallets[0]) survexblock.save() if len(manywallets) > 1: - message = " ! Wallet *REF {} - {} scan folders from DB search in {}".format(refscan, len(manywallets), survexblock.survexfile.path) + message = " ! Wallet *REF {} - {} scan folders from DB search in {} -- parsing continues".format(refscan, len(manywallets), survexblock.survexfile.path) print((self.insp+message)) - DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message, url=url) else: - message = " ! Wallet *REF '{}' - NOT found in DB search '{}'".format(refscan, survexblock.survexfile.path) + message = " ! Wallet *REF '{}' - NOT found in DB search '{}' -- parsing continues".format(refscan, survexblock.survexfile.path) print((self.insp+message)) - DataIssue.objects.create(parser='survex', message=message) + DataIssue.objects.create(parser='survex', message=message, url=url) def LoadSurvexQM(self, survexblock, qmline): insp = self.insp @@ -668,7 +668,7 @@ class LoadingSurvex(): return self.svxdirs[headpath.lower()] def ReportNonCaveIncludes(self, headpath, includelabel): - """Ignore surface, kataser and gps *include survex files + """Ignore surface, kataser and gpx *include survex files """ if headpath in self.ignorenoncave: #message = f" - {headpath} is (while creating '{includelabel}' sfile & sdirectory)" @@ -677,11 +677,12 @@ class LoadingSurvex(): return for i in self.ignoreprefix: if headpath.startswith(i): - #message = f" - {headpath} starts with (while creating '{includelabel}' sfile & sdirectory)" - #print("\n"+message) + message = f" - {headpath} starts with (while creating '{includelabel}' sfile & sdirectory)" + print("\n"+message) #print("\n"+message,file=sys.stderr) return - message = f" ! Error: '{headpath}' not a cave or in the ignore list of surface surveys. (while creating '{includelabel}')" + message = f" ! Error: '{headpath}' not a cave or in the ignore list of surface surveys. (while creating '{includelabel}' in db)" + # getting this triggered for gpx/2018 but not for gpx/2017 ?! print("\n"+message) print("\n"+message,file=sys.stderr) DataIssue.objects.create(parser='survex', message=message) @@ -725,7 +726,7 @@ class LoadingSurvex(): if cave: newdirectory.cave = cave newfile.cave = cave - #print("\n"+str(newdirectory.cave),file=sys.stderr) + print("\n"+str(newdirectory.cave),file=sys.stderr) else: # probably a surface survey self.ReportNonCaveIncludes(headpath, svxid) @@ -1214,8 +1215,9 @@ class LoadingSurvex(): sp = subprocess.run([settings.CAVERN, "--log", f'--output={outputdir}', f'{fullpath}.svx'], capture_output=True, check=False, text=True) if sp.returncode != 0: - message = f' ! Error running {settings.CAVERN}: {fullpath}' - DataIssue.objects.create(parser='entrances', message=message) + message = f' ! Error running {settings.CAVERN}: {fullpath}' + url = f'/survexfile{fullpath}.svx'.replace(settings.SURVEX_DATA, "") + DataIssue.objects.create(parser='xEntrances', message=message, url=url) print(message) print(f'stderr:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode)) self.caverncount += 1 @@ -1408,6 +1410,7 @@ def LoadSurvexBlocks(): DataIssue.objects.filter(parser='survexleg').delete() DataIssue.objects.filter(parser='survexunits').delete() DataIssue.objects.filter(parser='entrances').delete() + DataIssue.objects.filter(parser='xEntrances').delete() survexfileroot = MakeSurvexFileRoot() # this next makes a block_object assciated with a file_object.path = SURVEX_TOPNAME diff --git a/templates/cave.html b/templates/cave.html index d089d5b..afc916f 100644 --- a/templates/cave.html +++ b/templates/cave.html @@ -1,6 +1,6 @@ {% extends "cavebase.html" %} - {% block extraheaders %} + {% if cave.survex_file %}