From 36995ec05169974139fbfd6b768e6e935d97ab46 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Tue, 20 Sep 2022 02:36:40 +0300 Subject: [PATCH] handling survex files not linked in completely --- core/views/scans.py | 20 ++++++------ core/views/uploads.py | 71 +++++++++++++++++++++++++------------------ parsers/scans.py | 7 +++-- 3 files changed, 56 insertions(+), 42 deletions(-) diff --git a/core/views/scans.py b/core/views/scans.py index f641423..80b334d 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -35,6 +35,9 @@ add this file in to the todo list thinggy. def populatewallet(w): '''Copy survex data here just for display, not permanently + + Only gets data from the survex file when it was parsed on import.. + so doesn't work if there is no *ref value ''' survexpeople = [] blocks = SurvexBlock.objects.filter(scanswallet = w) @@ -76,16 +79,15 @@ def caveifywallet(w): w.displaynames = blocknames def fillblankpeople(w): - # this isn't working..? why? + # this isn't working..? why? Because it needs a *ref and an import wp = w.people() - if not wp: # an -empty list - populatewallet(w) - else: - w.persons = wp - if len(wp) == 1: - nobody = wp[0].lower() - if nobody == 'unknown' or nobody == 'nobody' or nobody == ' ' or nobody == '': - populatewallet(w) + w.persons = wp + if len(wp) == 1: + # print(f' - {wp=}') + nobody = wp[0].lower() + if nobody == 'unknown' or nobody == 'nobody' or nobody == ' ' or nobody == '': + print(f' - {wp=} {nobody=}') + populatewallet(w) def fillblankothers(w): earliest = datetime.datetime.now().date() diff --git a/core/views/uploads.py b/core/views/uploads.py index 8c8721b..f2fa85a 100644 --- a/core/views/uploads.py +++ b/core/views/uploads.py @@ -142,7 +142,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): complaints.append("A date is mandatory. No data can be updated or edited unless you specify a date. Look in the survex file if there is one.") # People - if waldata["people"]==["NOBODY"] or waldata["people"]==["Unknown"]: + if not waldata["people"] or waldata["people"]==["NOBODY"] or waldata["people"]==["Unknown"] or waldata["people"]==[""]: complaints.append("Someody must have done this. Look in the survex file, or in the logbook entries for this date, for the people who created this data.") # survex, but get_ticks has already done much of this ?? @@ -158,11 +158,20 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): sx = sx + ".svx" svxfiles.append(sx) if not (Path(settings.SURVEX_DATA) / sx).is_file(): - file_complaint = f"{wallet} Incorrect survex file name in {wallet} wallet data. {sx} was not found in LOSER repo" + file_complaint = f"{wallet} Incorrect survex file name. {sx} was not found in LOSER repo" complaints.append(file_complaint) message = f"! {file_complaint}" print(message) DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) # set URL to this wallet folder + else: + try: + svxfile = SurvexFile.objects.get(path=sx) + except: + file_complaint = f"{wallet} Survex file name {sx} was not imported via a *include statement." + complaints.append(file_complaint) + message = f"! {file_complaint}" + print(message) + DataIssue.objects.update_or_create(parser='scans', message=message, url=wurl) # set URL to this wallet folder if waldata["survex not required"] and waldata["survex file"] != "": survex_complaint = "Survex is stated as not required and yet there is a survex file!" @@ -208,7 +217,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl): if type(caveid) is list: for i in caveid: i = i.replace("/","-") - caveobject = getCave(i) # only th elast one gets recorded.. ouch. + caveobject = getCave(i) # only the last one gets recorded.. ouch. else: caveid = caveid caveobject = getCave(caveid) @@ -537,34 +546,36 @@ def scanupload(request, path=None): #print(f' - {path=}') try: svxfile = SurvexFile.objects.get(path=path) - except: - message = f'Specified survex file not found - database apparently empty, you probably need to do a full re-import of all data.' - return render(request, 'errors/generic.html', {'message': message}) - print(f' - {svxfile=}') - if svxfile.cave: - caves.append(svxfile.cave) - caverefs.append(svxfile.cave.reference()) - blocks = SurvexBlock.objects.filter(survexfile= svxfile) - for b in blocks: - print(f' - - {b=} {b.scanswallet=} {b.date=}') - if b.scanswallet: - refs.append(b.scanswallet) - if b.scanswallet.walletname == wallet: - if b.date: - dates.append(b.date) - if b.name != b.title: - names.append(str(b.name) + "|" + str(b.title)) - else: - names.append(str(b.name)) - # we can use the people, across all blocks that have this *ref - QSpeople = SurvexPersonRole.objects.filter(survexblock=b) - print(f' - - {QSpeople=}') - for p in QSpeople: - print(f' - - {p.personname} ') - team.append(p.personname) - # else: - # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}') + print(f' - {svxfile=}') + if svxfile.cave: + caves.append(svxfile.cave) + caverefs.append(svxfile.cave.reference()) + blocks = SurvexBlock.objects.filter(survexfile= svxfile) + for b in blocks: + print(f' - - {b=} {b.scanswallet=} {b.date=}') + if b.scanswallet: + refs.append(b.scanswallet) + if b.scanswallet.walletname == wallet: + if b.date: + dates.append(b.date) + if b.name != b.title: + names.append(str(b.name) + "|" + str(b.title)) + else: + names.append(str(b.name)) + # we can use the people, across all blocks that have this *ref + QSpeople = SurvexPersonRole.objects.filter(survexblock=b) + print(f' - - {QSpeople=}') + for p in QSpeople: + print(f' - - {p.personname} ') + team.append(p.personname) + # else: + # print(f' - Wallet not matching *ref {b.scanswallet=} {wallet}') + except: + message = f'Specified survex file not found - database may be empty, or this survex file is not *included anywhere.' + # return render(request, 'errors/generic.html', {'message': message}) + pass + if dates: waldata["date"] = min(dates).isoformat() print(f' - - {team=} ') diff --git a/parsers/scans.py b/parsers/scans.py index f9ec449..60b3d17 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -21,10 +21,10 @@ from troggle.core.views.scans import datewallet ''' contentsjson = "contents.json" -#indexhtml = "walletindex.html" + git = settings.GIT -# to do: create a 'low priority' field, so that any such wallet does not appear in summary reports +# to do: Actually read all the JSON files and set the survex file field appropriately! def CheckEmptyDate(wallet): @@ -107,7 +107,7 @@ def load_all_scans(): for walletname, fpath, fisdir in GetListDir(fpath): if fisdir: wallet = Wallet(fpath=fpath, walletname=walletname) - # this is where we should load the contents.json for people so we can report on them later + # this is where we should load the contents.json for people, cave and date so we can report on them later # this is where we should record the year explicitly # line 347 of view/uploads.py and needs refactoring for loading contentsjson CheckEmptyDate(wallet) @@ -128,6 +128,7 @@ def load_all_scans(): if Path(walletpath, contentsjson).is_file(): walletname = walletpath.name wallet, created = Wallet.objects.update_or_create(walletname=walletname) + # should now also load the json and use it ! check &ref is correct or missing too if created: print(f"\n{walletname} created: only JSON, no actual uploaded scan files.", end=' ') CheckEmptyDate(wallet)