import datetime from pathlib import Path from urllib.parse import unquote as urlunquote from django.conf import settings from django.http import HttpResponse from django.shortcuts import render from troggle.core.models.caves import GetCaveLookup from troggle.core.models.survex import SingleScan, SurvexBlock, SurvexPersonRole from troggle.core.models.wallets import Wallet from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition from troggle.core.views.expo import getmimetype from troggle.parsers.caves import add_cave_to_pending_list from troggle.parsers.people import GetPersonExpeditionNameLookup from troggle.parsers.survex import set_walletdate """one of these views serves files as binary blobs, and simply set the mime type based on the file extension, as does the urls.py dispatcher which sends them here. Here they should actually have the filetype checked by looking inside the file before being served. need to check if inavlid query string is invalid, or produces multiple replies and render a user-friendly error page. Note that caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the manywallets dict. TODO cave for a wallet - just gets the last one, randomly. SHould make this a list or many:many ideally -- add the participants on an explicit wallet list to .slugpeople so that they get proper URL-linked on the per-person wallet report, and do the same thing for per-cave and per-year wallet reports add this file in to the todo list thinggy. """ def populatewallet(w): """Copy survex data here just for display, not permanently Only gets data from the survex file when it was parsed on import, or edited (& thus parsed) online, so doesn't work if there was no *ref value """ slugpeople = set() blocks = SurvexBlock.objects.filter(scanswallet=w) for b in blocks: for personrole in b.survexpersonrole_set.all(): slugpeople.add(personrole.person) # Person objects, not the names anymore w.slugpeople = slugpeople def caveifywallet(w): """Gets the cave from the list of survex files, only selects one of them though. Only used for display. FIX THIS to display many caves """ # print(f' - Caveify {w=}') blocknames = [] blocks = SurvexBlock.objects.filter(scanswallet=w) for b in blocks: # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath if b.survexfile.cave: w.caveobj = ( b.survexfile.cave ) # just gets the last one, randomly. SHould make this a list or many:many ideally w.cave = w.caveobj if b.name: blocknames.append(b.name) if w.name(): w.displaynames = [w.name()] else: w.displaynames = blocknames def fillblankpeople(w): """Find people attached to a wallet via the survex files if no one explicitly attached. the JSON string which may OR MAY NOT be formatted as a list. w.slugpeople is from the survexfiles w.persons is from the explicit list of peoples' names in the wallet The template choses how to display them. """ def nobody(wplist): if len(wplist) > 1: return False nobod = wp[0].lower() if nobod == "unknown" or nobod == "nobody" or nobod == " " or nobod == "": return True else: return False wp = w.people() # just a list of names as strings, direct from JSON. if not isinstance(wp, list): # might be None print(f"{w} NOT A LIST {type(wp)}: {wp}") populatewallet(w) return if not wp: # e.g. empty list # print(f"{w} {wp=}") populatewallet(w) # sets w.slugpeople return if nobody(wp): populatewallet(w) # sets w.slugpeople else: w.persons = parse_name_list(w) populatewallet(w) # sets w.slugpeople if hasattr(w, "slugpeople"): w.persons = w.persons.difference(w.slugpeople) return def is_cave(wallet, id): if not id: return False Gcavelookup = GetCaveLookup() id = id.strip("' []'") if id in Gcavelookup: return True else: # Historic wallets used just 2 or 3 digits and were all 1623 area. So, just for these wallets, # assume it is 1623-xxx if f"1623-{id}" in Gcavelookup: print(f" - Should modify wallet {wallet} to use 1623- prefix for cave <{id}>") Gcavelookup[id] = Gcavelookup[f"1623-{id}"] # restoring an ambiguous alias, BAD idea. return True else: print(f" - Wallet {wallet} Failed to find cave object from id <{id}>") if id.lower() != "unknown" and id != "": print(f" - adding <{id}> to pendingcaves DataIssues") add_cave_to_pending_list(id, wallet, f"Wallet {wallet} - Could not find id <{id}>") return False def fillblankothers(w): """This is on the way to having a many:many relationship between Caves and Wallets """ if not w.walletdate: set_walletdate(w) Gcavelookup = GetCaveLookup() wcaveid = w.cave() if not wcaveid or wcaveid == "": caveifywallet(w) else: if type(wcaveid) == list: for i in wcaveid: i = i.strip("' []'") if is_cave(w,i): w.caveobj = Gcavelookup[i] # just sets it to the last one found. nasty. bug waiting to happen elif wcaveid.find(',') != -1: # it's a list of cave ids as a string ids = wcaveid.split(',') for i in ids: i = i.strip("' []'") if is_cave(w,i): w.caveobj = Gcavelookup[i] # just sets it to the last one found. nasty. bug waiting to happen else: if is_cave(w,wcaveid): w.caveobj = Gcavelookup[wcaveid.strip("' []'")] def fixsurvextick(w, ticks): ticks["S"] = w.fixsurvextick(ticks["S"]) def parse_name_list(w): """Given a list of strings, which are names, and the year of an expo, return a set of Persons """ namelist = w.people() peeps = set() expo = Expedition.objects.get(year=w.year()) crew = GetPersonExpeditionNameLookup(expo) for n in namelist: # if n.lower().startswith("lydia"): # print(f"{w} {n=} ") # for x in crew: # if x.lower()==n.lower(): # print(f"{w} {n=} {x=}") if n.lower() in crew: peeps.add(crew[n.lower()].person) else: if n.startswith("*"): #ignore people flagged as guests or not-expo anyway, such as ARGE pass nobod = n.lower() if nobod == "unknown" or nobod == "nobody" or nobod == " " or nobod == "": pass else: wurl = f"/walletedit/{w.walletname.replace('#',':')}" message = f"{w} name '{n.lower()}' NOT found in GetPersonExpeditionNameLookup({w.year()}) ?!" print(message) DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) return peeps def walletslistperson(request, slug): """Page which displays a list of all the wallets for a specific person """ # Remember that 'personexpedition__expedition' is interpreted by Django to mean the # 'expedition' object which is connected by a foreign key to the 'personexpedition' # object, which is a field of the PersonLogEntry object: # PersonLogEntry.objects.filter(personexpedition__expedition=expo) def personwallet(p): manywallets = set() # Get the persons from the survexblocks on the survexfiles attached to the wallet directly sps = SurvexPersonRole.objects.filter(person=p) for sp in sps: w = sp.survexblock.scanswallet if w: manywallets.add(w) # Now read the text strings in the list of wallet people and identify the person pes = PersonExpedition.objects.filter(person=p) for person_expo in pes: expo = person_expo.expedition year = expo.year crew = GetPersonExpeditionNameLookup(expo) wallets = Wallet.objects.filter(walletyear__year=year) for w in wallets: if w in manywallets: # we already know this is a wallet we need to report on continue for n in w.people(): # if n.lower().startswith("lydia"): # print(f"{w} {n=} ") # for x in crew: # if x.lower()==n.lower(): # print(f"{w} {n=} {x=}") if n.lower() in crew: if crew[n.lower()] == person_expo: manywallets.add(w) # print(f"{w} Found a non-survex wallet for {person_expo}") else: if n.startswith("*"): #ignore people flagged as guests or not-expo anyway, such as ARGE pass nobod = n.lower() if nobod == "unknown" or nobod == "nobody" or nobod == " " or nobod == "": pass else: wurl = f"/walletedit/{w.walletname.replace('#',':')}" message = f"{w} name '{n.lower()}' NOT found in GetPersonExpeditionNameLookup({year}) ?!" print(message) DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) for w in manywallets: fillblankpeople(w) fillblankothers(w) w.ticks = w.get_ticks() # the complaints in colour form fixsurvextick(w, w.ticks) return manywallets # print("-walletslistperson") p = Person.objects.get(slug=slug) manywallets = personwallet(p) expeditions = Expedition.objects.all() print("--") return render( request, "personwallets.html", {"manywallets": manywallets, "settings": settings, "person": p, "expeditions": expeditions}, ) def setwalletsdates(): """This sets all the undated wallets, but they should already all be dated on import or on edit""" wallets = Wallet.objects.filter(walletdate=None) print(f"undated wallets: {len(wallets)}") for w in wallets: w.walletdate = w.date() w.save() def walletslistyear(request, year): """Page which displays a list of all the wallets in a specific year. We have a field .walletyear, which we set on import. """ def ticksyearwallet(year): manywallets = [] wallets = Wallet.objects.filter(walletyear__year=year) for w in wallets: manywallets.append(w) fillblankpeople(w) fillblankothers(w) w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc fixsurvextick(w, w.ticks) return manywallets # print("-walletslistyear") if year < 1976 or year > 2050: return render(request, "errors/generic.html", {"message": "Year out of range. Must be between 1976 and 2050"}) # return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'}) year = str(year) manywallets = ticksyearwallet(year) expeditions = Expedition.objects.all() #bad Django style expedition = expeditions.filter(year=year) length_ug = 0.0 for w in manywallets: for sb in w.survexblock_set.all(): length_ug += sb.legslength print("--") return render( request, "yearwallets.html", { "manywallets": manywallets, "settings": settings, "year": year, "expeditions": expeditions, "expedition": expedition, "length_ug": length_ug, }, ) def cavewallets(request, caveid): """Returns all the wallets for just one cave""" print("-cavewallets") Gcavelookup = GetCaveLookup() if caveid in Gcavelookup: cave = Gcavelookup[caveid] else: return render(request, "errors/badslug.html", {"badslug": f"{caveid} - from cavewallets()"}) # remove duplication. Sorting is done in the template # But this only gets wallets which have survex files attached.. wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave)) # all the ones without a survexblock attached via a *ref, search for match in JSON zilchwallets = set(Wallet.objects.exclude(survexblock__survexfile__cave=cave)) for z in zilchwallets: zcaveid = z.cave() if zcaveid: cleanid = str(zcaveid).strip("' []'") if cleanid.find(',') != -1: # it's a list of cave ids wurl = f"/walletedit/{z.walletname.replace('#',':')}" message = f" ! In {z.walletname} cavewallets, we do not handle lists of cave ids yet '{cleanid}'" print(message) DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl) # it's a list of cave ids as a string. Identify any orphan caves hidden here ids = cleanid.split(',') for i in ids: i = i.strip("' []'") if is_cave(z,i): fcave = Gcavelookup[i.strip("' []'")] # just sets it to the last one found. nasty. bug waiting to happen elif cleanid in Gcavelookup: fcave = Gcavelookup[cleanid] if str(fcave.slug()) == caveid: # print(f' - Found one ! {z.walletname=} {zcaveid=}') wallets.add(z) elif f"1623-{cleanid}" in Gcavelookup: # special hack for all the old wallets which are 1623 fcave = Gcavelookup[f"1623-{cleanid}"] if str(fcave.slug()) == caveid: # print(f' - Found one ! {z.walletname=} {zcaveid=}') wallets.add(z) elif cleanid in ['surface', 'unknown', '']: # message = f" ! In {z.walletname} cavewallets, ignoring '{cleanid}' as not a cave" # print(message) pass else: wurl = f"/walletedit/{z.walletname.replace('#',':')}" message = f" ! In {z.walletname} cavewallets, there is an unrecognised cave name '{cleanid}', adding to pending list." print(message) DataIssue.objects.update_or_create(parser="scans", message=message, url=wurl) add_cave_to_pending_list(cleanid, z, f"an unrecognised cave name in {z.walletname}") manywallets = list(set(wallets)) for w in manywallets: fillblankpeople(w) fillblankothers(w) w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc fixsurvextick(w, w.ticks) expeditions = Expedition.objects.all() print("--") return render( request, "cavewallets.html", {"manywallets": manywallets, "settings": settings, "cave": cave, "expeditions": expeditions}, ) def oldwallet(request, path): """Now called only for non-standard wallet structures for pre-2000 wallets""" # print([ s.walletname for s in Wallet.objects.all() ]) print(f"! - oldwallet path:{path}") try: wallet = Wallet.objects.get(walletname=urlunquote(path)) return render(request, "wallet_old.html", {"wallet": wallet, "settings": settings}) except: message = f"Scan folder error or not found '{path}' ." return render(request, "errors/generic.html", {"message": message}) def scansingle(request, path, file): """sends a single binary file to the user for display - browser decides how using mimetype This is very unsafe""" try: wallet = Wallet.objects.get(walletname=urlunquote(path)) singlescan = SingleScan.objects.get(wallet=wallet, name=file) imagefile = Path(singlescan.ffile, file) if imagefile.is_file(): message = f" - scansingle {imagefile} {path}:{file}:{getmimetype(file)}:" print(message) return HttpResponse(content=open(imagefile, "rb"), content_type=getmimetype(file)) # any type of image else: message = f"Scan folder file '{imagefile}' not found. {path=} {file=}" print(message) return render(request, "errors/generic.html", {"message": message}) except: message = f"Scan folder or scan item access error '{path}' and '{file}'." return render(request, "errors/generic.html", {"message": message}) def allscans(request): """Returns all the wallets in the system, we would like to use the Django queryset SQL optimisation https://docs.djangoproject.com/en/dev/ref/models/querysets/#prefetch-related to get the related singlescan and survexblock objects but that requires rewriting this to do the query on those, not on the wallets """ manywallets = Wallet.objects.all() # NB all of them # manywallets = Wallet.objects.all().prefetch_related('singlescan') fails as the link is defined on 'singlescan' not on 'wallet' expeditions = Expedition.objects.all() return render( request, "manywallets.html", {"manywallets": manywallets, "settings": settings, "expeditions": expeditions} )