diff --git a/core/models/wallets.py b/core/models/wallets.py index 0335524..2de8f9d 100644 --- a/core/models/wallets.py +++ b/core/models/wallets.py @@ -99,8 +99,7 @@ class Wallet(models.Model): walletdate = models.DateField(blank=True, null=True) walletyear = models.DateField(blank=True, null=True) caves = models.ManyToManyField("Cave", related_name="wallets") - # not yet - #persons = models.ManyToManyField("Person", related_name="wallets") + persons = models.ManyToManyField("Person", related_name="wallets") class Meta: ordering = ("walletname",) @@ -126,6 +125,8 @@ class Wallet(models.Model): return newstuff if stuff: return [stuff] # single object, not a string, but now in a list. + + return [] def get_absolute_url(self): @@ -220,25 +221,7 @@ class Wallet(models.Model): except: print(f"FAIL adding cave to wallet.caves '{i}'") pass - - # if type(cavelist) is list: - # for i in cavelist: - # if i != "": - # i = i.replace("/", "-") - # caveobject = get_cave_leniently(i) - # self.caves.add(caveobject) # new many-to-many field - # else: - # # either single cave or the square barckets have been removed and it s a singoe string - # ids = cavelist.split(",") - # for i in ids: - # j = i.replace("'","").replace("/", "-").strip('[] "') - # if i != "": - # try: - # caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data - # self.caves.add(caveobject) - # except: - # print(f"FAIL adding cave to wallet.caves '{j}'") - # pass + def year(self): """This gets the year syntactically without opening and reading the JSON""" if len(self.walletname) < 5: @@ -294,6 +277,15 @@ class Wallet(models.Model): return None return jsondata["name"] + + def survexfiles(self): + if not (jsondata := self.get_json()): # WALRUS + return None + + filelist = Wallet.input_to_list(jsondata["survex file"]) + #print(f"'{self} {jsondata['survex file']}' => {filelist}") + return filelist + def get_fnames(self): '''Filenames without the suffix, i.e. without the ".jpg"''' dirpath = Path(settings.SCANS_ROOT, self.fpath) # does nowt as fpath is a rooted path already diff --git a/core/views/scans.py b/core/views/scans.py index 0143bea..2494cd9 100644 --- a/core/views/scans.py +++ b/core/views/scans.py @@ -27,28 +27,26 @@ Note that caveifywallet() etc do NOT save the object to the db. They are ephemer manywallets dict. TODO -cave for a wallet - just gets the last one, randomly. SHould make this a list or many:many ideally - -- add the participants on an explicit wallet list to .slugpeople so that they get proper URL-linked on the per-person wallet report, and do the same thing for per-cave and per-year wallet reports add this file in to the todo list thinggy. """ - - + +def fix_manywallets(many): + for w in many: + fillblankpeople(w) + fillblankothers(w) + w.ticks = w.get_ticks() # the complaints in colour form + fixsurvextick(w, w.ticks) + def populatewallet(w): - """Copy survex data here just for display, not permanently - + """Need to tidy this up, now all the work is done at parse time not here Only gets data from the survex file when it was parsed on import, or edited (& thus parsed) online, so doesn't work if there was no *ref value """ - slugpeople = set() - blocks = SurvexBlock.objects.filter(scanswallet=w) - for b in blocks: - for personrole in b.survexpersonrole_set.all(): - slugpeople.add(personrole.person) # Person objects, not the names anymore - w.slugpeople = slugpeople + w.slugpeople = w.persons.all() def caveifywallet(w): @@ -104,10 +102,10 @@ def fillblankpeople(w): if nobody(wp): populatewallet(w) # sets w.slugpeople else: - w.persons = parse_name_list(w) + w.peeps = parse_name_list(w) populatewallet(w) # sets w.slugpeople if hasattr(w, "slugpeople"): - w.persons = w.persons.difference(w.slugpeople) + w.peeps = w.peeps.difference(w.slugpeople) return @@ -243,11 +241,7 @@ def walletslistperson(request, slug): print(message) DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl) - for w in manywallets: - fillblankpeople(w) - fillblankothers(w) - w.ticks = w.get_ticks() # the complaints in colour form - fixsurvextick(w, w.ticks) + fix_manywallets(manywallets) return manywallets # print("-walletslistperson") @@ -257,6 +251,7 @@ def walletslistperson(request, slug): expeditions = Expedition.objects.all() length_ug = 0.0 for w in manywallets: + print(w.persons) for sb in w.survexblock_set.all(): length_ug += sb.legslength return render( @@ -284,25 +279,18 @@ def walletslistyear(request, year): def ticksyearwallet(year): manywallets = [] - wallets = Wallet.objects.filter(walletyear__year=year) - for w in wallets: - manywallets.append(w) - fillblankpeople(w) - fillblankothers(w) - w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc - fixsurvextick(w, w.ticks) + manywallets = Wallet.objects.filter(walletyear__year=year) + fix_manywallets(manywallets) return manywallets # print("-walletslistyear") if year < 1976 or year > 2050: return render(request, "errors/generic.html", {"message": "Year out of range. Must be between 1976 and 2050"}) - # return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'}) - year = str(year) manywallets = ticksyearwallet(year) - expeditions = Expedition.objects.all() #bad Django style + expeditions = Expedition.objects.all() expedition = expeditions.filter(year=year) length_ug = 0.0 for w in manywallets: @@ -334,11 +322,8 @@ def cavewallets(request, caveid): wallets = cave.wallets.all() manywallets = list(set(wallets)) - for w in manywallets: - fillblankpeople(w) - fillblankothers(w) - w.ticks = w.get_ticks() # the complaints in colour form, from the json file on disc - fixsurvextick(w, w.ticks) + fix_manywallets(manywallets) + expeditions = Expedition.objects.all() length_ug = 0.0 for w in manywallets: diff --git a/parsers/imports.py b/parsers/imports.py index 112dd79..a469986 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -56,7 +56,7 @@ def import_survex(): troggle.parsers.survex.LoadSurvexBlocks() print(" - Survex entrances x/y/z Positions") with transaction.atomic(): - troggle.parsers.survex.caveifywallets() + troggle.parsers.survex.survexifywallets() with transaction.atomic(): troggle.parsers.locations.LoadPositions() diff --git a/parsers/scans.py b/parsers/scans.py index b9bae25..ad9c110 100644 --- a/parsers/scans.py +++ b/parsers/scans.py @@ -7,6 +7,9 @@ from troggle.core.models.troggle import DataIssue from troggle.core.models.wallets import Wallet """Searches through all the survey scans directories (wallets) in expofiles, looking for images to be referenced. Loads all the wallets . + +todo - update the survexblock scanswallet field to refer to the wallet, if it isn't already. +But we can't do that, we only have the survexfile, not the blcok. But we could set it for ALL the blocks? Hmm. """ contentsjson = "contents.json" @@ -25,7 +28,7 @@ def set_JSONwalletdate(wallet): def set_caves(wallet): _ = wallet.allcaves() # don't need return value. Just calling this saves it as w.caves This ONLY gets the list on the wallet JSON - + def load_all_scans(): """This iterates through the scans directories (either here or on the remote server) and builds up the models we can access later. @@ -173,6 +176,7 @@ def load_all_scans(): for wallet in Wallet.objects.all(): # this reads JSON wallet.check_survexlist() + diff --git a/parsers/survex.py b/parsers/survex.py index a86399f..e668853 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -138,7 +138,7 @@ def get_offending_filename(path): """ return "/survexfile/" + path + ".svx" -# THIS SHOULD NOT BE GLOBAL ! SHould be per instance of file loader.. + trip_people_cache = {} # indexed by survexblock, so never needs cleaning out def get_team_on_trip(survexblock): """Uses a cache to avoid a database query if it doesn't need to. @@ -146,7 +146,8 @@ def get_team_on_trip(survexblock): global trip_people_cache if survexblock in trip_people_cache: - return trip_people_cache[survexblock] + if len(trip_people_cache[survexblock]) > 0: + return trip_people_cache[survexblock] qpeople = SurvexPersonRole.objects.filter(survexblock=survexblock) # not very good Django style trip_people_cache[survexblock] = qpeople # this is a query list @@ -154,7 +155,8 @@ def get_team_on_trip(survexblock): def get_people_on_trip(survexblock): """Gets the displayable names of the people on a survexbock trip. - Only used for complete team.""" + Only used for complete team. + Seems to be only used for error messages.""" qpeople = get_team_on_trip(survexblock) # qpeople is a Query List people = [] @@ -699,7 +701,7 @@ class LoadingSurvex: else: message = f"! DATE Warning LONG DATE '{oline}' ({survexblock}) {survexblock.survexfile.path}" print(self.insp+message) - stash_data_issue(parser='svxdate', message=message, url=None, sb=(survexblock.survexfile.path)) + stash_data_issue(parser='xSvxDate', message=message, url=None, sb=(survexblock.survexfile.path)) if len(line) == 10: @@ -712,7 +714,7 @@ class LoadingSurvex: message = f"! DATE Warning only accurate to the month, setting to 1st '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}" print(self.insp + message) stash_data_issue( - parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) ) survexblock.date = datetime.strptime(line.replace(".", "-"), "%Y-%m") # sets to first of month elif len(line) == 4: @@ -720,7 +722,7 @@ class LoadingSurvex: message = f"! DATE WARNING only accurate to the YEAR, setting to 1st January '{oline}' ({survexblock}) {survexblock.survexfile.path} {perps}" print(self.insp + message) stash_data_issue( - parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) ) survexblock.date = datetime.strptime(line, "%Y") # sets to January 1st elif len(line) == 9 or len(line) == 8: @@ -728,7 +730,7 @@ class LoadingSurvex: message = f"! DATE format WARNING, single digit day or month number,'{oline}' [{line[-5]}][{line[-2]}] ({survexblock}) {survexblock.survexfile.path}" print(self.insp + message) stash_data_issue( - parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) ) if line[-2] == "-" or line[-2] == ".": line = line[:-1] + '0' + line[-1] @@ -745,7 +747,7 @@ class LoadingSurvex: ) print(self.insp + message) stash_data_issue( - parser="svxdate", message=message, url=None, sb=(survexblock.survexfile.path) + parser="xSvxDate", message=message, url=None, sb=(survexblock.survexfile.path) ) else: # these errors are reporting the wrong survexblock, which is actually a SurvexFile (!) @@ -985,7 +987,7 @@ class LoadingSurvex: perps = get_people_on_trip(survexblock) message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' malformed id '{args}' {perps}" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) return if not letterx: @@ -994,24 +996,24 @@ class LoadingSurvex: letterx = "X" message = f" ! Wallet *REF has LETTER in '{survexblock.survexfile.path}' malformed id '{args}' {perps}" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) if len(wallet) < 2: wallet = "0" + wallet if not (int(yr) > 1960 and int(yr) < 2050): message = " ! Wallet year out of bounds {yr} '{refscan}' {survexblock.survexfile.path}" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) refscan = f"{yr}#{letterx}{wallet}" try: if int(wallet) > 99: message = f" ! Wallet *REF {refscan} - very big (more than 99) so probably wrong in '{survexblock.survexfile.path}'" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) except: message = f" ! Wallet *REF {refscan} - not numeric in '{survexblock.survexfile.path}'" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) manywallets = Wallet.objects.filter( walletname=refscan @@ -1020,13 +1022,13 @@ class LoadingSurvex: if len(manywallets) > 1: message = f" ! Wallet *REF {refscan} - more than one found {len(manywallets)} wallets in db with same id {survexblock.survexfile.path}" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) if survexblock.scanswallet: if survexblock.scanswallet.walletname != refscan: message = f" ! Wallet *REF {refscan} in {survexblock.survexfile.path} - Already a DIFFERENT wallet is set for this block '{survexblock.scanswallet.walletname}'" print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) else: survexblock.scanswallet = manywallets[0] # this is a ForeignKey field survexblock.save() @@ -1037,7 +1039,7 @@ class LoadingSurvex: perps = get_people_on_trip(survexblock) message = f" ! Wallet *REF bad in '{survexblock.survexfile.path}' '{refscan}' NOT in database i.e. wallet does not exist {perps}." print(self.insp + message) - stash_data_issue(parser="survex", message=message, url=url) + stash_data_issue(parser="ref", message=message, url=url) def LoadSurvexDataNormal(self, survexblock, args): """Sets the order for data elements in this and following blocks, e.g. @@ -2481,17 +2483,82 @@ def MakeFileRoot(svxpath): return fileroot -def caveifywallets(): +def set_survexblocks(wallet): + if svxfiles := wallet.survexfiles(): # reads from JSON, should be cached + for svx in svxfiles: + # svx is a string, need to resolve to a survexfile object + #o = SurvexFile.objects.get(path=svx) + blocks = SurvexBlock.objects.filter(survexfile__path=svx) + for b in blocks: + if b.scanswallet == wallet: + pass + else: + b.scanswallet = wallet + b.save() + # print(f"setting {wallet} on {b.survexfile} : {b}") + +def survexifywallets(): """Gets the caves from the list of survexblocks + + We seem to hve a LOT of blocks with no atatched scnaswallet. Is this because we are + not inheriting *ref properly in the survexfile ? """ + print(f" - Update wallets with survex data") + + start = time.time() + # if there is a wallet for a block, add the people to the wallet + sprsall = SurvexPersonRole.objects.all().select_related("person").select_related("survexblock") + for spr in sprsall: + w = spr.survexblock.scanswallet + if w: + w.persons.add(spr.person) + + duration = time.time() - start + print(f" - TIME: add people to wallets {duration:7.2f} s", file=sys.stderr) + start = time.time() + wallets = Wallet.objects.all() for w in wallets: - blocks = SurvexBlock.objects.filter(scanswallet=w) + set_survexblocks(w) # reads JSON, sets survexblocks if survexfiles specified on wallet JSON + + duration = time.time() - start + print(f" - TIME: set survexblock:wallet using JSON survexfiles {duration:7.2f} s", file=sys.stderr) + start = time.time() + + for w in wallets: + blocks = SurvexBlock.objects.filter(scanswallet=w).select_related("survexfile") for b in blocks: - # NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath if b.survexfile.cave: w.caves.add(b.survexfile.cave) - + w.save() + + duration = time.time() - start + print(f" - TIME: add caves to wallets {duration:7.2f} s", file=sys.stderr) + start = time.time() + + + + cuccblocks = set() + for spr in SurvexPersonRole.objects.all(): + cuccblocks.add(spr.survexblock) + + sentinelbad = Wallet.objects.get(walletname="1983#00") + for b in cuccblocks: + if b.date > date(2001, 1, 1): # do we care about older ones? 1999 certainly has different wallet system + if not b.scanswallet: + if b.parent.scanswallet: + if b.parent.scanswallet != sentinelbad: + b.scanswallet = b.parent.scanswallet + continue + message = f" ! *REF missing {b.date} {b.survexfile}.svx : '{b}'" + # print(message, file=sys.stderr) + url = get_offending_filename(b.survexfile.path) + DataIssue.objects.update_or_create(parser="ref", message=message, url=url) + + duration = time.time() - start + print(f" - TIME: check missing *ref on survexblocks {duration:7.2f} s", file=sys.stderr) + start = time.time() + def LoadSurvexBlocks(): global dup_includes @@ -2516,10 +2583,11 @@ def LoadSurvexBlocks(): global dataissues dataissues = [] DataIssue.objects.filter(parser="survex").delete() - DataIssue.objects.filter(parser="svxdate").delete() + DataIssue.objects.filter(parser="xSvxDate").delete() DataIssue.objects.filter(parser="survexleg").delete() DataIssue.objects.filter(parser="survexunits").delete() DataIssue.objects.filter(parser="survex team").delete() + DataIssue.objects.filter(parser="ref").delete() # DataIssue.objects.filter(parser="xEntrances").delete() print(" - survex Data Issues flushed") mem1 = get_process_memory() diff --git a/templates/cavewallets.html b/templates/cavewallets.html index 315164a..2e67be7 100644 --- a/templates/cavewallets.html +++ b/templates/cavewallets.html @@ -33,13 +33,15 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c {% if wallet.walletdate %}{{wallet.walletdate}}{% else %} {% endif %} {% if wallet.name %}{{wallet.name|truncatechars:20}}{% else %}{% if wallet.displaynames %} {% for dn in wallet.displaynames %}{{dn}}{%if not forloop.last %}, {% endif %} {% endfor %}{% else %} {% endif %}{% endif %} + {% if wallet.slugpeople %} {%for p in wallet.slugpeople%}{{p.fullname}}{%if not forloop.last %}, {% endif %}{% endfor %} {% endif %} - {%for p in wallet.persons%} - {{p.fullname}}{%if not forloop.last %}, {% endif %} - {% endfor %} {{wallet.singlescan_set.all|length}} + {%for p in wallet.peeps%} + {{p.fullname}}{%if not forloop.last %}, {% endif %} + {% endfor %} + {% for survexblock in wallet.survexblock_set.all %} {{survexblock}} diff --git a/templates/personwallets.html b/templates/personwallets.html index 526e142..00534b2 100644 --- a/templates/personwallets.html +++ b/templates/personwallets.html @@ -13,6 +13,7 @@ traced to produce Tunnel or Therion drawings and eventually the final complete c

See also wallets