expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-21 23:01:52 +00:00
Code Issues 19 Projects 3 Activity

refactoring duplicate code

Browse Source
This commit is contained in:
Philip Sargent 2023-10-23 22:54:46 +03:00
parent b522899216
commit 33477f2b40
4 changed files with 65 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
core/models/wallets.py
View File

@ -105,6 +105,29 @@ class Wallet(models.Model):
class Meta:
ordering = ("walletname",)
@staticmethod
def input_to_list(stuff):
"""With wallets we often have to deal with either a list object (from valid JSON parsing)
or a string which may or may not also be a list, but munged by user eror on a form etc.
This function returns a list, either the JSON list, or a list with a single object in it,
or a list of strings. It silently absorbs empty strings an consumes odd quotes and square
brackets.
Always return a list, even if it is an empty list"""
if type(stuff) == list:
newstuff = []
for o in stuff:
if o: # not an empty string, None
newstuff.append(o)
return newstuff
if type(stuff) == str:
newstuff = stuff.split(",")
for s in newstuff:
s = s.strip('[] ').replace("'","").replace('"','').replace("/", "-").replace(" ", "_").strip('[] ')
return newstuff
if stuff:
return [stuff] # single object, not a string, but now in a list.
def get_absolute_url(self):
# we do not use URL_ROOT any more.
return reverse("singlewallet", kwargs={"path": re.sub("#", "%23", self.walletname)})
@ -114,6 +137,8 @@ class Wallet(models.Model):
Do it every time it is queried, to be sure the result is fresh.. well, no.
Do it every time we have a new python instance.
Reads JSON date and sets w.walletdate
import DataIssue locally to prevent import cycle problem"""
if hasattr(self, "JSONdata"):
@ -170,8 +195,7 @@ class Wallet(models.Model):
if not (waldata := self.get_json()): # WALRUS
return None
if waldata["survex file"]:
if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way
waldata["survex file"] = [waldata["survex file"]]
waldata["survex file"] = Wallet.input_to_list(waldata["survex file"])
for sx in waldata["survex file"]:
# this logic appears in several places, inc get_ticks(). and wallets_edit.py Refactor.
if sx != "":
@ -182,28 +206,39 @@ class Wallet(models.Model):
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
def allcaves(self):
"""Reads all the JSON data just to get the JSON date."""
"""Called when parsing importing all data. Called on all new wallets, but before
the survex files are parsed"""
if not (jsondata := self.get_json()): # WALRUS
return None
cavelist = jsondata["cave"]
if type(cavelist) is list:
for i in cavelist:
if i != "":
i = i.replace("/", "-")
caveobject = get_cave_leniently(i)
self.caves.add(caveobject) # new many-to-many field
else:
# either single cave or the square barckets have been removed and it s a singoe string
ids = cavelist.split(",")
for i in ids:
j = i.replace("'","").replace("/", "-").strip('[] "')
if i != "":
try:
caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data
self.caves.add(caveobject)
except:
print(f"FAIL adding cave to wallet.caves '{j}'")
pass
#cavelist = jsondata["cave"]
cavelist = Wallet.input_to_list(jsondata["cave"])
for i in cavelist:
try:
caveobject = get_cave_leniently(i)
if caveobject:
self.caves.add(caveobject)
except:
print(f"FAIL adding cave to wallet.caves '{i}'")
pass
# if type(cavelist) is list:
# for i in cavelist:
# if i != "":
# i = i.replace("/", "-")
# caveobject = get_cave_leniently(i)
# self.caves.add(caveobject) # new many-to-many field
# else:
# # either single cave or the square barckets have been removed and it s a singoe string
# ids = cavelist.split(",")
# for i in ids:
# j = i.replace("'","").replace("/", "-").strip('[] "')
# if i != "":
# try:
# caveobject = get_cave_leniently(j) # may fail if garbage value ,e.g. space, in wallet data
# self.caves.add(caveobject)
# except:
# print(f"FAIL adding cave to wallet.caves '{j}'")
# pass
def year(self):
"""This gets the year syntactically without opening and reading the JSON"""
if len(self.walletname) < 5:

10
core/views/scans.py
View File

@ -174,23 +174,17 @@ def parse_name_list(w):
crew = GetPersonExpeditionNameLookup(expo)
for n in namelist:
# if n.lower().startswith("lydia"):
# print(f"{w} {n=} ")
# for x in crew:
# if x.lower()==n.lower():
# print(f"{w} {n=} {x=}")
if n.lower() in crew:
peeps.add(crew[n.lower()].person)
else:
if n.startswith("*"): #ignore people flagged as guests or not-expo anyway, such as ARGE
if check := n.startswith("*"): #ignore people flagged as guests or not-expo anyway, such as ARGE
continue
nobod = n.lower()
if nobod == "unknown" or nobod == "nobody" or nobod == " " or nobod == "":
continue
else:
wurl = f"/walletedit/{w.walletname.replace('#',':')}"
message = f"{w} name '{n.lower()}' NOT found in GetPersonExpeditionNameLookup({w.year()}) ?!"
message = f"{w} name '{n.lower()}' NOT found in GetPersonExpeditionNameLookup({w.year()}) n.startswith* = {check} ?!"
print(message)
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
return peeps

13
parsers/scans.py
View File

@ -13,15 +13,14 @@ contentsjson = "contents.json"
git = settings.GIT
# to do: Actually read all the JSON files and set the survex file field appropriately!
def set_walletyear(wallet):
_ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear Syntactic.
def set_JSONwalletdate(wallet):
"""At this point in the import process, the survex files have not been imported so
we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here."""
we cannot get dates from them. There are about 40 JSON files (in 2022) which we read here.
Actually, doing anything that reads the JSON sets .walletdate"""
_ = wallet.date() # don't need return value. Sets .walletdate as side effect
def set_caves(wallet):
@ -111,11 +110,11 @@ def load_all_scans():
else:
print("", flush=True, end="")
# Create the wallet object. But we don't have a date for it yet.
wallet = Wallet(fpath=fpath, walletname=walletname)
wallet = Wallet.objects.create(walletname=walletname, fpath=fpath)
wallets[walletname] = wallet
set_walletyear(wallet)
wallet.save()
set_caves(wallet)
wallet.save()
singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet)
singlescan.save()
@ -156,15 +155,13 @@ def load_all_scans():
# The wallets found from JSON should all have dates already
wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
wallets[walletname] = wallet
# Now also load the json
set_JSONwalletdate(wallet)
set_walletyear(wallet)
set_caves(wallet)
wallet.save()
if not created:
print(
f"\n - {walletname} was not created, but was not in directory walk of /surveyscans/. Who created it?"
)
wallet.save()
print(f"\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets")
# Only the 1999 wallets have filenames which mean that the walletyear will be unset:

2
templates/base.html
View File

@ -47,7 +47,7 @@
<a href="{% url "dataissues" %}">Data Issues</a> |
<a href="/handbook/computing/todo-data.html">tasks to do </a> |
<a id="entsLink" href="{% url "eastings" %}">ents</a> |
<a id="entsLink" href="{% url "entranceindex" %}">ents</a> |
<a id="folklink" href="/folk">expoers</a> |
<a id="caversLink" href="{% url "notablepersons" %}">survey lengths</a> |
<a href="{% url "stats" %}">statistics</a> |