expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-21 23:01:52 +00:00
Code Issues 19 Projects 3 Activity

checking wallets earlier int he process

Browse Source
This commit is contained in:
Philip Sargent 2023-10-23 02:32:44 +03:00
parent 8f87e4f77a
commit 54ffab3e93
5 changed files with 63 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
core/models/wallets.py
View File

@ -10,6 +10,7 @@ from django.conf import settings
from django.db import models
from django.urls import reverse
from troggle.core.models.troggle import DataIssue
from troggle.core.views.caves import get_cave_leniently
# from troggle.core.models.survex import SurvexBlock
@ -143,8 +144,26 @@ class Wallet(models.Model):
message = f"! {str(self.walletname)} Date format not ISO {waldata['date']}. Failed to load from {jsonfile} JSON file"
from troggle.core.models.troggle import DataIssue
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
return waldata
def check_survexlist(self):
wurl = f"/walletedit/{self.walletname}".replace('#', ':')
if not (waldata := self.get_json()): # WALRUS
return None
if waldata["survex file"]:
if not type(waldata["survex file"]) == list: # a string also is a sequence type, so do it this way
waldata["survex file"] = [waldata["survex file"]]
for sx in waldata["survex file"]:
# this logic appears in several places, inc get_ticks(). and wallets_edit.py Refactor.
if sx != "":
if Path(sx).suffix.lower() != ".svx":
sx = sx + ".svx"
if not (Path(settings.SURVEX_DATA) / sx).is_file():
message=f"{self} Survex file {sx} was not found in LOSER repo"
DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
def allcaves(self):
"""Reads all the JSON data just to get the JSON date."""
if not (jsondata := self.get_json()): # WALRUS
@ -209,23 +228,19 @@ class Wallet(models.Model):
self.save()
return self.walletdate
# for gods sake redo this, it parse JSON twice every time..
def people(self):
if not self.get_json():
if not (jsondata := self.get_json()): # WALRUS
return None
jsondata = self.get_json()
return jsondata["people"]
def cave(self):
if not self.get_json():
if not (jsondata := self.get_json()): # WALRUS
return None
jsondata = self.get_json()
return jsondata["cave"]
def name(self):
if not self.get_json():
if not (jsondata := self.get_json()): # WALRUS
return None
jsondata = self.get_json()
return jsondata["name"]
def get_fnames(self):

1
core/views/caves.py
View File

@ -405,6 +405,7 @@ def edit_cave(request, path="", slug=None):
if form.is_valid(): # and ceFormSet.is_valid():
# print(f'! POST is valid. {cave}')
cave = form.save(commit=False)
print(cave)
if not cave.filename:
cave.filename = cave.areacode + "-" + cave.number() + ".html"
if not cave.url:

8
core/views/wallets_edit.py
View File

@ -156,7 +156,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
message = f"! {file_complaint}"
print(message)
DataIssue.objects.update_or_create(
parser="scans", message=message, url=wurl
parser="wallets", message=message, url=wurl
) # set URL to this wallet folder
else:
try:
@ -165,6 +165,10 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
except MultipleObjectsReturned:
# can happen if connecting a wallet to a survex file.. i think..
QSsvxfiles = SurvexFile.objects.filter(path=sxpath)
message = f"! {wallet} Urk, multiple SurvexFile objects {sxpath}"
DataIssue.objects.update_or_create(
parser="wallets", message=message, url=wurl
) # set URL to this wallet folder
for s in QSsvxfiles:
print(s.path, s.cave, s.primary)
# QSsvxfiles[0] # dont' know how this happened, fix later..
@ -176,7 +180,7 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
message = f"! {file_complaint}"
print(message)
DataIssue.objects.update_or_create(
parser="scans", message=message, url=wurl
parser="wallets", message=message, url=wurl
) # set URL to this wallet folder
if waldata["survex not required"] and waldata["survex file"] != [""]:

27
parsers/scans.py
View File

@ -17,7 +17,7 @@ git = settings.GIT
def set_walletyear(wallet):
_ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear
_ = wallet.year() # don't need return value. Just calling this saves it as w.walletyear Syntactic.
def set_JSONwalletdate(wallet):
"""At this point in the import process, the survex files have not been imported so
@ -30,12 +30,9 @@ def set_caves(wallet):
def load_all_scans():
"""This iterates through the scans directories (either here or on the remote server)
and builds up the models we can access later.
It does NOT read or validate anything in the JSON data attached to each wallet. Those checks
are done at runtime, when a wallet is accessed, not at import time.
Loads people as a simple string of fullnames. We should replace this with a list of Person slugs.
Loads people as a simple string of fullnames. We should replace this with a list of Person slugs,
and change the wallet editor to save People as slugs.
"""
print(" - Loading Survey Scans")
@ -115,11 +112,11 @@ def load_all_scans():
print("", flush=True, end="")
# Create the wallet object. But we don't have a date for it yet.
wallet = Wallet(fpath=fpath, walletname=walletname)
wallets[walletname] = wallet
set_walletyear(wallet)
wallet.save()
set_caves(wallet)
wallets[walletname] = wallet
singlescan = SingleScan(ffile=fpath, name=p.name, wallet=wallet)
singlescan.save()
@ -160,7 +157,6 @@ def load_all_scans():
wallet, created = Wallet.objects.update_or_create(walletname=walletname, fpath=fpath)
wallets[walletname] = wallet
# Now also load the json
# BUT can't check linked survex blocks as they haven't been imported yet
set_JSONwalletdate(wallet)
set_walletyear(wallet)
set_caves(wallet)
@ -170,8 +166,17 @@ def load_all_scans():
)
wallet.save()
print(f"\n - found another {wjson:,} JSON files, making a total of {len(wallets):,} wallets")
# Only the 1999 wallets have filenames which mean that the walletyear will be unset:
wallets = Wallet.objects.filter(walletyear=None)
for w in wallets:
w.walletyear = datetime.date(1999, 1, 1)
w.walletyear = datetime.date(1999, 1, 1)
# Although the survex files haven't been processed yet, we can at least check if the wallets refer to a real file or not
for wallet in Wallet.objects.all():
# this reads JSON
wallet.check_survexlist()

25
templates/manywallets.html
View File

@ -27,28 +27,37 @@ otherwise they come from *ref statements in survex files as of the most recent d
<!-- This should all be restructured to use .prefetch_related() and .select_related()
see https://docs.djangoproject.com/en/dev/ref/models/querysets/#prefetch-related
-->
<table width=95%>
<tr><th>Scans folder</th><th>Files</th><th>Survex blocks</th><th>Cave</th></tr>
{% for scanswallet in manywallets %}
<tr><th>Scans folder</th><th>Files</th><th>Survex blocks</th><th>Cave (new)</th><th>Cave (old)</th></tr>
{% for wallet in manywallets %}
<tr>
<td style="padding:2px"><a href="{{scanswallet.get_absolute_url}}">{{scanswallet.walletname}}</a></td>
<td align="right" style="padding:2px">{{scanswallet.singlescan_set.all|length}}</td>
<td style="padding:2px"><a href="{{wallet.get_absolute_url}}">{{wallet.walletname}}</a></td>
<td align="right" style="padding:2px">{{wallet.singlescan_set.all|length}}</td>
<td style="padding:2px">
{% for survexblock in scanswallet.survexblock_set.all %}
{% for survexblock in wallet.survexblock_set.all %}
<a href="{% url "svx" survexblock.survexfile.path %}">{{survexblock}}</a>
{% endfor %}
</td>
<td style="padding:2px; font-family: monospace; font-size: 90%;">
{% for c in wallet.caves.all %}
<a href="/cave/scans/{{c.slug}}">{{c}}</a>
{% endfor %}
</td>
<td style="padding:2px">
{% for survexblock in scanswallet.survexblock_set.all %}
{% for survexblock in wallet.survexblock_set.all %}
{% ifchanged survexblock.survexfile.cave %}
<a href="/cave/scans/{{survexblock.survexfile.cave.slug}}">{{survexblock.survexfile.cave.slug}}</a>
{% endifchanged %}
{% empty %}
{% if scanswallet.cave %}
<em><a href="/cave/scans/{{scanswallet.cave}}">{{scanswallet.cave}}</a></em>
{% if wallet.cave %}
<em><a href="/cave/scans/{{wallet.cave}}">{{wallet.cave}}</a></em>
{% endif %}
{% endfor %}
</td>
</tr>
{% endfor %}
</table>