xle/troggle-unchained
2
0
Fork 0
forked from expo/troggle
Code Issues 4 Pull Requests Projects 1 Wiki Activity

Merge branch 'python3-new' of ssh://expo.survex.com/home/expo/troggle into python3-new

Browse Source
This commit is contained in:
Martin Green
2022-08-01 16:05:17 +02:00
parent f491264b9e eed35d01a8
commit c1aaf07885
25 changed files with 712 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
core/views/scans.py
View File

@@ -1,5 +1,6 @@
import os, stat
import re
import datetime
from pathlib import Path
from urllib.parse import urljoin, unquote as urlunquote
from urllib.request import urlopen
@@ -8,9 +9,12 @@ from django.conf import settings
from django.shortcuts import render
from django.http import HttpResponse
from troggle.core.models.survex import Wallet, SingleScan
from troggle.core.models.survex import Wallet, SingleScan, SurvexBlock
from troggle.core.models.troggle import Person
from troggle.core.models.caves import GetCaveLookup
from troggle.core.views.expo import getmimetype
#from troggle.parsers.people import GetPersonExpeditionNameLookup
#import parsers.surveys
'''one of these views serves files as binary blobs, and simply set the mime type based on the file extension,
@@ -19,8 +23,148 @@ by looking inside the file before being served.
need to check if inavlid query string is invalid, or produces multiple replies
and render a user-friendly error page.
Note that datewallet(), caveifywallet() etc do NOT save the object to the db. They are ephemeral, just for the page rendering of the
manywallets dict.
'''
def populatewallet(w):
'''Copy survex data here just for display, not permanently
'''
survexpeople = []
blocks = SurvexBlock.objects.filter(scanswallet = w)
for b in blocks:
for personrole in b.survexpersonrole_set.all():
survexpeople.append(personrole.personname)
w.persons = list(set(survexpeople))
def datewallet(w, earliest):
first = earliest
blocks = SurvexBlock.objects.filter(scanswallet = w)
for b in blocks:
if b.date:
if b.date < first:
first = b.date
if first == earliest:
# no date found
w.date = None
else:
w.date = first
def caveifywallet(w):
'''Gets the cave from the list of survex files,
only selects one of them though. Only used for display.
'''
blocks = SurvexBlock.objects.filter(scanswallet = w)
for b in blocks:
# NB b.cave is not populated by parser. Use b.survexfile.cave instead, or we could parse b.survexpath
if b.survexfile.cave:
w.cave = b.survexfile.cave # just gets the last one, randomly. SHould make this a list or many:many ideally
def fillblankpeople(w):
wp = w.people()
if not wp: # an -empty list
populatewallet(w)
else:
if len(wp) == 1:
nobody = wp[0].lower()
if nobody == 'unknown' or nobody == 'nobody' or nobody == ' ':
populatewallet(w)
def fillblankothers(w):
earliest = datetime.datetime.now().date()
if not w.date():
datewallet(w, earliest)
c = w.cave()
if not c:
caveifywallet(w)
def walletslistperson(request, first_name, last_name):
'''Page which displays a list of all the wallets for a specific person
HORRIBLE linear search through everything. Index and do SQL query properly
'''
# This is where we face having to re-do everything to do with names properly, rather than the horrible series of hacks over 20 years..
#GetPersonExpeditionNameLookup
def tickspersonwallet(p):
manywallets = []
wallets = Wallet.objects.all()
for w in wallets:
w.persons = w.people() # ephemeral attribute for web page
fillblankpeople(w)
if w.persons:
if p.fullname in w.persons:
manywallets.append(w)
fillblankothers(w)
w.ticks = w.get_ticks() # the complaints in colour form
return manywallets
try:
if last_name:
p = Person.objects.get(fullname= f'{first_name} {last_name}')
else:
# speciall Wookey-hack
p = Person.objects.get(first_name= f'{first_name}')
except:
#raise
return render(request, 'errors/generic.html', {'message': f'Unrecognised name of a expo person: "{first_name} {last_name}"'})
manywallets = tickspersonwallet(p)
return render(request, 'personwallets.html', { 'manywallets':manywallets, 'settings': settings, 'person': p})
def walletslistyear(request, year):
'''Page which displays a list of all the wallets in a specific year
'''
def ticksyearwallet(year):
manywallets = []
wallets = Wallet.objects.all()
for w in wallets:
if year == w.year():
manywallets.append(w)
fillblankpeople(w)
fillblankothers(w)
w.ticks = w.get_ticks() # the complaints in colour form
else:
continue
return manywallets
if year < 1976 or year > 2050:
return render(request, 'errors/generic.html', {'message': 'Year out of range. Must be between 1976 and 2050'})
else:
year = str(year)
#return render(request, 'errors/generic.html', {'message': 'This page logic not implemented yet'})
manywallets = ticksyearwallet(year)
return render(request, 'yearwallets.html', { 'manywallets':manywallets, 'settings': settings, 'year': year})
def cavewallets(request, caveid):
'''Returns all the wallets for just one cave
'''
Gcavelookup = GetCaveLookup()
if caveid in Gcavelookup:
cave = Gcavelookup[caveid]
else:
return render(request,'errors/badslug.html', {'badslug': caveid})
# remove duplication. SOrting is done in the template
wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave)) # NB a filtered set
manywallets = list(wallets)
for w in manywallets:
fillblankpeople(w)
fillblankothers(w)
w.ticks = w.get_ticks() # the complaints in colour form
return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave})
def oldwallet(request, path):
'''Now called only for non-standard wallet structures for pre-2000 wallets
'''
@@ -59,28 +203,13 @@ def scansingle(request, path, file):
return render(request, 'errors/generic.html', {'message': message})
def allwallets(request):
def allscans(request):
'''Returns all the wallets in the system, we would like to use
the Django queryset SQL optimisation https://docs.djangoproject.com/en/3.2/ref/models/querysets/#prefetch-related
to get the related singlescan and survexblock objects but that requires rewriting this to do the query on those, not on
the wallets
'''
manywallets = Wallet.objects.all()
manywallets = Wallet.objects.all() # NB all of them
# manywallets = Wallet.objects.all().prefetch_related('singlescan') fails as the link is defined on 'singlescan' not on 'wallet'
return render(request, 'manywallets.html', { 'manywallets':manywallets, 'settings': settings })
def cavewallets(request, cave_id):
'''Returns all the wallets for just one cave,
'''
Gcavelookup = GetCaveLookup()
if cave_id in Gcavelookup:
cave = Gcavelookup[cave_id]
else:
return render(request,'errors/badslug.html', {'badslug': cave_id})
# remove duplication. SOrting is done in the template
wallets = set(Wallet.objects.filter(survexblock__survexfile__cave=cave))
manywallets = list(wallets)
return render(request, 'cavewallets.html', { 'manywallets':manywallets, 'settings': settings, 'cave': cave})

4
core/views/statistics.py
View File

@@ -51,7 +51,7 @@ def pathsreport(request):
"SURVEX_DATA" : str( settings.SURVEX_DATA),
"SCANS_ROOT" : str( settings.SCANS_ROOT),
# "SURVEYS" : str( settings.SURVEYS),
"SCANS_URL" : str( settings.SCANS_URL),
# "SCANS_URL" : str( settings.SCANS_URL),
"SURVEXPORT" : str( settings.SURVEXPORT),
"DRAWINGS_DATA" : str( settings.DRAWINGS_DATA),
"URL_ROOT" : str( settings.URL_ROOT)
@@ -88,7 +88,7 @@ def pathsreport(request):
"SURVEX_DATA" : type(settings.SURVEX_DATA),
"SCANS_ROOT" : type(settings.SCANS_ROOT),
# "SURVEYS" : type(settings.SURVEYS),
"SCANS_URL" : type(settings.SCANS_URL),
# "SCANS_URL" : type(settings.SCANS_URL),
"SURVEXPORT" : type(settings.SURVEXPORT),
"DRAWINGS_DATA" : type(settings.DRAWINGS_DATA),
"URL_ROOT" : type(settings.URL_ROOT)

46
core/views/uploads.py
View File

@@ -23,12 +23,12 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
#from troggle import settings
from troggle.parsers.imports import import_caves, import_people, import_surveyscans
from troggle.parsers.imports import import_logbooks, import_QMs, import_drawingsfiles, import_survex
from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml
from troggle.parsers.scans import wallet_blank_json, wallet_blank_html, contentsjson, indexhtml, CopyWalletData
# from databaseReset import reinit_db # don't do this. databaseRest runs code *at import time*
from troggle.core.models.troggle import DataIssue
from troggle.core.models.troggle import Expedition, Person, PersonExpedition
from troggle.core.models.caves import LogbookEntry, QM, Cave, PersonTrip
from troggle.core.models.survex import DrawingFile
from troggle.core.models.survex import DrawingFile, Wallet
from troggle.core.views.scans import oldwallet, walletindex
from troggle.core.views.caves import getCave
@@ -93,13 +93,16 @@ xlate = {"url": "description url",
"electronic": "electronic survey",
"pland": "plan drawn",
"elevd": "elev drawn",
"psg": "name",
"psg": "name", # a name for this wallet
"survex": "survex file",
}
def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
'''Taken from old script wallets.py and edited to make more comprehensible
Loads the survex files names and processes all complaints
All needs to be restructred to use the get_ticks() function on the Wallets class in core/models/survex.py
which does the same thing
'''
# Date
if not waldata["date"]:
@@ -115,13 +118,14 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
if not type(waldata["survex file"])==list: # a string also is a sequence type, so do it this way
waldata["survex file"] = [waldata["survex file"]]
for svx in waldata["survex file"]:
svxfiles.append(svx)
if not (Path(settings.SURVEX_DATA) / svx).is_file():
file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo"
complaints.append(file_complaint)
message = f"! {file_complaint}"
print(message)
DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
if svx !="":
svxfiles.append(svx)
if not (Path(settings.SURVEX_DATA) / svx).is_file():
file_complaint = f"{wallet} Incorrect survex file name in wallet data: {svx} not found in LOSER repo"
complaints.append(file_complaint)
message = f"! {file_complaint}"
print(message)
DataIssue.objects.create(parser='scans', message=message, url=wurl) # set URL to this wallet folder
if waldata["survex not required"] and waldata["survex file"] != "":
survex_complaint = "Survex is stated as not required and yet there is a survex file!"
@@ -133,20 +137,21 @@ def get_complaints(complaints, waldata, svxfiles, files, wallet, wurl):
# Notes required
if not waldata["electronic survey"]:
notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False)
notes_scanned = reduce(operator.or_, [f.endswith("note") for f in files], notes_scanned)
notes_scanned = reduce(operator.or_, [Path(f).stem.endswith("notes") for f in files], notes_scanned)
if not notes_scanned:
complaints.append("The notes needs scanning (or renaming): no noteNN.jpg or XXnote.jpg file found; and this is not an electronic survey.")
# Plan drawing required
plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False)
plan_scanned = reduce(operator.or_, [f.endswith("plan") for f in files], plan_scanned)
plan_scanned = reduce(operator.or_, [Path(f).stem.endswith("plan") for f in files], plan_scanned)
plan_drawing_required = not (plan_scanned or waldata["plan drawn"] or waldata["plan not required"])
if plan_drawing_required:
complaints.append("The plan needs drawing (or renaming, or tick 'Plan drawn' checkbox or 'Plan not required' checkbox): no planNN.jpg or XXplan.jpg file found.")
# Elev drawing required
elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False)
elev_scanned = reduce(operator.or_, [f.endswith("elev") for f in files], elev_scanned)
elev_scanned = reduce(operator.or_, [Path(f).stem.endswith("elev") for f in files], elev_scanned)
elev_scanned = reduce(operator.or_, [Path(f).stem.endswith("elevation") for f in files], elev_scanned)
elev_drawing_required = not (elev_scanned or waldata["elev drawn"] or waldata["elev not required"])
if elev_drawing_required:
complaints.append("The elevation needs drawing (or renaming, or tick 'Elev drawn' checkbox or 'Elev not required' checkbox): no elevNN.jpg or XXelev.jpg file found.")
@@ -290,6 +295,21 @@ def scanupload(request, path=None):
with open(contents_path, "w") as jfile:
json.dump(wd, jfile, indent = 1)
# print(f'--- FINISHED saving to JSON\n')
# This copies the new data to the drawings repo and commit it
# needs the troggle object wallet, not a string
try:
w, created = Wallet.objects.get_or_create(walletname=wallet)
print(f'wallet string {wallet}, wallet object {w} created new?: {created}')
if created:
w.fpath = Path(settings.SCANS_ROOT, wallet[0:4], wallet)
w.save()
CopyWalletData(w)
except:
print(f'wallet string {wallet}, FAIL TO GET WALLET OBJECT, maybe we need to create it ?')
raise
else:
print(f'--- INVALID JSON Update form submitted')
print(formj.errors)