expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-25 08:41:51 +00:00
Code Issues 19 Projects 3 Activity

fix bugs made visible by py 3.11

Browse Source
This commit is contained in:
Philip Sargent 2022-11-23 21:59:42 +00:00
parent b06d1dae42
commit 1eab261b30
8 changed files with 84 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.txt
View File

@ -50,7 +50,7 @@ Follow the instructions contained in the file to fill out your settings.
Python3, Django, and Database setup Python3, Django, and Database setup
----------------------------------- -----------------------------------
We are now using Django 3.2 and will move to 4.2 in 2024 We are now using Django 3.2 and will move to 4.2 in 2024
We are installing with python3.9 We are installing with python3.10 or 3.11 (the server is running 3.9)
Install Django using pip, not with apt, on your test system in a venv. Install Django using pip, not with apt, on your test system in a venv.
Conventionally on our main master expo server we install everything that we can as debian packages, not using pip. Conventionally on our main master expo server we install everything that we can as debian packages, not using pip.

14
core/TESTS/test_imports.py
View File

@ -152,9 +152,12 @@ class SubprocessTest(TestCase):
TROGGLE_PATH = Path(settings.REPOS_ROOT_PATH) / "troggle" TROGGLE_PATH = Path(settings.REPOS_ROOT_PATH) / "troggle"
for cwd in [settings.SURVEX_DATA, settings.EXPOWEB, settings.DRAWINGS_DATA, TROGGLE_PATH]: for cwd in [settings.SURVEX_DATA, settings.EXPOWEB, settings.DRAWINGS_DATA, TROGGLE_PATH]:
sp = subprocess.run([settings.GIT, "status"], cwd=cwd, capture_output=True, text=True) sp = subprocess.run([settings.GIT, "status"], cwd=cwd, capture_output=True, text=True)
print(f'git output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') out = str(sp.stdout)
if len(out) > 160:
out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
print(f'git output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}')
if sp.returncode != 0: if sp.returncode != 0:
print(f'git output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') print(f'git output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}')
self.assertTrue( sp.returncode == 0, f'{cwd} - git is unhappy') self.assertTrue( sp.returncode == 0, f'{cwd} - git is unhappy')
@ -180,9 +183,12 @@ class SubprocessTest(TestCase):
cwd = settings.SURVEX_DATA cwd = settings.SURVEX_DATA
for survey in ["1623.svx", "1626.svx"]: for survey in ["1623.svx", "1626.svx"]:
sp = subprocess.run([settings.CAVERN, survey], cwd=cwd, capture_output=True, text=True) sp = subprocess.run([settings.CAVERN, survey], cwd=cwd, capture_output=True, text=True)
print(f'survex output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') out = str(sp.stdout)
if len(out) > 160:
out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
# print(f'survex output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}')
if sp.returncode != 0: if sp.returncode != 0:
print(f'survex output: {cwd}:\n # {sp.stderr=}\n # {sp.stdout=} \n # return code: {str(sp.returncode)}') print(f'survex output: {cwd}:\n # {sp.stderr=}\n # sp.stdout={out} \n # return code: {str(sp.returncode)}')
self.assertTrue( sp.returncode == 0, f'{cwd} - survex is unhappy') self.assertTrue( sp.returncode == 0, f'{cwd} - survex is unhappy')

10
core/utils.py
View File

@ -180,11 +180,17 @@ def writetrogglefile(filepath, filecontent):
#os.chmod(filepath, 0o664) # set file permissions to rw-rw-r-- #os.chmod(filepath, 0o664) # set file permissions to rw-rw-r--
sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True) sp = subprocess.run([git, "add", filename], cwd=cwd, capture_output=True, check=True, text=True)
if sp.returncode != 0: if sp.returncode != 0:
print(f'git ADD {cwd}:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode)) out = sp.stdout
if len(out) > 160:
out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
print(f'git ADD {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out + '\n\nreturn code: ' + str(sp.returncode))
sp = subprocess.run([git, "commit", "-m", f'Troggle online: cave or entrance edit -{filename}'], cwd=cwd, capture_output=True, check=True, text=True) sp = subprocess.run([git, "commit", "-m", f'Troggle online: cave or entrance edit -{filename}'], cwd=cwd, capture_output=True, check=True, text=True)
if sp.returncode != 0: if sp.returncode != 0:
print(f'git COMMIT {cwd}:\n\n' + str(sp.stderr) + '\n\n' + str(sp.stdout) + '\n\nreturn code: ' + str(sp.returncode)) out = sp.stdout
if len(out) > 160:
out = out[:75] + "\n <Long output curtailed>\n" + out[-75:]
print(f'git COMMIT {cwd}:\n\n' + str(sp.stderr) + '\n\n' + out + '\n\nreturn code: ' + str(sp.returncode))
# not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that. # not catching and re-raising any exceptions yet, inc. the stderr etc.,. We should do that.

2
core/views/survex.py
View File

@ -273,7 +273,7 @@ def svx(request, survex_file):
difflist.insert(0, message) difflist.insert(0, message)
#print [ form.data['code'] ] #print [ form.data['code'] ]
svxincludes = re.findall(r'\*include\s+(\S+)(?i)', form.data['code'] or "") svxincludes = re.findall(r'(?i)\*include\s+(\S+)', form.data['code'] or "")
vmap = {'settings': settings, vmap = {'settings': settings,
'warning': warning, 'warning': warning,

23
core/views/uploads.py
View File

@ -471,7 +471,7 @@ def scanupload(request, path=None):
try: # crashes in Django os.chmod call if on WSL, but does save file! try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f) saved_filename = fs.save(f.name, content=f)
except: except:
print(f'\n !! Permissions failure ?! on attempting to save file {f.name}') print(f'\n !! Permissions failure ?! on attempting to save scanfile {f.name}')
if 'saved_filename' in locals(): if 'saved_filename' in locals():
if saved_filename.is_file(): if saved_filename.is_file():
actual_saved.append(saved_filename) actual_saved.append(saved_filename)
@ -757,7 +757,13 @@ def photoupload(request, folder=None):
formd = TextForm(request.POST) formd = TextForm(request.POST)
if formd.is_valid(): if formd.is_valid():
newphotographer = request.POST["photographer"] newphotographer = request.POST["photographer"]
try:
(yearpath / newphotographer).mkdir(exist_ok=True) (yearpath / newphotographer).mkdir(exist_ok=True)
except:
message =f'\n !! Permissions failure ?! 0 attempting to mkdir "{(yearpath / newphotographer)}"'
print(message)
return render(request,'errors/generic.html', {'message': message})
else: else:
form = FilesRenameForm(request.POST,request.FILES) form = FilesRenameForm(request.POST,request.FILES)
if form.is_valid(): if form.is_valid():
@ -775,7 +781,7 @@ def photoupload(request, folder=None):
try: # crashes in Django os.chmod call if on WSL, but does save file! try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(renameto, content=f) saved_filename = fs.save(renameto, content=f)
except: except:
print(f'\n !! Permissions failure ?! on attempting to save file {f.name}') print(f'\n !! Permissions failure ?! 1 attempting to save "{f.name}" in "{dirpath}" {renameto=}')
if 'saved_filename' in locals(): if 'saved_filename' in locals():
if saved_filename.is_file(): if saved_filename.is_file():
actual_saved.append(saved_filename) actual_saved.append(saved_filename)
@ -784,17 +790,17 @@ def photoupload(request, folder=None):
try: # crashes in Django os.chmod call if on WSL, but does save file! try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f) saved_filename = fs.save(f.name, content=f)
except: except:
print(f'\n !! Permissions failure ?! on attempting to save file {f.name}') print(f'\n !! Permissions failure ?! 2 attempting to save "{f.name}" in "{dirpath}" {renameto=}')
if 'saved_filename' in locals(): if 'saved_filename' in locals():
if saved_filename.is_file(): if saved_filename.is_file():
actual_saved.append(saved_filename) actual_saved.append(saved_filename)
filesaved = True filesaved = True
else: # multiole is a list of content else: # multiple is a list of content
for f in multiple: for f in multiple:
try: # crashes in Django os.chmod call if on WSL, but does save file! try: # crashes in Django os.chmod call if on WSL, but does save file!
saved_filename = fs.save(f.name, content=f) saved_filename = fs.save(f.name, content=f)
except: except:
print(f'\n !! Permissions failure ?! on attempting to save file {f.name}') print(f'\n !! Permissions failure ?! 3 attempting to save "{f.name}" in "{dirpath}" {renameto=}')
if 'saved_filename' in locals(): if 'saved_filename' in locals():
if saved_filename.is_file(): if saved_filename.is_file():
actual_saved.append(saved_filename) actual_saved.append(saved_filename)
@ -872,7 +878,8 @@ def dwgupload(request, folder=None, gitdisable='no'):
# print(f'! - FORM dwgupload - POST valid: "{request.FILES["uploadfiles"]}" ') # print(f'! - FORM dwgupload - POST valid: "{request.FILES["uploadfiles"]}" ')
f = request.FILES["uploadfiles"] f = request.FILES["uploadfiles"]
multiple = request.FILES.getlist('uploadfiles') multiple = request.FILES.getlist('uploadfiles')
fs = FileSystemStorage(os.path.join(settings.DRAWINGS_DATA, folder)) savepath = Path(settings.DRAWINGS_DATA, folder)
fs = FileSystemStorage(savepath)
actual_saved = [] actual_saved = []
refused = [] refused = []
@ -892,7 +899,7 @@ def dwgupload(request, folder=None, gitdisable='no'):
try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file! try: # crashes in Django os.chmod call if on WSL without metadata drvfs, but does save file!
saved_filename = fs.save(f.name, content=f) saved_filename = fs.save(f.name, content=f)
except: except:
print(f'! - FORM dwgupload - \n!! Permissions failure ?! on attempting to save file {f.name}. Attempting to continue..') print(f'! - FORM dwgupload - \n!! Permissions failure ?! on attempting to save file "{f.name}" in "{savepath}". Attempting to continue..')
if 'saved_filename' in locals(): if 'saved_filename' in locals():
if Path(dirpath, saved_filename).is_file(): if Path(dirpath, saved_filename).is_file():
actual_saved.append(saved_filename) actual_saved.append(saved_filename)
@ -912,7 +919,7 @@ def dwgupload(request, folder=None, gitdisable='no'):
message = f'! - FORM dwgupload - NOT A FILE {Path(dirpath, saved_filename)=}. ' message = f'! - FORM dwgupload - NOT A FILE {Path(dirpath, saved_filename)=}. '
print(message) print(message)
else: else:
message = f'! - FORM dwgupload - Save failure for {saved_filename}. Changes NOT saved.' message = f'! - FORM dwgupload - Save failure for {f.name}. Changes NOT saved.'
print(message) print(message)
return render(request,'errors/generic.html', {'message': message}) return render(request,'errors/generic.html', {'message': message})

23
os-trog.sh
View File

@ -15,7 +15,7 @@ sudo apt install sqlite3 -y
sudo apt install python3-pip -y sudo apt install python3-pip -y
# this installs a shed-load of other stuff: binutils etc.sudo apt install survex-aven # this installs a shed-load of other stuff: binutils etc.sudo apt install survex-aven
sudo apt install git openssh-client tunnelx therion -y sudo apt install git openssh-client -y
# On a clean debian 11 (bullseye) installation with Xfce & ssh, # On a clean debian 11 (bullseye) installation with Xfce & ssh,
#on ubuntu 20.04: #on ubuntu 20.04:
@ -26,7 +26,7 @@ sudo apt install git openssh-client tunnelx therion -y
# On Ubuntu 20.04, with python10, the pip install fails. # On Ubuntu 20.04, with python10, the pip install fails.
# So you ned to get the pip from source # So you need to get the pip from source
# sudo curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 # sudo curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
# but really you should be using 22.04 # but really you should be using 22.04
# and also, isf using debian, # and also, isf using debian,
@ -36,10 +36,10 @@ sudo apt install git openssh-client tunnelx therion -y
sudo useradd expo sudo useradd expo
sudo usermod -a -G sudo expo # to put expo in sudoers group, re-login required sudo usermod -a -G sudo expo # to put expo in sudoers group, re-login required
# default since 22.04 # default since 22.04
# sudo apt install python3.11 # sudo apt install python3.10
sudo apt install python3.11-venv -y sudo apt install python3.10-venv -y
sudo apt install python3.11-dev -y sudo apt install python3.10-dev -y
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1
sudo apt install mariadb-server -y sudo apt install mariadb-server -y
sudo apt install libmariadb-dev -y sudo apt install libmariadb-dev -y
@ -54,3 +54,14 @@ sudo apt install tunnelx therion -y
# Go to https://expo.survex.com/handbook/troggle/troglaptop.html#dbtools # Go to https://expo.survex.com/handbook/troggle/troglaptop.html#dbtools
# sudo service mysql start # sudo service mysql start
git config --global user.email "you@example.com"
git config --global user.name "Your Name"
echo '###'
echo '### Currently set version of python'
python --version
echo '###'
echo '### Now YOU have to configure the git settings for YOURSELF (not "expo")'

3
parsers/logbooks.py
View File

@ -362,7 +362,7 @@ def parser_html_01(year, expedition, txt):
# print(f" #0 - tid: {tid}") # print(f" #0 - tid: {tid}")
try: try:
#print(f" #1 - tid: {tid}") #print(f" #1 - tid: {tid}")
s = re.match(r"(?s)\s*(?:<p>)?(.*?)</?p>(.*)$(?i)", trippara) s = re.match(r"(?i)(?s)\s*(?:<p>)?(.*?)</?p>(.*)$", trippara)
if not s: if not s:
message = " ! - Skipping logentry {year} failure to parse header: " + tid + trippara[:300] + "..." message = " ! - Skipping logentry {year} failure to parse header: " + tid + trippara[:300] + "..."
DataIssue.objects.create(parser='logbooks', message=message) DataIssue.objects.create(parser='logbooks', message=message)
@ -449,6 +449,7 @@ def parser_html_01(year, expedition, txt):
logdataissues[tid]=message logdataissues[tid]=message
print(message) print(message)
errorcount += 1 errorcount += 1
raise
if errorcount >5 : if errorcount >5 :
message = f" !!- TOO MANY ERRORS - aborting at '{tid}' logbook: {year}" message = f" !!- TOO MANY ERRORS - aborting at '{tid}' logbook: {year}"
DataIssue.objects.create(parser='logbooks', message=message) DataIssue.objects.create(parser='logbooks', message=message)

31
venv-trog.sh
View File

@ -8,12 +8,12 @@ echo '-- Run this in a terminal in the real troggle directory: "bash venv-trog.s
# use the script os-trog.sh # use the script os-trog.sh
# If you are using Debian, then stick with the default version of python # If you are using Debian, then stick with the default version of python
# If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.10 or 3.11 # If you are using Ubuntu, then it is easy to use a later version of python, e.g. 3.11
# NOW we set up troggle # NOW we set up troggle
PYTHON=python3.10 PYTHON=python3.10
VENAME=p10d5 # python3.x and django 3.2 VENAME=p10d3 # python3.x and django 3.2
echo "** You are logged in as `id -u -n`" echo "** You are logged in as `id -u -n`"
echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/" echo "The 50MB pip cache will be in /home/`id -u -n`/.cache/"
echo "The 150MB venv will created in /home/`id -u -n`/$VENAME/" echo "The 150MB venv will created in /home/`id -u -n`/$VENAME/"
@ -79,9 +79,17 @@ ln -s ${TROGDIR}/../drawings drawings
if [ -d ${TROGDIR}/../expofiles ]; then if [ -d ${TROGDIR}/../expofiles ]; then
ln -s ${TROGDIR}/../expofiles expofiles ln -s ${TROGDIR}/../expofiles expofiles
else else
if [ ! -d /mnt/f/expofiles ]; then
sudo mkdir /mnt/f
sudo mount -t drvfs F: /mnt/f
else
ln -s /mnt/f/expofiles expofiles ln -s /mnt/f/expofiles expofiles
fi
fi fi
echo "### Setting file permissions.. may take a while.."
git config --global --add safe.directory '*'
sudo chmod -r 0777 *
echo "### links to expoweb, troggle etc. complete:" echo "### links to expoweb, troggle etc. complete:"
ls -tla ls -tla
@ -134,8 +142,25 @@ echo "### Now do
'cd troggle' 'cd troggle'
'django-admin' 'django-admin'
'python manage.py check' 'python manage.py check'
## this tests if you have set up ssh correcting. Refer to documentation https://expo.survex.com/handbook/computing/keyexchange.html
## you need to follow the Linux instructions.
'ssh expo@expo.survex.com'
## the next tests will fail unless ~/expofiles is set correctly to a folder on your machine
## the tests may ALSO fail because of ssh and permissions errors
# Ran 85 tests in 83.492s
# FAILED (failures=5)
## So you will need to run
$sudo chown -Rhv philip:philip ~/$VENAME (if your username is philip)
# and then REBOOT (or at least, exit WSL and terminate and restart WSL)
# because this chmod only takes effect then.
'python manage.py test -v 2' 'python manage.py test -v 2'
'./pre-run.sh' './pre-run.sh' (runs the tests again)
'python databaseReset.py reset $VENAME' 'python databaseReset.py reset $VENAME'
'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)' 'python manage.py runserver 0.0.0.0:8000 (and allow access when the firewall window pops up)'
" "
if [ ! -d /mnt/f/expofiles ]; then
echo '### No valid expofiles directory. Fix this before any tests will work.
fi