diff --git a/databaseReset.py b/databaseReset.py index 55d1182..01d000a 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -448,10 +448,6 @@ if __name__ == "__main__": jq.enq("survex", import_loadpos) elif "drawings" in sys.argv: jq.enq("drawings", import_drawingsfiles) - elif "dumplogbooks" in sys.argv: # untested in 2020 - dumplogbooks() - # elif "writecaves" in sys.argv: # untested in 2020 - will overwrite input files!! - # writeCaves() elif "profile" in sys.argv: if runlabel == "del": jq.loadprofiles() diff --git a/parsers/QMs.py b/parsers/QMs.py index 79f9f0f..da3d6c7 100644 --- a/parsers/QMs.py +++ b/parsers/QMs.py @@ -1,11 +1,10 @@ import csv import os import re -from datetime import date from django.conf import settings -from troggle.core.models.caves import QM, Cave, LogbookEntry +from troggle.core.models.caves import QM, Cave from troggle.core.models.troggle import DataIssue from troggle.core.utils import save_carefully @@ -79,7 +78,7 @@ def parseCaveQMs(cave, inputFile, ticked=False): try: n += 1 year = int(line[0][1:5]) - logslug = f"PH_{int(year)}_{int(n):02d}" + f"PH_{int(year)}_{int(n):02d}" QMnum = re.match(r".*?-\d*?-X?(?P\d*)", line[0]).group("numb") newQM = QM() # newQM.found_by=placeholder @@ -108,7 +107,7 @@ def parseCaveQMs(cave, inputFile, ticked=False): number=QMnum, found_by__date__year=year ) # if we don't have this one in the DB, save it if ( - preexistingQM.new_since_parsing == False + preexistingQM.new_since_parsing is False ): # if the pre-existing QM has not been modified, overwrite it - VERY OLD THING preexistingQM.delete() newQM.expoyear = year diff --git a/parsers/caves.py b/parsers/caves.py index fa2aea3..f7f16b3 100644 --- a/parsers/caves.py +++ b/parsers/caves.py @@ -1,6 +1,5 @@ import os import re -import sys from pathlib import Path from django.conf import settings @@ -44,7 +43,7 @@ def dummy_entrance(k, slug, msg="DUMMY"): if ent: ent.save() # must save to have id before foreign keys work. try: # Now create a entrance slug ID - es = EntranceSlug(entrance=ent, slug=slug, primary=False) + EntranceSlug(entrance=ent, slug=slug, primary=False) except: message = f" ! {k:11s} {msg}-{slug} entrance create failure" DataIssue.objects.create(parser="caves", message=message, url=f"{slug}") @@ -66,9 +65,8 @@ def set_dummy_entrance(id, slug, cave, msg="DUMMY"): global entrances_xslug try: entrance = dummy_entrance(id, slug, msg="DUMMY") - letter = "" entrances_xslug[slug] = entrance - ce = CaveAndEntrance.objects.update_or_create(cave=cave, entrance_letter="", entrance=entrance) + CaveAndEntrance.objects.update_or_create(cave=cave, entrance_letter="", entrance=entrance) message = f" ! Warning: Dummy Entrance successfully set for entrance {id} on cave {cave}" DataIssue.objects.create(parser="caves", message=message, url=f"{cave.url}") @@ -134,37 +132,37 @@ def do_pending_cave(k, url, area): print(message) return - default_note = f"_Survex file found in loser repo but no description in expoweb


\n" - default_note += f"INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then " - default_note += f'

\n\n - (0) look in the cave number index for notes on this cave, ' - default_note += f"

\n\n - (1) search in the survex file for the *ref to find a " - default_note += f"relevant wallet, e.g.2009#11 and read the notes image files
\n - " + default_note = "_Survex file found in loser repo but no description in expoweb


\n" + default_note += "INSTRUCTIONS: First open 'This survex file' (link above the CaveView panel) to find the date and info. Then " + default_note += '

\n\n - (0) look in the cave number index for notes on this cave, ' + default_note += "

\n\n - (1) search in the survex file for the *ref to find a " + default_note += "relevant wallet, e.g.2009#11 and read the notes image files
\n - " default_note += ( - f"

\n\n - (2) search in the Expo for that year e.g. 2009 to find a " + "

\n\n - (2) search in the Expo for that year e.g. 2009 to find a " ) - default_note += f"relevant logbook entry, remember that the date may have been recorded incorrectly, " + default_note += "relevant logbook entry, remember that the date may have been recorded incorrectly, " default_note += ( - f"so check for trips i.e. logbook entries involving the same people as were listed in the survex file, " + "so check for trips i.e. logbook entries involving the same people as were listed in the survex file, " ) default_note += ( - f"and you should also check the scanned copy of the logbook (linked from each logbook entry page) " + "and you should also check the scanned copy of the logbook (linked from each logbook entry page) " ) - default_note += f"just in case a vital trip was not transcribed, then
\n - " + default_note += "just in case a vital trip was not transcribed, then
\n - " default_note += ( - f"click on 'Edit this cave' and copy the information you find in the survex file and the logbook" + "click on 'Edit this cave' and copy the information you find in the survex file and the logbook" ) - default_note += f"and delete all the text in the 'Notes' section - which is the text you are reading now." - default_note += f"

\n\n - Only two fields on this form are essential. " - default_note += f"Documentation of all the fields on 'Edit this cave' form is in handbook/survey/caveentryfields" - default_note += f"

\n\n - " - default_note += f"You will also need to create a new entrance from the 'Edit this cave' page. Ignore the existing dummy one, it will evaporate on the next full import." - default_note += f"

\n\n - " - default_note += f"When you Submit it will create a new file in expoweb/cave_data/ " + default_note += "and delete all the text in the 'Notes' section - which is the text you are reading now." + default_note += "

\n\n - Only two fields on this form are essential. " + default_note += "Documentation of all the fields on 'Edit this cave' form is in handbook/survey/caveentryfields" + default_note += "

\n\n - " + default_note += "You will also need to create a new entrance from the 'Edit this cave' page. Ignore the existing dummy one, it will evaporate on the next full import." + default_note += "

\n\n - " + default_note += "When you Submit it will create a new file in expoweb/cave_data/ " default_note += ( - f"

\n\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. " + "

\n\n - Now you can edit the entrance info: click on Edit below for the dummy entrance. " ) - default_note += f"and then Submit to save it (if you forget to do this, a dummy entrance will be created for your new cave description)." - default_note += f"

\n\n - Finally, you need to find a nerd to edit the file 'expoweb/cave_data/pending.txt' " + default_note += "and then Submit to save it (if you forget to do this, a dummy entrance will be created for your new cave description)." + default_note += "

\n\n - Finally, you need to find a nerd to edit the file 'expoweb/cave_data/pending.txt' " default_note += ( f"to remove the line
{slug}
as it is no longer 'pending' but 'done. Well Done." ) @@ -187,7 +185,7 @@ def do_pending_cave(k, url, area): print(message) try: # Now create a cave slug ID - cs = CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False) + CaveSlug.objects.update_or_create(cave=cave, slug=slug, primary=False) except: message = f" ! {k:11s} PENDING cave SLUG create failure" DataIssue.objects.create(parser="caves", message=message) @@ -292,7 +290,7 @@ def readentrance(filename): for slug in slugs: # print("entrance slug:{} filename:{}".format(slug, filename)) try: - cs = EntranceSlug.objects.update_or_create(entrance=e, slug=slug, primary=primary) + EntranceSlug.objects.update_or_create(entrance=e, slug=slug, primary=primary) except: # need to cope with duplicates message = f" ! FAILED to get precisely one ENTRANCE when updating using: cave_entrance/{filename}" @@ -303,10 +301,9 @@ def readentrance(filename): DataIssue.objects.create(parser="caves", message=message, url=f"/cave/{slug}/edit/") print(message) for k in kents: - if k.slug() != None: + if k.slug() is not None: print(" ! - OVERWRITING this one: slug:" + str(k.slug())) k.notes = "DUPLICATE entrance found on import. Please fix\n" + k.notes - c = k primary = False # else: # more than one item in long list. But this is not an error, and the max and min have been checked by getXML # slug = Path(filename).stem @@ -417,7 +414,7 @@ def readcave(filename): DataIssue.objects.create(parser="caves", message=message) print(message) for k in kaves: - if k.slug() != None: + if k.slug() is not None: print(" ! - OVERWRITING this one: slug:" + str(k.slug())) k.notes = "DUPLICATE kataster number found on import. Please fix\n" + k.notes c = k @@ -466,7 +463,7 @@ def readcave(filename): else: entrance = Entrance.objects.get(entranceslug__slug=eslug) entrances_xslug[eslug] = entrance - ce = CaveAndEntrance.objects.update_or_create( + CaveAndEntrance.objects.update_or_create( cave=c, entrance_letter=letter, entrance=entrance ) except: @@ -586,7 +583,7 @@ def readcaves(): readcave(filename) print(" - Setting up all the variously useful alias names") - mycavelookup = GetCaveLookup() + GetCaveLookup() print(" - Setting pending caves") # Do this last, so we can detect if they are created and no longer 'pending' diff --git a/parsers/drawings.py b/parsers/drawings.py index d3e8491..963046d 100644 --- a/parsers/drawings.py +++ b/parsers/drawings.py @@ -1,17 +1,11 @@ -import csv -import datetime import os import re import stat -import sys -import types -from functools import reduce from pathlib import Path import settings -from troggle.core.models.survex import DrawingFile, SingleScan, Wallet +from troggle.core.models.survex import DrawingFile, Wallet from troggle.core.models.troggle import DataIssue -from troggle.core.utils import save_carefully """Searches through all the :drawings: repository looking for tunnel and therion files diff --git a/parsers/imports.py b/parsers/imports.py index d59c792..c1de034 100644 --- a/parsers/imports.py +++ b/parsers/imports.py @@ -1,11 +1,6 @@ -import os -import sys import django -from django.contrib.auth.models import User -from django.core import management -from django.db import close_old_connections, connection, connections, transaction -from django.http import HttpResponse +from django.db import transaction import troggle.parsers.caves import troggle.parsers.drawings diff --git a/parsers/logbooks.py b/parsers/logbooks.py index 61f17e3..0ab902a 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -1,17 +1,14 @@ -import csv import os import re -import time from datetime import date, datetime from pathlib import Path from random import randint from django.conf import settings from django.template.defaultfilters import slugify -from django.utils.timezone import get_current_timezone, make_aware from parsers.people import GetPersonExpeditionNameLookup -from troggle.core.models.caves import Cave, GetCaveLookup, LogbookEntry, PersonTrip +from troggle.core.models.caves import GetCaveLookup, LogbookEntry, PersonTrip from troggle.core.models.troggle import DataIssue, Expedition from troggle.core.utils import TROG, save_carefully @@ -254,7 +251,7 @@ def EnterLogIntoDbase(date, place, title, text, trippeople, expedition, logtime_ text = text.replace("\n\n\n", "\n\n") # Check for an existing copy of the current entry, and save - expeditionday = expedition.get_expedition_day(date) + expedition.get_expedition_day(date) lookupAttribs = {"date": date, "title": title} # 'cave' is converted to a string doing this, which renders as the cave slug. # but it is a db query which we should try to avoid - rewrite this @@ -574,7 +571,6 @@ def parser_blog(year, expedition, txt, sq=""): """ global logentries global logdataissues - errorcount = 0 tripheads = re.findall( r"
{survexfile.path} " @@ -1181,7 +1177,7 @@ class LoadingSurvex: slengthtotal = 0.0 nlegstotal = 0 self.relativefilename = path - cave = self.IdentifyCave(path) # this will produce null for survex files which are geographic collections + self.IdentifyCave(path) # this will produce null for survex files which are geographic collections self.currentsurvexfile = survexblock.survexfile self.currentsurvexfile.save() # django insists on this although it is already saved !? @@ -1628,7 +1624,7 @@ class LoadingSurvex: DataIssue.objects.create(parser="xEntrances", message=message, url=url) print(message) print( - f"stderr:\n\n" + str(sp.stderr) + "\n\n" + str(sp.stdout) + "\n\nreturn code: " + str(sp.returncode) + "stderr:\n\n" + str(sp.stderr) + "\n\n" + str(sp.stdout) + "\n\nreturn code: " + str(sp.returncode) ) self.caverncount += 1 @@ -1643,7 +1639,7 @@ class LoadingSurvex: svxpath = Path(fullpath + ".svx") logpath = Path(fullpath + ".log") - outputdir = Path(svxpath).parent + Path(svxpath).parent if not svxpath.is_file(): message = f' ! BAD survex file "{fullpath}" specified in *include in {calledpath} ' @@ -1664,7 +1660,7 @@ class LoadingSurvex: DataIssue.objects.create(parser="entrances", message=message) print(message) print( - f"stderr:\n\n" + str(sp.stderr) + "\n\n" + str(sp.stdout) + "\n\nreturn code: " + str(sp.returncode) + "stderr:\n\n" + str(sp.stderr) + "\n\n" + str(sp.stdout) + "\n\nreturn code: " + str(sp.returncode) ) self.caverndate = os.path.getmtime(sp.stdout.strip()) else: @@ -1786,7 +1782,7 @@ def FindAndLoadSurvex(survexblockroot): f"\n - {len(unseens)} survex files found which were not included in main tree. ({len(svx_scan.svxfileslist)} in main tree)", file=sys.stderr, ) - print(f" -- Now loading the previously-omitted survex files.", file=sys.stderr) + print(" -- Now loading the previously-omitted survex files.", file=sys.stderr) with open(Path(settings.SURVEX_DATA, "_unseens.svx"), "w") as u: u.write( @@ -1794,10 +1790,10 @@ def FindAndLoadSurvex(survexblockroot): ) u.write(f"; autogenerated by parser/survex.py from databasereset.py on '{datetime.now(timezone.utc)}'\n") u.write(f"; omitting any file beginning with {excpts}\n\n") - u.write(f"*begin unseens\n") + u.write("*begin unseens\n") for x in sorted(unseens): u.write(f" *include {x}\n") - u.write(f"*end unseens\n") + u.write("*end unseens\n") survexfileroot = survexblockroot.survexfile # i.e. SURVEX_TOPNAME only @@ -2065,7 +2061,6 @@ def LoadPositions(): print(f" - Generating a list of Pos from {topdata}.svx and then loading...") found = 0 - skip = {} print("\n") # extra line because cavern overwrites the text buffer somehow # cavern defaults to using same cwd as supplied input file @@ -2113,7 +2108,7 @@ def LoadPositions(): try: survexblockroot = SurvexBlock.objects.get(id=1) except: - message = f" ! FAILED to find root SurvexBlock" + message = " ! FAILED to find root SurvexBlock" print(message) DataIssue.objects.create(parser="entrances", message=message) raise @@ -2131,17 +2126,16 @@ def LoadPositions(): try: sbqs = SurvexBlock.objects.filter(survexpath=blockpath) if len(sbqs) == 1: - sb = sbqs[0] + sbqs[0] if len(sbqs) > 1: message = f" ! MULTIPLE SurvexBlocks {len(sbqs):3} matching Entrance point {blockpath} {sid} '{id}'" print(message) DataIssue.objects.create(parser="entrances", message=message) - sb = sbqs[0] + sbqs[0] elif len(sbqs) <= 0: message = f" ! ZERO SurvexBlocks matching Entrance point {blockpath} {sid} '{id}'" print(message) DataIssue.objects.create(parser="entrances", message=message) - sb = survexblockroot except: message = f" ! FAIL in getting SurvexBlock matching Entrance point {blockpath} {sid}" print(message) diff --git a/reset-django.py b/reset-django.py index c847fa4..d822d07 100644 --- a/reset-django.py +++ b/reset-django.py @@ -1,6 +1,5 @@ import os import shutil -from pprint import pprint """Cleans all django-created files and compiled python. Used by the pre-run.sh script which cleans and initialises everything before @@ -79,7 +78,7 @@ def main(): delete_sqlite3() print("All cleanup operations performed successfully.") - except Exception as e: + except Exception: print("There was some error! Aaargh. \n") raise diff --git a/settings.py b/settings.py index f93d00b..d21d396 100644 --- a/settings.py +++ b/settings.py @@ -14,10 +14,7 @@ https://docs.djangoproject.com/en/dev/ref/settings/ # 3.Local application/library specific imports. # 4.You should put a blank line between each group of imports. -import os -import urllib.parse -import django print("* importing troggle/settings.py") diff --git a/urls.py b/urls.py index b1e2637..0156422 100644 --- a/urls.py +++ b/urls.py @@ -1,15 +1,12 @@ from django.conf import settings -from django.contrib import admin, auth -from django.urls import include, path, re_path, resolve, reverse -from django.views.generic.base import RedirectView -from django.views.generic.edit import UpdateView -from django.views.generic.list import ListView +from django.contrib import admin +from django.urls import include, path, re_path from troggle.core.views import statistics, survex from troggle.core.views.auth import expologin, expologout from troggle.core.views.caves import (cave3d, caveEntrance, caveindex, cavepage, caveQMs, edit_cave, - edit_entrance, ent, get_entrances, qm) + edit_entrance, get_entrances, qm) from troggle.core.views.drawings import dwgallfiles, dwgfilesingle from troggle.core.views.editor_helpers import image_selector, new_image_form from troggle.core.views.expo import (editexpopage, expofiles_redirect, @@ -21,12 +18,10 @@ from troggle.core.views.logbooks import (Expeditions_jsonListView, logbookentry, notablepersons, person, personexpedition) from troggle.core.views.other import (controlpanel, exportlogbook, frontpage, - todos, troggle404) -from troggle.core.views.prospect import prospecting, prospecting_image + todos) +from troggle.core.views.prospect import prospecting from troggle.core.views.scans import (allscans, cavewallets, scansingle, walletslistperson, walletslistyear) -from troggle.core.views.statistics import dataissues, pathsreport, stats -from troggle.core.views.survex import survexcavesingle, survexcaveslist, svx from troggle.core.views.uploads import dwgupload, photoupload, scanupload """This sets the actualurlpatterns[] and urlpatterns[] lists which django uses