From f666b9c3963b321296ec51e5d8c8dd149ff60478 Mon Sep 17 00:00:00 2001
From: Sam Wenham <sam@wenhams.co.uk>
Date: Sat, 30 Mar 2019 13:58:38 +0000
Subject: [PATCH] Update new management command for DB reset Switch to
content_type from mimetype Make DB reset not nuke so much Tidy logbook parser
---
core/management/commands/reset_db.py | 89 ++++++++++++++--------------
core/views_other.py | 37 ++++++------
databaseReset.py | 20 +++----
parsers/logbooks.py | 19 +++---
parsers/people.py | 4 +-
templates/frontpage.html | 2 +-
6 files changed, 85 insertions(+), 86 deletions(-)
diff --git a/core/management/commands/reset_db.py b/core/management/commands/reset_db.py
index 29f8e81..e95ddd9 100644
--- a/core/management/commands/reset_db.py
+++ b/core/management/commands/reset_db.py
@@ -2,6 +2,14 @@ from django.core.management.base import BaseCommand, CommandError
from optparse import make_option
from troggle.core.models import Cave
import settings
+import os
+
+from django.db import connection
+from django.core import management
+from django.contrib.auth.models import User
+from django.core.urlresolvers import reverse
+from troggle.core.models import Cave, Entrance
+import troggle.flatpages.models
databasename=settings.DATABASES['default']['NAME']
expouser=settings.EXPOUSER
@@ -12,22 +20,13 @@ class Command(BaseCommand):
help = 'This is normal usage, clear database and reread everything'
option_list = BaseCommand.option_list + (
- make_option('--foo',
+ make_option('--reset',
action='store_true',
- dest='foo',
+ dest='reset',
default=False,
- help='test'),
+ help='Reset the entier DB from files'),
)
- def add_arguments(self, parser):
-
- parser.add_argument(
- '--foo',
- action='store_true',
- dest='foo',
- help='Help text',
- )
-
def handle(self, *args, **options):
print(args)
print(options)
@@ -46,8 +45,8 @@ class Command(BaseCommand):
self.import_QMs()
elif "tunnel" in args:
self.import_tunnelfiles()
- elif "reset" in args:
- self.reset()
+ elif options['reset']:
+ self.reset(self)
elif "survex" in args:
self.import_survex()
elif "survexpos" in args:
@@ -61,13 +60,15 @@ class Command(BaseCommand):
self.dumplogbooks()
elif "writeCaves" in args:
self.writeCaves()
- elif "foo" in args:
- self.stdout.write('Tesing....')
+ elif options['foo']:
+ self.stdout.write(self.style.WARNING('Tesing....'))
else:
- self.stdout.write("%s not recognised" % args)
- self.usage(options)
+ #self.stdout.write("%s not recognised" % args)
+ #self.usage(options)
+ self.stdout.write("poo")
+ #print(args)
- def reload_db():
+ def reload_db(obj):
if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
try:
os.remove(databasename)
@@ -86,22 +87,22 @@ class Command(BaseCommand):
user.is_superuser = True
user.save()
- def make_dirs():
+ def make_dirs(obj):
"""Make directories that troggle requires"""
# should also deal with permissions here.
if not os.path.isdir(settings.PHOTOS_ROOT):
os.mkdir(settings.PHOTOS_ROOT)
- def import_caves():
+ def import_caves(obj):
import parsers.caves
- print("importing caves")
+ print("Importing Caves")
parsers.caves.readcaves()
- def import_people():
+ def import_people(obj):
import parsers.people
parsers.people.LoadPersonsExpos()
- def import_logbooks():
+ def import_logbooks(obj):
# The below line was causing errors I didn't understand (it said LOGFILE was a string), and I couldn't be bothered to figure
# what was going on so I just catch the error with a try. - AC 21 May
try:
@@ -112,57 +113,57 @@ class Command(BaseCommand):
import parsers.logbooks
parsers.logbooks.LoadLogbooks()
- def import_survex():
+ def import_survex(obj):
import parsers.survex
parsers.survex.LoadAllSurvexBlocks()
parsers.survex.LoadPos()
- def import_QMs():
+ def import_QMs(obj):
import parsers.QMs
- def import_surveys():
+ def import_surveys(obj):
import parsers.surveys
parsers.surveys.parseSurveys(logfile=settings.LOGFILE)
- def import_surveyscans():
+ def import_surveyscans(obj):
import parsers.surveys
parsers.surveys.LoadListScans()
- def import_tunnelfiles():
+ def import_tunnelfiles(obj):
import parsers.surveys
parsers.surveys.LoadTunnelFiles()
- def reset():
+ def reset(self, mgmt_obj):
""" Wipe the troggle database and import everything from legacy data
"""
- reload_db()
- make_dirs()
- pageredirects()
- import_caves()
- import_people()
- import_surveyscans()
- import_survex()
- import_logbooks()
- import_QMs()
+ self.reload_db()
+ self.make_dirs()
+ self.pageredirects()
+ self.import_caves()
+ self.import_people()
+ self.import_surveyscans()
+ self.import_survex()
+ self.import_logbooks()
+ self.import_QMs()
try:
- import_tunnelfiles()
+ self.import_tunnelfiles()
except:
print("Tunnel files parser broken.")
- import_surveys()
+ self.import_surveys()
- def pageredirects():
+ def pageredirects(obj):
for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]:
f = troggle.flatpages.models.Redirect(originalURL=oldURL, newURL=newURL)
f.save()
- def writeCaves():
+ def writeCaves(obj):
for cave in Cave.objects.all():
cave.writeDataFile()
for entrance in Entrance.objects.all():
entrance.writeDataFile()
- def usage(self, parser):
+ def troggle_usage(obj):
print("""Usage is 'manage.py reset_db <command>'
where command is:
reset - this is normal usage, clear database and reread everything
diff --git a/core/views_other.py b/core/views_other.py
index d99cc32..6a195b9 100644
--- a/core/views_other.py
+++ b/core/views_other.py
@@ -59,8 +59,8 @@ def controlPanel(request):
databaseReset.make_dirs()
for item in importlist:
if item in request.POST:
- print "running"+ " databaseReset."+item+"()"
- exec "databaseReset."+item+"()"
+ print("running"+ " databaseReset."+item+"()")
+ exec("databaseReset."+item+"()")
jobs_completed.append(item)
else:
if request.user.is_authenticated(): #The user is logged in, but is not a superuser.
@@ -72,14 +72,14 @@ def controlPanel(request):
def downloadCavetab(request):
from export import tocavetab
- response = HttpResponse(mimetype='text/csv')
+ response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename=CAVETAB2.CSV'
tocavetab.writeCaveTab(response)
return response
def downloadSurveys(request):
from export import tosurveys
- response = HttpResponse(mimetype='text/csv')
+ response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename=Surveys.csv'
tosurveys.writeCaveTab(response)
return response
@@ -94,20 +94,19 @@ def downloadLogbook(request,year=None,extension=None,queryset=None):
logbook_entries=queryset
filename='logbook'
else:
+ response = HttpResponse(content_type='text/plain')
return response(r"Error: Logbook downloader doesn't know what year you want")
if 'year' in request.GET:
year=request.GET['year']
if 'extension' in request.GET:
extension=request.GET['extension']
-
-
-
+
if extension =='txt':
- response = HttpResponse(mimetype='text/plain')
+ response = HttpResponse(content_type='text/plain')
style='2008'
elif extension == 'html':
- response = HttpResponse(mimetype='text/html')
+ response = HttpResponse(content_type='text/html')
style='2005'
template='logbook'+style+'style.'+extension
@@ -124,11 +123,11 @@ def downloadQMs(request):
try:
cave=Cave.objects.get(kataster_number=request.GET['cave_id'])
except Cave.DoesNotExist:
- cave=Cave.objects.get(name=cave_id)
+ cave=Cave.objects.get(name=request.GET['cave_id'])
from export import toqms
- response = HttpResponse(mimetype='text/csv')
+ response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename=qm.csv'
toqms.writeQmTable(response,cave)
return response
@@ -136,7 +135,7 @@ def downloadQMs(request):
def ajax_test(request):
post_text = request.POST['post_data']
return HttpResponse("{'response_text': '"+post_text+" recieved.'}",
- mimetype="application/json")
+ content_type="application/json")
def eyecandy(request):
return
@@ -144,9 +143,9 @@ def eyecandy(request):
def ajax_QM_number(request):
if request.method=='POST':
cave=Cave.objects.get(id=request.POST['cave'])
- print cave
+ print(cave)
exp=Expedition.objects.get(pk=request.POST['year'])
- print exp
+ print(exp)
res=cave.new_QM_number(exp.year)
return HttpResponse(res)
@@ -167,7 +166,7 @@ def logbook_entry_suggestions(request):
#unwiki_QMs=re.findall(unwiki_QM_pattern,lbo.text)
unwiki_QMs=[m.groupdict() for m in unwiki_QM_pattern.finditer(lbo.text)]
- print unwiki_QMs
+ print(unwiki_QMs)
for qm in unwiki_QMs:
#try:
if len(qm['year'])==2:
@@ -180,7 +179,7 @@ def logbook_entry_suggestions(request):
try:
lbo=LogbookEntry.objects.get(date__year=qm['year'],title__icontains="placeholder for QMs in")
except:
- print "failed to get placeholder for year "+str(qm['year'])
+ print("failed to get placeholder for year "+str(qm['year']))
temp_QM=QM(found_by=lbo,number=qm['number'],grade=qm['grade'])
temp_QM.grade=qm['grade']
@@ -188,7 +187,7 @@ def logbook_entry_suggestions(request):
#except:
#print 'failed'
- print unwiki_QMs
+ print(unwiki_QMs)
#wikilink_QMs=re.findall(wikilink_QM_pattern,lbo.text)
@@ -199,9 +198,9 @@ def logbook_entry_suggestions(request):
#for qm in wikilink_QMs:
#Try to look up the QM.
- print 'got 208'
+ print('got 208')
any_suggestions=True
- print 'got 210'
+ print('got 210')
return render_with_context(request,'suggestions.html',
{
'unwiki_QMs':unwiki_QMs,
diff --git a/databaseReset.py b/databaseReset.py
index c296274..7a5d0fa 100644
--- a/databaseReset.py
+++ b/databaseReset.py
@@ -109,10 +109,10 @@ def import_auto_logbooks():
for lbe in troggle.core.models.LogbookEntry.objects.all():
lbe.delete()
for expedition in troggle.core.models.Expedition.objects.all():
- directory = os.path.join(settings.EXPOWEB,
- "years",
- expedition.year,
- "autologbook")
+ directory = os.path.join(settings.EXPOWEB,
+ "years",
+ expedition.year,
+ "autologbook")
for root, dirs, filenames in os.walk(directory):
for filename in filenames:
print(os.path.join(root, filename))
@@ -195,9 +195,9 @@ if __name__ == "__main__":
elif "scans" in sys.argv:
import_surveyscans()
elif "caves" in sys.argv:
- reload_db()
- make_dirs()
- pageredirects()
+ # reload_db()
+ # make_dirs()
+ # pageredirects()
import_caves()
elif "people" in sys.argv:
import_people()
@@ -218,14 +218,14 @@ if __name__ == "__main__":
import_descriptions()
parse_descriptions()
elif "survex" in sys.argv:
- management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
+ # management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
import_survex()
elif "survexpos" in sys.argv:
- management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
+ # management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
import parsers.survex
parsers.survex.LoadPos()
elif "logbooks" in sys.argv:
- management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
+ # management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex
import_logbooks()
elif "autologbooks" in sys.argv:
import_auto_logbooks()
diff --git a/parsers/logbooks.py b/parsers/logbooks.py
index ffd8e21..fbe00a3 100644
--- a/parsers/logbooks.py
+++ b/parsers/logbooks.py
@@ -115,7 +115,7 @@ def ParseDate(tripdate, year):
assert False, tripdate
return datetime.date(year, month, day)
-# 2007, 2008, 2006
+# 2006, 2008 - 2010
def Parselogwikitxt(year, expedition, txt):
trippara = re.findall(r"===(.*?)===([\s\S]*?)(?====)", txt)
for triphead, triptext in trippara:
@@ -140,9 +140,9 @@ def Parselogwikitxt(year, expedition, txt):
#print "\n", tripcave, "--- ppp", trippeople, len(triptext)
EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
-# 2002, 2004, 2005
+# 2002, 2004, 2005, 2007, 2011 - 2018
def Parseloghtmltxt(year, expedition, txt):
- print(" - Using log html parser")
+ #print(" - Starting log html parser")
tripparas = re.findall(r"<hr\s*/>([\s\S]*?)(?=<hr)", txt)
logbook_entry_count = 0
for trippara in tripparas:
@@ -163,7 +163,6 @@ def Parseloghtmltxt(year, expedition, txt):
print("can't parse: ", trippara) # this is 2007 which needs editing
#assert s, trippara
continue
-
tripid, tripid1, tripdate, trippeople, triptitle, triptext, tu = s.groups()
ldate = ParseDate(tripdate.strip(), year)
#assert tripid[:-1] == "t" + tripdate, (tripid, tripdate)
@@ -174,7 +173,7 @@ def Parseloghtmltxt(year, expedition, txt):
tripcave = triptitles[0]
else:
tripcave = "UNKNOWN"
- #print "\n", tripcave, "--- ppp", trippeople, len(triptext)
+ #print("\n", tripcave, "--- ppp", trippeople, len(triptext))
ltriptext = re.sub(r"</p>", "", triptext)
ltriptext = re.sub(r"\s*?\n\s*", " ", ltriptext)
ltriptext = re.sub(r"<p>", "\n\n", ltriptext).strip()
@@ -183,7 +182,7 @@ def Parseloghtmltxt(year, expedition, txt):
print(" - No trip entrys found in logbook, check the syntax matches htmltxt format")
-# main parser for pre-2001. simpler because the data has been hacked so much to fit it
+# main parser for 1991 - 2001. simpler because the data has been hacked so much to fit it
def Parseloghtml01(year, expedition, txt):
tripparas = re.findall(r"<hr[\s/]*>([\s\S]*?)(?=<hr)", txt)
for trippara in tripparas:
@@ -229,7 +228,7 @@ def Parseloghtml01(year, expedition, txt):
# could includ the tripid (url link for cross referencing)
EnterLogIntoDbase(date=ldate, place=tripcave, title=triptitle, text=ltriptext, trippeople=trippeople, expedition=expedition, logtime_underground=0)
-
+# parser for 2003
def Parseloghtml03(year, expedition, txt):
tripparas = re.findall(r"<hr\s*/>([\s\S]*?)(?=<hr)", txt)
for trippara in tripparas:
@@ -281,8 +280,7 @@ def SetDatesFromLogbookEntries(expedition):
def LoadLogbookForExpedition(expedition):
""" Parses all logbook entries for one expedition """
- expowebbase = os.path.join(settings.EXPOWEB, "years")
- #year = str(expedition.year)
+ expowebbase = os.path.join(settings.EXPOWEB, "years")
yearlinks = settings.LOGBOOK_PARSER_SETTINGS
logbook_parseable = False
@@ -294,6 +292,7 @@ def LoadLogbookForExpedition(expedition):
file_in.close()
parsefunc = year_settings[1]
logbook_parseable = True
+ print(" - Parsing logbook: " + year_settings[0] + "\n - Using parser: " + year_settings[1])
else:
try:
file_in = open(os.path.join(expowebbase, expedition.year, settings.DEFAULT_LOGBOOK_FILE))
@@ -304,7 +303,7 @@ def LoadLogbookForExpedition(expedition):
parsefunc = settings.DEFAULT_LOGBOOK_PARSER
except (IOError):
logbook_parseable = False
- print("Couldn't open default logbook file and nothing set for expo " + expedition.year)
+ print("Couldn't open default logbook file and nothing in settings for expo " + expedition.year)
if logbook_parseable:
parser = globals()[parsefunc]
diff --git a/parsers/people.py b/parsers/people.py
index 3c3fc03..48d6c17 100644
--- a/parsers/people.py
+++ b/parsers/people.py
@@ -67,8 +67,8 @@ def LoadPersonsExpos():
for personline in personreader:
name = personline[header["Name"]]
- name = re.sub("<.*?>", "", name)
- mname = re.match("(\w+)(?:\s((?:van |ten )?\w+))?(?:\s\(([^)]*)\))?", name)
+ name = re.sub(r"<.*?>", "", name)
+ mname = re.match(r"(\w+)(?:\s((?:van |ten )?\w+))?(?:\s\(([^)]*)\))?", name)
nickname = mname.group(3) or ""
lookupAttribs={'first_name':mname.group(1), 'last_name':(mname.group(2) or "")}
diff --git a/templates/frontpage.html b/templates/frontpage.html
index 28206b6..eed6098 100644
--- a/templates/frontpage.html
+++ b/templates/frontpage.html
@@ -16,7 +16,7 @@
{% if entry.is_deletion %}
{{ entry.object_repr }}
{% else %}
- <a href="admin/{{ entry.get_admin_url }}">{{ entry.object_repr }}</a>
+ <a href="admin/{{ entry.get_admin_url }}/">{{ entry.object_repr }}</a>
{% endif %}
<br/>
{% if entry.content_type %}