From a4532a29da6c2a92553daeafbd3c7eca5b42f861 Mon Sep 17 00:00:00 2001 From: Sam Wenham Date: Sat, 30 Mar 2019 13:58:38 +0000 Subject: [PATCH] Update new management command for DB reset Switch to content_type from mimetype Make DB reset not nuke so much Tidy logbook parser --- core/management/commands/reset_db.py | 89 ++++++++++++++-------------- core/views_other.py | 37 ++++++------ databaseReset.py | 20 +++---- parsers/logbooks.py | 19 +++--- parsers/people.py | 4 +- templates/frontpage.html | 2 +- 6 files changed, 85 insertions(+), 86 deletions(-) diff --git a/core/management/commands/reset_db.py b/core/management/commands/reset_db.py index 29f8e81..e95ddd9 100644 --- a/core/management/commands/reset_db.py +++ b/core/management/commands/reset_db.py @@ -2,6 +2,14 @@ from django.core.management.base import BaseCommand, CommandError from optparse import make_option from troggle.core.models import Cave import settings +import os + +from django.db import connection +from django.core import management +from django.contrib.auth.models import User +from django.core.urlresolvers import reverse +from troggle.core.models import Cave, Entrance +import troggle.flatpages.models databasename=settings.DATABASES['default']['NAME'] expouser=settings.EXPOUSER @@ -12,22 +20,13 @@ class Command(BaseCommand): help = 'This is normal usage, clear database and reread everything' option_list = BaseCommand.option_list + ( - make_option('--foo', + make_option('--reset', action='store_true', - dest='foo', + dest='reset', default=False, - help='test'), + help='Reset the entier DB from files'), ) - def add_arguments(self, parser): - - parser.add_argument( - '--foo', - action='store_true', - dest='foo', - help='Help text', - ) - def handle(self, *args, **options): print(args) print(options) @@ -46,8 +45,8 @@ class Command(BaseCommand): self.import_QMs() elif "tunnel" in args: self.import_tunnelfiles() - elif "reset" in args: - self.reset() + elif options['reset']: + self.reset(self) elif "survex" in args: self.import_survex() elif "survexpos" in args: @@ -61,13 +60,15 @@ class Command(BaseCommand): self.dumplogbooks() elif "writeCaves" in args: self.writeCaves() - elif "foo" in args: - self.stdout.write('Tesing....') + elif options['foo']: + self.stdout.write(self.style.WARNING('Tesing....')) else: - self.stdout.write("%s not recognised" % args) - self.usage(options) + #self.stdout.write("%s not recognised" % args) + #self.usage(options) + self.stdout.write("poo") + #print(args) - def reload_db(): + def reload_db(obj): if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': try: os.remove(databasename) @@ -86,22 +87,22 @@ class Command(BaseCommand): user.is_superuser = True user.save() - def make_dirs(): + def make_dirs(obj): """Make directories that troggle requires""" # should also deal with permissions here. if not os.path.isdir(settings.PHOTOS_ROOT): os.mkdir(settings.PHOTOS_ROOT) - def import_caves(): + def import_caves(obj): import parsers.caves - print("importing caves") + print("Importing Caves") parsers.caves.readcaves() - def import_people(): + def import_people(obj): import parsers.people parsers.people.LoadPersonsExpos() - def import_logbooks(): + def import_logbooks(obj): # The below line was causing errors I didn't understand (it said LOGFILE was a string), and I couldn't be bothered to figure # what was going on so I just catch the error with a try. - AC 21 May try: @@ -112,57 +113,57 @@ class Command(BaseCommand): import parsers.logbooks parsers.logbooks.LoadLogbooks() - def import_survex(): + def import_survex(obj): import parsers.survex parsers.survex.LoadAllSurvexBlocks() parsers.survex.LoadPos() - def import_QMs(): + def import_QMs(obj): import parsers.QMs - def import_surveys(): + def import_surveys(obj): import parsers.surveys parsers.surveys.parseSurveys(logfile=settings.LOGFILE) - def import_surveyscans(): + def import_surveyscans(obj): import parsers.surveys parsers.surveys.LoadListScans() - def import_tunnelfiles(): + def import_tunnelfiles(obj): import parsers.surveys parsers.surveys.LoadTunnelFiles() - def reset(): + def reset(self, mgmt_obj): """ Wipe the troggle database and import everything from legacy data """ - reload_db() - make_dirs() - pageredirects() - import_caves() - import_people() - import_surveyscans() - import_survex() - import_logbooks() - import_QMs() + self.reload_db() + self.make_dirs() + self.pageredirects() + self.import_caves() + self.import_people() + self.import_surveyscans() + self.import_survex() + self.import_logbooks() + self.import_QMs() try: - import_tunnelfiles() + self.import_tunnelfiles() except: print("Tunnel files parser broken.") - import_surveys() + self.import_surveys() - def pageredirects(): + def pageredirects(obj): for oldURL, newURL in [("indxal.htm", reverse("caveindex"))]: f = troggle.flatpages.models.Redirect(originalURL=oldURL, newURL=newURL) f.save() - def writeCaves(): + def writeCaves(obj): for cave in Cave.objects.all(): cave.writeDataFile() for entrance in Entrance.objects.all(): entrance.writeDataFile() - def usage(self, parser): + def troggle_usage(obj): print("""Usage is 'manage.py reset_db ' where command is: reset - this is normal usage, clear database and reread everything diff --git a/core/views_other.py b/core/views_other.py index d99cc32..6a195b9 100644 --- a/core/views_other.py +++ b/core/views_other.py @@ -59,8 +59,8 @@ def controlPanel(request): databaseReset.make_dirs() for item in importlist: if item in request.POST: - print "running"+ " databaseReset."+item+"()" - exec "databaseReset."+item+"()" + print("running"+ " databaseReset."+item+"()") + exec("databaseReset."+item+"()") jobs_completed.append(item) else: if request.user.is_authenticated(): #The user is logged in, but is not a superuser. @@ -72,14 +72,14 @@ def controlPanel(request): def downloadCavetab(request): from export import tocavetab - response = HttpResponse(mimetype='text/csv') + response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename=CAVETAB2.CSV' tocavetab.writeCaveTab(response) return response def downloadSurveys(request): from export import tosurveys - response = HttpResponse(mimetype='text/csv') + response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename=Surveys.csv' tosurveys.writeCaveTab(response) return response @@ -94,20 +94,19 @@ def downloadLogbook(request,year=None,extension=None,queryset=None): logbook_entries=queryset filename='logbook' else: + response = HttpResponse(content_type='text/plain') return response(r"Error: Logbook downloader doesn't know what year you want") if 'year' in request.GET: year=request.GET['year'] if 'extension' in request.GET: extension=request.GET['extension'] - - - + if extension =='txt': - response = HttpResponse(mimetype='text/plain') + response = HttpResponse(content_type='text/plain') style='2008' elif extension == 'html': - response = HttpResponse(mimetype='text/html') + response = HttpResponse(content_type='text/html') style='2005' template='logbook'+style+'style.'+extension @@ -124,11 +123,11 @@ def downloadQMs(request): try: cave=Cave.objects.get(kataster_number=request.GET['cave_id']) except Cave.DoesNotExist: - cave=Cave.objects.get(name=cave_id) + cave=Cave.objects.get(name=request.GET['cave_id']) from export import toqms - response = HttpResponse(mimetype='text/csv') + response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename=qm.csv' toqms.writeQmTable(response,cave) return response @@ -136,7 +135,7 @@ def downloadQMs(request): def ajax_test(request): post_text = request.POST['post_data'] return HttpResponse("{'response_text': '"+post_text+" recieved.'}", - mimetype="application/json") + content_type="application/json") def eyecandy(request): return @@ -144,9 +143,9 @@ def eyecandy(request): def ajax_QM_number(request): if request.method=='POST': cave=Cave.objects.get(id=request.POST['cave']) - print cave + print(cave) exp=Expedition.objects.get(pk=request.POST['year']) - print exp + print(exp) res=cave.new_QM_number(exp.year) return HttpResponse(res) @@ -167,7 +166,7 @@ def logbook_entry_suggestions(request): #unwiki_QMs=re.findall(unwiki_QM_pattern,lbo.text) unwiki_QMs=[m.groupdict() for m in unwiki_QM_pattern.finditer(lbo.text)] - print unwiki_QMs + print(unwiki_QMs) for qm in unwiki_QMs: #try: if len(qm['year'])==2: @@ -180,7 +179,7 @@ def logbook_entry_suggestions(request): try: lbo=LogbookEntry.objects.get(date__year=qm['year'],title__icontains="placeholder for QMs in") except: - print "failed to get placeholder for year "+str(qm['year']) + print("failed to get placeholder for year "+str(qm['year'])) temp_QM=QM(found_by=lbo,number=qm['number'],grade=qm['grade']) temp_QM.grade=qm['grade'] @@ -188,7 +187,7 @@ def logbook_entry_suggestions(request): #except: #print 'failed' - print unwiki_QMs + print(unwiki_QMs) #wikilink_QMs=re.findall(wikilink_QM_pattern,lbo.text) @@ -199,9 +198,9 @@ def logbook_entry_suggestions(request): #for qm in wikilink_QMs: #Try to look up the QM. - print 'got 208' + print('got 208') any_suggestions=True - print 'got 210' + print('got 210') return render_with_context(request,'suggestions.html', { 'unwiki_QMs':unwiki_QMs, diff --git a/databaseReset.py b/databaseReset.py index c296274..7a5d0fa 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -109,10 +109,10 @@ def import_auto_logbooks(): for lbe in troggle.core.models.LogbookEntry.objects.all(): lbe.delete() for expedition in troggle.core.models.Expedition.objects.all(): - directory = os.path.join(settings.EXPOWEB, - "years", - expedition.year, - "autologbook") + directory = os.path.join(settings.EXPOWEB, + "years", + expedition.year, + "autologbook") for root, dirs, filenames in os.walk(directory): for filename in filenames: print(os.path.join(root, filename)) @@ -195,9 +195,9 @@ if __name__ == "__main__": elif "scans" in sys.argv: import_surveyscans() elif "caves" in sys.argv: - reload_db() - make_dirs() - pageredirects() + # reload_db() + # make_dirs() + # pageredirects() import_caves() elif "people" in sys.argv: import_people() @@ -218,14 +218,14 @@ if __name__ == "__main__": import_descriptions() parse_descriptions() elif "survex" in sys.argv: - management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex + # management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex import_survex() elif "survexpos" in sys.argv: - management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex + # management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex import parsers.survex parsers.survex.LoadPos() elif "logbooks" in sys.argv: - management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex + # management.call_command('syncdb', interactive=False) # this sets the path so that import settings works in import_survex import_logbooks() elif "autologbooks" in sys.argv: import_auto_logbooks() diff --git a/parsers/logbooks.py b/parsers/logbooks.py index ffd8e21..fbe00a3 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -115,7 +115,7 @@ def ParseDate(tripdate, year): assert False, tripdate return datetime.date(year, month, day) -# 2007, 2008, 2006 +# 2006, 2008 - 2010 def Parselogwikitxt(year, expedition, txt): trippara = re.findall(r"===(.*?)===([\s\S]*?)(?====)", txt) for triphead, triptext in trippara: @@ -140,9 +140,9 @@ def Parselogwikitxt(year, expedition, txt): #print "\n", tripcave, "--- ppp", trippeople, len(triptext) EnterLogIntoDbase(date = ldate, place = tripcave, title = tripplace, text = triptext, trippeople=trippeople, expedition=expedition, logtime_underground=0) -# 2002, 2004, 2005 +# 2002, 2004, 2005, 2007, 2011 - 2018 def Parseloghtmltxt(year, expedition, txt): - print(" - Using log html parser") + #print(" - Starting log html parser") tripparas = re.findall(r"([\s\S]*?)(?=", "", triptext) ltriptext = re.sub(r"\s*?\n\s*", " ", ltriptext) ltriptext = re.sub(r"

", "\n\n", ltriptext).strip() @@ -183,7 +182,7 @@ def Parseloghtmltxt(year, expedition, txt): print(" - No trip entrys found in logbook, check the syntax matches htmltxt format") -# main parser for pre-2001. simpler because the data has been hacked so much to fit it +# main parser for 1991 - 2001. simpler because the data has been hacked so much to fit it def Parseloghtml01(year, expedition, txt): tripparas = re.findall(r"([\s\S]*?)(?=([\s\S]*?)(?=", "", name) - mname = re.match("(\w+)(?:\s((?:van |ten )?\w+))?(?:\s\(([^)]*)\))?", name) + name = re.sub(r"<.*?>", "", name) + mname = re.match(r"(\w+)(?:\s((?:van |ten )?\w+))?(?:\s\(([^)]*)\))?", name) nickname = mname.group(3) or "" lookupAttribs={'first_name':mname.group(1), 'last_name':(mname.group(2) or "")} diff --git a/templates/frontpage.html b/templates/frontpage.html index 28206b6..eed6098 100644 --- a/templates/frontpage.html +++ b/templates/frontpage.html @@ -16,7 +16,7 @@ {% if entry.is_deletion %} {{ entry.object_repr }} {% else %} - {{ entry.object_repr }} + {{ entry.object_repr }} {% endif %}
{% if entry.content_type %}