From 240c7eff100cbcfa5e4d777fe4f823a777b7d4a4 Mon Sep 17 00:00:00 2001 From: Rad Date: Thu, 28 Feb 2019 18:07:50 +0000 Subject: [PATCH 1/2] survex parser added --- parsers/peopleM.py | 27 +++++++++++++++++++ parsers/surveysM.py | 65 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 parsers/peopleM.py create mode 100644 parsers/surveysM.py diff --git a/parsers/peopleM.py b/parsers/peopleM.py new file mode 100644 index 0000000..62c7ce0 --- /dev/null +++ b/parsers/peopleM.py @@ -0,0 +1,27 @@ +from django.conf import settings +import troggle.core.models as models + +def load(): + folkfile = open(settings.EXPOWEB+"noinfo/folk.csv") + personlines = folkfile.read().splitlines() + persontable = [x.split(',') for x in personlines] + years = [persontable[0][i] for i in range(5,len(persontable[0]))] + for year in years: + newexpedition = models.ExpeditionM( date = year ) + newexpedition.save() + for row in persontable[1:]: #skip header + attendedid = [i for i, x in enumerate(row) if '1' in x] + attendedyears = [persontable[0][i] for i in attendedid if i >= 5] + name = row[0] + print(name+' has attended: '+', '.join(attendedyears)) + newperson = models.PersonM( + name = name) + newperson.save() + for year in attendedyears: + target = models.ExpeditionM.objects.get(date=year) + newperson.expos_attended.add( target ) + print('Person -> Expo table created!') + + + + diff --git a/parsers/surveysM.py b/parsers/surveysM.py new file mode 100644 index 0000000..2b94b02 --- /dev/null +++ b/parsers/surveysM.py @@ -0,0 +1,65 @@ +from django.conf import settings +import subprocess, re +import troggle.core.models as models + +def load(): + print('Load survex files and relations') + load_area('1623') + +def load_area(areacode): + + print('Searching all cave dirs files') + basedir = settings.SURVEX_DATA+'caves-'+areacode+'/' + + cavedirs = bash("find "+basedir+" -maxdepth 1 -type d").splitlines() #this command finds all directories + print('Obtained list of directories! (#dirs='+str(len(cavedirs))+')') + + for cavedir in cavedirs: + if cavedir==basedir: + continue #skip the basedir - a non-proper subdirectory + parentname = bash('echo '+cavedir+' | rev | cut -f1 -d \'/\' | rev').splitlines()[0] #get final bit of the directory + parentcave = models.CaveM.objects.filter(survex_file__icontains=cavedir) + if len(parentcave)>1: + print('Non unique parent - skipping. Name:'+parentname) + elif len(parentcave)==0: + print('Error! parent not created:'+parentname) + continue + else: #exaclty one match + print('Adding relations of:'+parentname) + parentcave = parentcave[0] + + surveyfiles = bash('find '+cavedir+' -name \'*.svx\'').splitlines() + for fn in surveyfiles: + print(fn) + svxcontents = open(fn,'r').read().splitlines() + try: + dateline = [x for x in svxcontents if ('*date' in x)][0] + date = re.findall('\\d\\d\\d\\d\\.\\d\\d\\.\\d\\d', dateline, re.S)[0] + + + except: + if( len( [x for x in svxcontents if ('*date' in x)] ) == 0 ): + continue #skip dateless files + print('Date format error in '+fn) + print('Dateline = '+ '"'.join([x for x in svxcontents if ('*date' in x)])) + date = '1900.01.01' + + + newsurvex = models.SurveyM(survex_file=fn, date=date) + newsurvex.save() + parentcave.surveys.add(newsurvex) + parentcave.save() + + +def file_exists(filename): + test = bash('if [ ! -f '+filename+' ] ; then echo MISSING; fi')#test for file exisence + if 'MISSING' in test: #send error message to the database + return False + return True + +def bash(cmd): #calls command in bash shell, returns output + process = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) + output, error = process.communicate() + return output + + From 9a7a1728a44168e3d113edfa500893474343ec80 Mon Sep 17 00:00:00 2001 From: Rad Date: Thu, 28 Feb 2019 18:46:40 +0000 Subject: [PATCH 2/2] working maps: cave -> desc, survey -> cave, expedition -> person. Added /millnialpeople/ page. --- core/models_millenial.py | 4 ++-- core/views_caves.py | 19 ++++++++++++++++++- databaseResetM.py | 9 +++++++++ parsers/cavesM.py | 3 +++ urls.py | 1 + 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/core/models_millenial.py b/core/models_millenial.py index c97be36..1534961 100644 --- a/core/models_millenial.py +++ b/core/models_millenial.py @@ -70,8 +70,8 @@ class ExpeditionM(models.Model): #instance of this class corresponds to one expo class SurveyM(models.Model): #instance of this class corresponds to one .svx file - one trip date = models.CharField(max_length=100) #date of the trip in format YYYY.MM.DD (dated:=date given by .svx file) - maxdepth = models.FloatField() #represents max depth of a node in this survey - + survex_file = models.TextField() + class Logbook_entryM(models.Model): #instance of this class corresponds to one bit of logbook (c.f. expo.survex.com/years/2015/logbook.html or simil) date = models.CharField(max_length=100) #date as typed into logbook contents = models.TextField() #contents of the logbook chunk diff --git a/core/views_caves.py b/core/views_caves.py index 302003d..170ef17 100644 --- a/core/views_caves.py +++ b/core/views_caves.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation, CaveM, Cave_descriptionM +from troggle.core.models import CaveSlug, Cave, CaveAndEntrance, Survey, Expedition, QM, CaveDescription, EntranceSlug, Entrance, Area, SurvexStation from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm import troggle.core.models as models import troggle.settings as settings @@ -21,6 +21,13 @@ import settings from PIL import Image, ImageDraw, ImageFont import string, os, sys, subprocess +# +# NEW CONTENT +# + + +from troggle.core.models import CaveM, Cave_descriptionM, ExpeditionM + def millenialcaves(request): #RW messing around area caves = CaveM.objects.all() @@ -31,6 +38,16 @@ def millenialdescription(request, slug): desc = Cave_descriptionM.objects.get(slug=slug) return render_with_context(request,'cave_uground_description.html', {'cave': desc}) +def millenialpeople(request): + expos = ExpeditionM.objects.all() + return render_with_context(request,'peoplemillenial.html' , {'expos': expos}) + + +# +# END NEW CONTENT +# + + def getCave(cave_id): """Returns a cave object when given a cave name or number. It is used by views including cavehref, ent, and qm.""" diff --git a/databaseResetM.py b/databaseResetM.py index 4530763..b70a4e7 100644 --- a/databaseResetM.py +++ b/databaseResetM.py @@ -50,10 +50,19 @@ def load_redirects(): f = troggle.flatpages.models.Redirect(originalURL = oldURL, newURL = newURL) f.save() +def load_surveys(): + SurveyM.objects.all().delete() + import troggle.parsers.surveysM + troggle.parsers.surveysM.load() + def load_caves(): import troggle.parsers.cavesM troggle.parsers.cavesM.load() +def load_people(): + import troggle.parsers.peopleM + troggle.parsers.peopleM.load() + def load_all(): load_caves() load_surveys() diff --git a/parsers/cavesM.py b/parsers/cavesM.py index 71bda0d..f9900d6 100644 --- a/parsers/cavesM.py +++ b/parsers/cavesM.py @@ -21,6 +21,9 @@ def load(): print('Loading caves of 1623 area') loadarea('1623') + print('Loading caves of 1626 area') + loadarea('1626') + def loadarea(areacode): diff --git a/urls.py b/urls.py index a38b7a4..4197712 100644 --- a/urls.py +++ b/urls.py @@ -24,6 +24,7 @@ actualurlpatterns = patterns('', url(r'^millenialcaves/?$', views_caves.millenialcaves, name="millenialcaves"), + url(r'^millenialpeople/?$', views_caves.millenialpeople, name="millenialpeople"), url(r'^cave/descriptionM/([^/]+)/?$', views_caves.millenialdescription), #url(r'^cave/description/([^/]+)/?$', views_caves.caveDescription),