From 3f78382d45c69b3d912288bbcc633711951b51b6 Mon Sep 17 00:00:00 2001 From: Sam Wenham Date: Wed, 26 Jun 2019 20:56:08 +0100 Subject: [PATCH] Remove this stupid hard coded name match --- core/admin.py | 7 +- core/models.py | 4 +- core/models_survex.py | 16 ++- core/views_caves.py | 223 +++++++++++++++--------------- databaseReset.py | 4 +- docker/requirements.txt.dj-1.7.11 | 1 + parsers/logbooks.py | 4 +- parsers/survex.py | 74 ++++++++-- settings.py | 2 +- templates/base.html | 195 +++++++++++++++++--------- 10 files changed, 330 insertions(+), 200 deletions(-) diff --git a/core/admin.py b/core/admin.py index 71bbd61..e77eaea 100644 --- a/core/admin.py +++ b/core/admin.py @@ -124,18 +124,19 @@ admin.site.register(CaveAndEntrance) admin.site.register(NewSubCave) admin.site.register(CaveDescription) admin.site.register(Entrance, EntranceAdmin) -admin.site.register(SurvexBlock, SurvexBlockAdmin) admin.site.register(Expedition) admin.site.register(Person,PersonAdmin) -admin.site.register(SurvexPersonRole) admin.site.register(PersonExpedition,PersonExpeditionAdmin) admin.site.register(LogbookEntry, LogbookEntryAdmin) #admin.site.register(PersonTrip) admin.site.register(QM, QMAdmin) admin.site.register(Survey, SurveyAdmin) admin.site.register(ScannedImage) -admin.site.register(SurvexStation) +admin.site.register(SurvexFile) +admin.site.register(SurvexStation) +admin.site.register(SurvexBlock) +admin.site.register(SurvexPersonRole) admin.site.register(SurvexScansFolder) admin.site.register(SurvexScanSingle) diff --git a/core/models.py b/core/models.py index f65efed..bbdaa0b 100644 --- a/core/models.py +++ b/core/models.py @@ -10,7 +10,7 @@ from django.db.models import Min, Max from django.conf import settings from decimal import Decimal, getcontext from django.core.urlresolvers import reverse -from imagekit.models import ImageModel +from imagekit.models import ProcessedImageField #ImageModel from django.template import Context, loader import settings getcontext().prec=2 #use 2 significant figures for decimal calculations @@ -248,7 +248,7 @@ class LogbookEntry(TroggleModel): ("html", "Html style logbook") ) - date = models.DateField()#MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.ld() + date = models.DateTimeField()#MJG wants to turn this into a datetime such that multiple Logbook entries on the same day can be ordered.ld() expeditionday = models.ForeignKey("ExpeditionDay", null=True)#MJG wants to KILL THIS (redundant information) expedition = models.ForeignKey(Expedition,blank=True,null=True) # yes this is double- title = models.CharField(max_length=settings.MAX_LOGBOOK_ENTRY_TITLE_LENGTH) diff --git a/core/models_survex.py b/core/models_survex.py index 3bae04c..897844e 100644 --- a/core/models_survex.py +++ b/core/models_survex.py @@ -29,6 +29,9 @@ class SurvexFile(models.Model): class Meta: ordering = ('id',) + + def __unicode__(self): + return self.path + '.svx' or 'no file' def exists(self): fname = os.path.join(settings.SURVEX_DATA, self.path + ".svx") @@ -66,6 +69,9 @@ class SurvexStation(models.Model): x = models.FloatField(blank=True, null=True) y = models.FloatField(blank=True, null=True) z = models.FloatField(blank=True, null=True) + + def __unicode__(self): + return self.block.cave.slug() + '/' + self.block.name + '/' + self.name or 'No station name' def path(self): r = self.name @@ -109,7 +115,7 @@ class SurvexBlock(models.Model): text = models.TextField() cave = models.ForeignKey('Cave', blank=True, null=True) - date = models.DateField(blank=True, null=True) + date = models.DateTimeField(blank=True, null=True) expeditionday = models.ForeignKey("ExpeditionDay", null=True) expedition = models.ForeignKey('Expedition', blank=True, null=True) @@ -177,7 +183,7 @@ ROLE_CHOICES = ( class SurvexPersonRole(models.Model): survexblock = models.ForeignKey('SurvexBlock') nrole = models.CharField(choices=ROLE_CHOICES, max_length=200, blank=True, null=True) - # increasing levels of precision + # increasing levels of precision personname = models.CharField(max_length=100) person = models.ForeignKey('Person', blank=True, null=True) personexpedition = models.ForeignKey('PersonExpedition', blank=True, null=True) @@ -194,6 +200,9 @@ class SurvexScansFolder(models.Model): class Meta: ordering = ('walletname',) + + def __unicode__(self): + return self.walletname or 'no wallet' def get_absolute_url(self): return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansfolder', kwargs={"path":re.sub("#", "%23", self.walletname)})) @@ -205,6 +214,9 @@ class SurvexScanSingle(models.Model): class Meta: ordering = ('name',) + + def __unicode__(self): + return self.survexscansfolder.walletname + '/' + self.name def get_absolute_url(self): return urlparse.urljoin(settings.URL_ROOT, reverse('surveyscansingle', kwargs={"path":re.sub("#", "%23", self.survexscansfolder.walletname), "file":self.name})) diff --git a/core/views_caves.py b/core/views_caves.py index af76b1e..bc5aa1a 100644 --- a/core/views_caves.py +++ b/core/views_caves.py @@ -50,7 +50,7 @@ def caveCmp(x, y): return numericalcmp(x.unofficial_number, y.unofficial_number) def caveindex(request): - caves = Cave.objects.all() + #caves = Cave.objects.all() notablecavehrefs = settings.NOTABLECAVESHREFS notablecaves = [Cave.objects.get(kataster_number=kataster_number) for kataster_number in notablecavehrefs ] caves1623 = list(Cave.objects.filter(area__short_name = "1623")) @@ -240,7 +240,7 @@ def entranceSlug(request, slug): def survexblock(request, survexpath): survexpath = re.sub("/", ".", survexpath) - print "jjjjjj", survexpath + print("jjjjjj", survexpath) survexblock = models.SurvexBlock.objects.get(survexpath=survexpath) #ftext = survexblock.filecontents() ftext = survexblock.text @@ -277,30 +277,30 @@ def get_qms(request, caveslug): return render(request,'options.html', {"items": [(e.entrance.slug(), e.entrance.slug()) for e in cave.entrances()]}) areanames = [ - #('', 'Location unclear'), - ('1a', '1a – Plateau: around Top Camp'), - ('1b', '1b – Western plateau near 182'), - ('1c', '1c – Eastern plateau near 204 walk-in path'), - ('1d', '1d – Further plateau around 76'), - ('2a', '2a – Southern Schwarzmooskogel near 201 path and the Nipple'), - ('2b', '2b – Eishöhle area'), - ('2b or 4 (unclear)', '2b or 4 (unclear)'), - ('2c', '2c – Kaninchenhöhle area'), - ('2d', '2d – Steinbrückenhöhle area'), - ('3', '3 – Bräuning Alm'), - ('4', '4 – Kratzer valley'), - ('5', '5 – Schwarzmoos-Wildensee'), - ('6', '6 – Far plateau'), - ('1626 or 6 (borderline)', '1626 or 6 (borderline)'), - ('7', '7 – Egglgrube'), - ('8a', '8a – Loser south face'), - ('8b', '8b – Loser below Dimmelwand'), - ('8c', '8c – Augst See'), - ('8d', '8d – Loser-Hochganger ridge'), - ('9', '9 – Gschwandt Alm'), - ('10', '10 – Altaussee'), - ('11', '11 – Augstbach') - ] + #('', 'Location unclear'), + ('1a', '1a – Plateau: around Top Camp'), + ('1b', '1b – Western plateau near 182'), + ('1c', '1c – Eastern plateau near 204 walk-in path'), + ('1d', '1d – Further plateau around 76'), + ('2a', '2a – Southern Schwarzmooskogel near 201 path and the Nipple'), + ('2b', '2b – Eishöhle area'), + ('2b or 4 (unclear)', '2b or 4 (unclear)'), + ('2c', '2c – Kaninchenhöhle area'), + ('2d', '2d – Steinbrückenhöhle area'), + ('3', '3 – Bräuning Alm'), + ('4', '4 – Kratzer valley'), + ('5', '5 – Schwarzmoos-Wildensee'), + ('6', '6 – Far plateau'), + ('1626 or 6 (borderline)', '1626 or 6 (borderline)'), + ('7', '7 – Egglgrube'), + ('8a', '8a – Loser south face'), + ('8b', '8b – Loser below Dimmelwand'), + ('8c', '8c – Augst See'), + ('8d', '8d – Loser-Hochganger ridge'), + ('9', '9 – Gschwandt Alm'), + ('10', '10 – Altaussee'), + ('11', '11 – Augstbach') + ] def prospecting(request): @@ -318,21 +318,21 @@ def prospecting(request): # big map first (zoom factor ignored) maps = { -# id left top right bottom zoom -# G&K G&K G&K G&K factor -"all": [33810.4, 85436.5, 38192.0, 81048.2, 0.35, - "All"], -"40": [36275.6, 82392.5, 36780.3, 81800.0, 3.0, - "Eishöhle"], -"76": [35440.0, 83220.0, 36090.0, 82670.0, 1.3, - "Eislufthöhle"], -"204": [36354.1, 84154.5, 37047.4, 83300, 3.0, - "Steinbrückenhöhle"], -"tc": [35230.0, 82690.0, 36110.0, 82100.0, 3.0, - "Near Top Camp"], +# id left top right bottom zoom +# G&K G&K G&K G&K factor +"all": [33810.4, 85436.5, 38192.0, 81048.2, 0.35, + "All"], +"40": [36275.6, 82392.5, 36780.3, 81800.0, 3.0, + "Eishöhle"], +"76": [35440.0, 83220.0, 36090.0, 82670.0, 1.3, + "Eislufthöhle"], +"204": [36354.1, 84154.5, 37047.4, 83300, 3.0, + "Steinbrückenhöhle"], +"tc": [35230.0, 82690.0, 36110.0, 82100.0, 3.0, + "Near Top Camp"], "grieß": - [36000.0, 86300.0, 38320.0, 84400.0, 4.0, - "Grießkogel Area"], + [36000.0, 86300.0, 38320.0, 84400.0, 4.0, + "Grießkogel Area"], } for n in maps.keys(): @@ -353,50 +353,50 @@ ZOOM = 4 DESC = 5 areacolours = { - '1a' : '#00ffff', - '1b' : '#ff00ff', - '1c' : '#ffff00', - '1d' : '#ffffff', - '2a' : '#ff0000', - '2b' : '#00ff00', - '2c' : '#008800', - '2d' : '#ff9900', - '3' : '#880000', - '4' : '#0000ff', - '6' : '#000000', # doubles for surface fixed pts, and anything else - '7' : '#808080' - } + '1a' : '#00ffff', + '1b' : '#ff00ff', + '1c' : '#ffff00', + '1d' : '#ffffff', + '2a' : '#ff0000', + '2b' : '#00ff00', + '2c' : '#008800', + '2d' : '#ff9900', + '3' : '#880000', + '4' : '#0000ff', + '6' : '#000000', # doubles for surface fixed pts, and anything else + '7' : '#808080' + } for FONT in [ - "/usr/share/fonts/truetype/freefont/FreeSans.ttf", - "/usr/X11R6/lib/X11/fonts/truetype/arial.ttf", - "C:\WINNT\Fonts\ARIAL.TTF" - ]: - if os.path.isfile(FONT): break + "/usr/share/fonts/truetype/freefont/FreeSans.ttf", + "/usr/X11R6/lib/X11/fonts/truetype/arial.ttf", + "C:\WINNT\Fonts\ARIAL.TTF" + ]: + if os.path.isfile(FONT): break TEXTSIZE = 16 CIRCLESIZE =8 LINEWIDTH = 2 myFont = ImageFont.truetype(FONT, TEXTSIZE) def mungecoord(x, y, mapcode, img): - # Top of Zinken is 73 1201 = dataset 34542 81967 - # Top of Hinter is 1073 562 = dataset 36670 83317 - # image is 1417 by 2201 - # FACTOR1 = 1000.0 / (36670.0-34542.0) - # FACTOR2 = (1201.0-562.0) / (83317 - 81967) - # FACTOR = (FACTOR1 + FACTOR2)/2 - # The factors aren't the same as the scanned map's at a slight angle. I - # can't be bothered to fix this. Since we zero on the Hinter it makes - # very little difference for caves in the areas round 76 or 204. - # xoffset = (x - 36670)*FACTOR - # yoffset = (y - 83317)*FACTOR - # return (1073 + xoffset, 562 - yoffset) + # Top of Zinken is 73 1201 = dataset 34542 81967 + # Top of Hinter is 1073 562 = dataset 36670 83317 + # image is 1417 by 2201 + # FACTOR1 = 1000.0 / (36670.0-34542.0) + # FACTOR2 = (1201.0-562.0) / (83317 - 81967) + # FACTOR = (FACTOR1 + FACTOR2)/2 + # The factors aren't the same as the scanned map's at a slight angle. I + # can't be bothered to fix this. Since we zero on the Hinter it makes + # very little difference for caves in the areas round 76 or 204. + # xoffset = (x - 36670)*FACTOR + # yoffset = (y - 83317)*FACTOR + # return (1073 + xoffset, 562 - yoffset) + + m = maps[mapcode] + factorX, factorY = img.size[0] / (m[R] - m[L]), img.size[1] / (m[T] - m[B]) + return ((x - m[L]) * factorX, (m[T] - y) * factorY) - m = maps[mapcode] - factorX, factorY = img.size[0] / (m[R] - m[L]), img.size[1] / (m[T] - m[B]) - return ((x - m[L]) * factorX, (m[T] - y) * factorY) - COL_TYPES = {True: "red", False: "#dddddd", "Reference": "#dddddd"} @@ -422,40 +422,40 @@ def prospecting_image(request, name): m = maps[name] #imgmaps = [] if name == "all": - img = mainImage + img = mainImage else: - M = maps['all'] - W, H = mainImage.size - l = int((m[L] - M[L]) / (M[R] - M[L]) * W) - t = int((m[T] - M[T]) / (M[B] - M[T]) * H) - r = int((m[R] - M[L]) / (M[R] - M[L]) * W) - b = int((m[B] - M[T]) / (M[B] - M[T]) * H) - img = mainImage.crop((l, t, r, b)) - w = int(round(m[ZOOM] * (m[R] - m[L]) / (M[R] - M[L]) * W)) - h = int(round(m[ZOOM] * (m[B] - m[T]) / (M[B] - M[T]) * H)) - img = img.resize((w, h), Image.BICUBIC) + M = maps['all'] + W, H = mainImage.size + l = int((m[L] - M[L]) / (M[R] - M[L]) * W) + t = int((m[T] - M[T]) / (M[B] - M[T]) * H) + r = int((m[R] - M[L]) / (M[R] - M[L]) * W) + b = int((m[B] - M[T]) / (M[B] - M[T]) * H) + img = mainImage.crop((l, t, r, b)) + w = int(round(m[ZOOM] * (m[R] - m[L]) / (M[R] - M[L]) * W)) + h = int(round(m[ZOOM] * (m[B] - m[T]) / (M[B] - M[T]) * H)) + img = img.resize((w, h), Image.BICUBIC) draw = ImageDraw.Draw(img) draw.setfont(myFont) if name == "all": for maparea in maps.keys(): - if maparea == "all": - continue - localm = maps[maparea] - l,t = mungecoord(localm[L], localm[T], "all", img) - r,b = mungecoord(localm[R], localm[B], "all", img) - text = maparea + " map" - textlen = draw.textsize(text)[0] + 3 - draw.rectangle([l, t, l+textlen, t+TEXTSIZE+2], fill='#ffffff') - draw.text((l+2, t+1), text, fill="#000000") - #imgmaps.append( [l, t, l+textlen, t+SIZE+2, "submap" + maparea, maparea + " subarea map"] ) - draw.line([l, t, r, t], fill='#777777', width=LINEWIDTH) - draw.line([l, b, r, b], fill='#777777', width=LINEWIDTH) - draw.line([l, t, l, b], fill='#777777', width=LINEWIDTH) - draw.line([r, t, r, b], fill='#777777', width=LINEWIDTH) - draw.line([l, t, l+textlen, t], fill='#777777', width=LINEWIDTH) - draw.line([l, t+TEXTSIZE+2, l+textlen, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH) - draw.line([l, t, l, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH) - draw.line([l+textlen, t, l+textlen, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH) + if maparea == "all": + continue + localm = maps[maparea] + l,t = mungecoord(localm[L], localm[T], "all", img) + r,b = mungecoord(localm[R], localm[B], "all", img) + text = maparea + " map" + textlen = draw.textsize(text)[0] + 3 + draw.rectangle([l, t, l+textlen, t+TEXTSIZE+2], fill='#ffffff') + draw.text((l+2, t+1), text, fill="#000000") + #imgmaps.append( [l, t, l+textlen, t+SIZE+2, "submap" + maparea, maparea + " subarea map"] ) + draw.line([l, t, r, t], fill='#777777', width=LINEWIDTH) + draw.line([l, b, r, b], fill='#777777', width=LINEWIDTH) + draw.line([l, t, l, b], fill='#777777', width=LINEWIDTH) + draw.line([r, t, r, b], fill='#777777', width=LINEWIDTH) + draw.line([l, t, l+textlen, t], fill='#777777', width=LINEWIDTH) + draw.line([l, t+TEXTSIZE+2, l+textlen, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH) + draw.line([l, t, l, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH) + draw.line([l+textlen, t, l+textlen, t+TEXTSIZE+2], fill='#777777', width=LINEWIDTH) #imgmaps[maparea] = [] # Draw scale bar m100 = int(100 / (m[R] - m[L]) * img.size[0]) @@ -477,14 +477,15 @@ def prospecting_image(request, name): plot("laser.0_5", "LSR5", "Reference", "Laser Point 0/5", name, draw, img) plot("225-96", "BAlm", "Reference", "Bräuning Alm trig point", name, draw, img) for entrance in Entrance.objects.all(): - station = entrance.best_station() - if station: - #try: - areaName = entrance.caveandentrance_set.all()[0].cave.getArea().short_name - plot(station, "%s-%s" % (areaName, str(entrance)[5:]), entrance.needs_surface_work(), str(entrance), name, draw, img) - #except: - # pass - + station = entrance.best_station() + if station: + #try: + areaName = entrance.caveandentrance_set.all()[0].cave.getArea().short_name + plot(station, "%s-%s" % (areaName, str(entrance) + [5:]), entrance.needs_surface_work(), str(entrance), name, draw, img) + #except: + # pass + for (N, E, D, num) in [(35975.37, 83018.21, 100,"177"), # Calculated from bearings (35350.00, 81630.00, 50, "71"), # From Auer map (36025.00, 82475.00, 50, "146"), # From mystery map diff --git a/databaseReset.py b/databaseReset.py index fd9b83a..4b4d7ef 100644 --- a/databaseReset.py +++ b/databaseReset.py @@ -90,9 +90,11 @@ def reset(): import_caves() import_people() import_surveyscans() - import_survex() + import_logbooks() import_QMs() + + import_survex() try: import_tunnelfiles() except: diff --git a/docker/requirements.txt.dj-1.7.11 b/docker/requirements.txt.dj-1.7.11 index ae0a7ad..7e0f0d7 100644 --- a/docker/requirements.txt.dj-1.7.11 +++ b/docker/requirements.txt.dj-1.7.11 @@ -6,3 +6,4 @@ django-imagekit Image django-tinymce==2.7.0 smartencoding +fuzzywuzzy diff --git a/parsers/logbooks.py b/parsers/logbooks.py index ee9a18d..12124ca 100644 --- a/parsers/logbooks.py +++ b/parsers/logbooks.py @@ -172,8 +172,8 @@ def Parseloghtmltxt(year, expedition, txt): tripid, tripid1, tripdate, trippeople, triptitle, triptext, tu = s.groups() ldate = ParseDate(tripdate.strip(), year) #assert tripid[:-1] == "t" + tripdate, (tripid, tripdate) - trippeople = re.sub(r"Ol(?!l)", "Olly", trippeople) - trippeople = re.sub(r"Wook(?!e)", "Wookey", trippeople) + #trippeople = re.sub(r"Ol(?!l)", "Olly", trippeople) + #trippeople = re.sub(r"Wook(?!e)", "Wookey", trippeople) triptitles = triptitle.split(" - ") if len(triptitles) >= 2: tripcave = triptitles[0] diff --git a/parsers/survex.py b/parsers/survex.py index 294de73..4d342d8 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -5,9 +5,14 @@ import troggle.settings as settings from subprocess import call, Popen, PIPE from troggle.parsers.people import GetPersonExpeditionNameLookup +from django.utils.timezone import get_current_timezone +from django.utils.timezone import make_aware + import re import os +from datetime import datetime +line_leg_regex = re.compile(r"[\d\-+.]+$") def LoadSurvexLineLeg(survexblock, stardata, sline, comment): ls = sline.lower().split() @@ -53,8 +58,8 @@ def LoadSurvexLineLeg(survexblock, stardata, sline, comment): survexleg.compass = 1000 survexleg.clino = -90.0 else: - assert re.match(r"[\d\-+.]+$", lcompass), ls - assert re.match(r"[\d\-+.]+$", lclino) and lclino != "-", ls + assert line_leg_regex.match(lcompass), ls + assert line_leg_regex.match(lclino) and lclino != "-", ls survexleg.compass = float(lcompass) survexleg.clino = float(lclino) @@ -85,6 +90,12 @@ def LoadSurvexLinePassage(survexblock, stardata, sline, comment): stardatadefault = {"type":"normal", "t":"leg", "from":0, "to":1, "tape":2, "compass":3, "clino":4} stardataparamconvert = {"length":"tape", "bearing":"compass", "gradient":"clino"} +regex_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$") +regex_ref = re.compile(r'.*?ref.*?(\d+)\s*#\s*(\d+)') +regex_star = re.compile(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$') +regex_team = re.compile(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)") +regex_team_member = re.compile(r" and | / |, | & | \+ |^both$|^none$(?i)") + def RecursiveLoad(survexblock, survexfile, fin, textlines): iblankbegins = 0 text = [ ] @@ -92,18 +103,25 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): teammembers = [ ] # uncomment to print out all files during parsing - print("Reading file: " + survexblock.survexfile.path) + print(" - Reading file: " + survexblock.survexfile.path) + stamp = datetime.now() + lineno = 0 while True: svxline = fin.readline().decode("latin1") if not svxline: + print(' - Not survex') return textlines.append(svxline) + lineno += 1 + + print(' - Line: %d' % lineno) + # break the line at the comment - sline, comment = re.match(r"([^;]*?)\s*(?:;\s*(.*))?\n?$", svxline.strip()).groups() + sline, comment = regex_comment.match(svxline.strip()).groups() # detect ref line pointing to the scans directory - mref = comment and re.match(r'.*?ref.*?(\d+)\s*#\s*(\d+)', comment) + mref = comment and regex_ref.match(comment) if mref: refscan = "%s#%s" % (mref.group(1), mref.group(2)) survexscansfolders = models.SurvexScansFolder.objects.filter(walletname=refscan) @@ -117,12 +135,15 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): continue # detect the star command - mstar = re.match(r'\s*\*[\s,]*(\w+)\s*(.*?)\s*(?:;.*)?$', sline) + mstar = regex_star.match(sline) if not mstar: if "from" in stardata: LoadSurvexLineLeg(survexblock, stardata, sline, comment) + print(' - From: ') + #print(stardata) elif stardata["type"] == "passage": LoadSurvexLinePassage(survexblock, stardata, sline, comment) + print(' - Pasage: ') #Missing "station" in stardata. continue @@ -131,6 +152,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): cmd = cmd.lower() if re.match("include$(?i)", cmd): includepath = os.path.join(os.path.split(survexfile.path)[0], re.sub(r"\.svx$", "", line)) + print(' - Include file found including - ' + includepath) includesurvexfile = models.SurvexFile(path=includepath, cave=survexfile.cave) includesurvexfile.save() includesurvexfile.SetDirectory() @@ -141,6 +163,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): elif re.match("begin$(?i)", cmd): if line: name = line.lower() + #print(' - Begin found for: ' + name) survexblockdown = models.SurvexBlock(name=name, begin_char=fin.tell(), parent=survexblock, survexpath=survexblock.survexpath+"."+name, cave=survexblock.cave, survexfile=survexfile, totalleglength=0.0) survexblockdown.save() textlinesdown = [ ] @@ -154,11 +177,16 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): else: survexblock.text = "".join(textlines) survexblock.save() + print(' - End found: ') + endstamp = datetime.now() + timetaken = endstamp - stamp + print(' - Time to process: ' + str(timetaken)) return elif re.match("date$(?i)", cmd): if len(line) == 10: - survexblock.date = re.sub(r"\.", "-", line) + #print(' - Date found: ' + line) + survexblock.date = make_aware(datetime.strptime(re.sub(r"\.", "-", line), '%Y-%m-%d'), get_current_timezone()) expeditions = models.Expedition.objects.filter(year=line[:4]) if expeditions: assert len(expeditions) == 1 @@ -167,9 +195,11 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): survexblock.save() elif re.match("team$(?i)", cmd): - mteammember = re.match(r"(Insts|Notes|Tape|Dog|Useless|Pics|Helper|Disto|Consultant)\s+(.*)$(?i)", line) + pass + print(' - Team found: ') + mteammember = regex_team.match(line) if mteammember: - for tm in re.split(r" and | / |, | & | \+ |^both$|^none$(?i)", mteammember.group(2)): + for tm in regex_team_member.split(mteammember.group(2)): if tm: personexpedition = survexblock.expedition and GetPersonExpeditionNameLookup(survexblock.expedition).get(tm.lower()) if (personexpedition, tm) not in teammembers: @@ -181,6 +211,7 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): personrole.save() elif cmd == "title": + #print(' - Title found: ') survextitle = models.SurvexTitle(survexblock=survexblock, title=line.strip('"'), cave=survexblock.cave) survextitle.save() @@ -189,8 +220,11 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): pass elif cmd == "data": + #print(' - Data found: ') ls = line.lower().split() stardata = { "type":ls[0] } + #print(' - Star data: ', stardata) + #print(ls) for i in range(0, len(ls)): stardata[stardataparamconvert.get(ls[i], ls[i])] = i - 1 if ls[0] in ["normal", "cartesian", "nosurvey"]: @@ -201,26 +235,32 @@ def RecursiveLoad(survexblock, survexfile, fin, textlines): assert ls[0] == "passage", line elif cmd == "equate": + #print(' - Equate found: ') LoadSurvexEquate(survexblock, line) elif cmd == "fix": + #print(' - Fix found: ') survexblock.MakeSurvexStation(line.split()[0]) else: + #print(' - Stuff') if cmd not in ["sd", "include", "units", "entrance", "data", "flags", "title", "export", "instrument", "calibrate", "set", "infer", "alias", "ref", "cs", "declination", "case"]: print("Unrecognised command in line:", cmd, line, survexblock, survexblock.survexfile.path) + endstamp = datetime.now() + timetaken = endstamp - stamp + print(' - Time to process: ' + str(timetaken)) def ReloadSurvexCave(survex_cave, area): - print(survex_cave, area) + print(' - Area: ' + str(area) + ' Cave: ' + str(survex_cave)) cave = models.Cave.objects.get(kataster_number=survex_cave, area__short_name=area) - print(cave) - #cave = models.Cave.objects.get(kataster_number=survex_cave) + print(' - ' + str(cave)) cave.survexblock_set.all().delete() cave.survexfile_set.all().delete() cave.survexdirectory_set.all().delete() + file_stamp_start = datetime.now() survexfile = models.SurvexFile(path="caves-" + cave.kat_area() + "/" + survex_cave + "/" + survex_cave, cave=cave) survexfile.save() survexfile.SetDirectory() @@ -228,6 +268,9 @@ def ReloadSurvexCave(survex_cave, area): survexblockroot = models.SurvexBlock(name="root", survexpath="caves-" + cave.kat_area(), begin_char=0, cave=cave, survexfile=survexfile, totalleglength=0.0) survexblockroot.save() fin = survexfile.OpenFile() + file_stamp_end = datetime.now() + file_time = file_stamp_end - file_stamp_start + print(' - Files time to process: ' + str(file_time)) textlines = [ ] RecursiveLoad(survexblockroot, survexfile, fin, textlines) survexblockroot.text = "".join(textlines) @@ -268,10 +311,15 @@ def LoadAllSurvexBlocks(): print(" - Reloading all caves") caves = models.Cave.objects.all() for cave in caves: + rec_stamp_start = datetime.now() if cave.kataster_number and os.path.isdir(os.path.join(settings.SURVEX_DATA, "caves-" + cave.kat_area(), cave.kataster_number)): if cave.kataster_number not in ['40']: - print("loading", cave, cave.kat_area()) + print(" - Loading " + str(cave) + " " + cave.kat_area()) ReloadSurvexCave(cave.kataster_number, cave.kat_area()) + rec_stamp_end = datetime.now() + timetaken = rec_stamp_end - rec_stamp_start + print(' - Time to process: ' + str(timetaken)) + print('--------') poslineregex = re.compile(r"^\(\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*),\s*([+-]?\d*\.\d*)\s*\)\s*([^\s]+)$") diff --git a/settings.py b/settings.py index 68b5ff1..376901a 100644 --- a/settings.py +++ b/settings.py @@ -129,7 +129,7 @@ INSTALLED_APPS = ( 'troggle.profiles', 'troggle.core', 'troggle.flatpages', - 'troggle.imagekit', + 'imagekit', ) MIDDLEWARE_CLASSES = ( diff --git a/templates/base.html b/templates/base.html index 666ff7a..9dbef01 100644 --- a/templates/base.html +++ b/templates/base.html @@ -3,8 +3,14 @@ - + + + + + + + {% block title %}Troggle{% endblock %} @@ -15,47 +21,108 @@ + + -
- -
+
+ +