diff --git a/core/admin.py b/core/admin.py index 831fdc6fc..5dfa6a8a3 100644 --- a/core/admin.py +++ b/core/admin.py @@ -37,12 +37,10 @@ class TroggleModelAdmin(admin.ModelAdmin): js = ("jquery/jquery.min.js", "js/QM_helper.js") # not currently available to troggle, see media/js/README -class RoleInline(admin.TabularInline): - model = SurvexPersonRole - extra = 4 +# class RoleInline(admin.TabularInline): + # model = SurvexPersonRole + # extra = 4 -class SurvexBlockAdmin(TroggleModelAdmin): - inlines = (RoleInline,) class PersonLogEntryInline(admin.TabularInline): model = PersonLogEntry @@ -82,9 +80,6 @@ class QMAdmin(TroggleModelAdmin): list_display = ("__str__", "grade") list_display_links = ("__str__",) list_filter = ('grade', 'cave', 'expoyear') - # list_editable = ("comment", "page_ref", "grade") - # list_per_page = 20 - # raw_id_fields = ("found_by", "ticked_off_by") class PersonExpeditionAdmin(TroggleModelAdmin): search_fields = ("person__first_name", "person__slug", "expedition__year") @@ -103,10 +98,12 @@ class SurvexStationAdmin(TroggleModelAdmin): class SurvexFileAdmin(TroggleModelAdmin): search_fields = ("path",) - +# class SurvexBlockAdmin(TroggleModelAdmin): + # inlines = (RoleInline,) + class SurvexBlockAdmin(TroggleModelAdmin): - search_fields = ("name", ) - list_display = ["survexfile", "name"] + search_fields = ("name", "title") + list_display = ["survexfile", "name", "title", "scanswallet", "ref_text"] class DrawingFileAdmin(TroggleModelAdmin): search_fields = ("dwgname",) diff --git a/core/models/survex.py b/core/models/survex.py index dad897813..75d14f76e 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -203,7 +203,7 @@ class SurvexFix(models.Model): But this does not include the virtual *fix locations which are in locations.py """ - objects = SurvexBlockLookUpManager() # overwrites Survexfix.objects and enables lookup() + objects = SurvexBlockLookUpManager() # overwrites Survexfix.objects and enables lookup() but is this a mistake, Block? name = models.CharField(max_length=100) class Meta: ordering = ("name",) diff --git a/core/views/statistics.py b/core/views/statistics.py index aa13fbf20..a67ef0841 100644 --- a/core/views/statistics.py +++ b/core/views/statistics.py @@ -44,7 +44,12 @@ def legs_by_expo(expos): else: success = "" tourists = PersonExpedition.objects.filter(expedition=expedition, noncaver=True).count() - legsbyexpo.append((expedition, {"success":success, "people": expoers, "tourists": tourists, "nsurvexlegs": legsyear, "survexleglength": survexleglength, "nblocks": nblocks})) + missing_length = ( nblocks > 0 and survexleglength <= 1) + if missing_length: + print(expedition, nblocks, survexleglength, missing_length) + + legsbyexpo.append((expedition, {"success":success, "people": expoers, "tourists": tourists, + "nsurvexlegs": legsyear, "survexleglength": survexleglength, "nblocks": nblocks, "missing_length": missing_length})) legsbyexpo.reverse() return legsbyexpo, addupsurvexlegs, addupsurvexlength diff --git a/parsers/survex.py b/parsers/survex.py index e6e1d727f..c10524712 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -1,5 +1,6 @@ import copy import io +import math import os import re import subprocess @@ -70,7 +71,7 @@ dup_includes = 0 debugprint = False # Turns on debug printout for just one *include file debugprinttrigger = "!" -dataissues = [] +dataissues = set() class SurvexLeg: """No longer a models.Model subclass, so no longer a database table""" @@ -108,9 +109,12 @@ def set_walletdate(w): w.save() def stash_data_issue(parser=None, message=None, url=None, sb=None): - """Avoid hitting the database for error messages until the end of the import""" + """Avoid hitting the database for error messages until the end of the import + + use a set, we do not want identically duplicate issues + """ global dataissues - dataissues.append((parser, message, url, sb)) + dataissues.add((parser, message, url, sb)) def store_data_issues(): """Take the stash and store it permanently in the database instead @@ -129,8 +133,8 @@ def store_data_issues(): di_list.append(DataIssue(parser=parser, message=message, url=url)) # Now commit to db DataIssue.objects.bulk_create(di_list) - dataissues = [] # in database now, so empty cache - + dataissues = set() + def get_offending_filename(path): """Used to provide the URL for a line in the DataErrors page which reports problems on importing data into troggle @@ -285,7 +289,8 @@ class LoadingSurvex: rx_teamone = re.compile(r"(?i)^\s*(.*)\s*$") rx_person = re.compile(r"(?i) and |/| / |, | , |&| & | \+ |^both$|^none$") - rx_tapelng = re.compile(r"(?i).*(tape|length).*$") + rx_tapelng = re.compile(r"(?i).*(tape|length).*$") # normal tape + rx_cartlng = re.compile(r"(?i).*([-+]?\d*\.\d+)") # cartesian units and scale rx_cave = re.compile(r"(?i)caves-(\d\d\d\d)/([-\d\w]+|\d\d\d\d-?\w+-\d+)") rx_comment = re.compile(r"([^;]*?)\s*(?:;\s*(.*))?\n?$") @@ -779,17 +784,46 @@ class LoadingSurvex: def LoadSurvexUnits(self, survexblock, line): # all for 4 survex files with measurements in feet. bugger. - # Won't need this once we move to using cavern or d3dump output for lengths + # ..and all the cartesian files (ARGE) with scaling factors tapeunits = self.rx_tapelng.match(line) # tape|length - if not tapeunits: + if not tapeunits: + angle = re.match(r"(?i).*(degrees|grads|percent).*$", line) + if angle: + # we don't care + return + lruds = re.match(r"(?i).*(left|right|up|down).*$", line) + if lruds: + # we don't care + return + scale = self.rx_cartlng.match(line) + if scale: + message = f"! *UNITS SCALE '{line}' ({survexblock}) {survexblock.survexfile.path} {len(scale.groups())} {scale.groups()[0]=}" + print(self.insp + message) + stash_data_issue(parser="survexunits", message=message, url=None, sb=(survexblock.survexfile.path)) + self.unitsfactor = float(scale.groups()[0]) + else: + message = f"! *UNITS SCALE FAIL '{line}' ({survexblock}) {survexblock.survexfile.path} " + print(self.insp + message) + stash_data_issue(parser="survexunits", message=message, url=None, sb=(survexblock.survexfile.path)) + + metres = re.match(r"(?i).*(METRIC|METRES|METERS)$", line) + if metres: + self.units = "metres" + else: + message = f"! *UNITS not meters - not converted '{line}' ({survexblock}) {survexblock.survexfile.path}" + print(self.insp + message) + stash_data_issue(parser="survexunits", message=message, url=None, sb=(survexblock.survexfile.path)) + return + return + convert = re.match(r"(?i)(\w*)\s*([\.\d]+)\s*(\w*)", line) if convert: factor = convert.groups()[1] self.unitsfactor = float(factor) - if debugprint: + if True: message = ( - f"! *UNITS NUMERICAL conversion [{factor}x] '{line}' ({survexblock}) {survexblock.survexfile.path}" + f"✓ *UNITS NUMERICAL conversion [{factor}x] '{line}' ({survexblock}) {survexblock.survexfile.path}" ) print(self.insp + message) stash_data_issue(parser="survexunits", message=message) @@ -960,6 +994,8 @@ class LoadingSurvex: """This reads compass, clino and tape data but only keeps the tape lengths, the rest is discarded after error-checking. Now skipping the error checking - returns as soon as the leg is not one we count. + + Much less checking for Cartesian data. REPLACE ALL THIS by reading the .log output of cavern for the file. But we need the lengths per Block, not by File. dump3d will do lengths per block. @@ -979,17 +1015,35 @@ class LoadingSurvex: if self.datastar["type"] == "passage": return if self.datastar["type"] == "cartesian": + # message = f" ! CARTESIAN data in {survexblock.survexfile.path} {self.unitsfactor=}." + # stash_data_issue( + # parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + # ) + ls = sline.lower().split() + # CARTESIAN, so there should be 5 fields: from to dx dy dz + # we don't care what they are called, we just use the last 3 + if len(ls) != 5: + print(f" Line: {sline}\nsvxline: {svxline}") + message = f" ! Not 5 CARTESIAN fields in line '{sline.lower()}' {self.datastar=} {ls=} in\n{survexblock}\n{survexblock.survexfile}\n{survexblock.survexfile.path}" + stash_data_issue( + parser="survexleg", message=message, url=None, sb=(survexblock.survexfile.path) + ) + return + leglength = math.sqrt(float(ls[2])**2 + float(ls[3])**2 + float(ls[4])**2) + if self.unitsfactor: + leglength = leglength * self.unitsfactor + self.legsnumber += 1 + survexblock.legslength += leglength + self.slength += leglength + return + if self.datastar["type"] == "nosurvey": return if self.datastar["type"] == "diving": return if self.datastar["type"] == "cylpolar": return - if debugprint: - print( - f" !! LEG data lineno:{self.lineno}\n !! sline:'{sline}'\n !! datastar['tape']: {self.datastar['tape']}" - ) if self.datastar["type"] != "normal": return @@ -1174,7 +1228,6 @@ class LoadingSurvex: # this produces a lot of printout, so don't print it reftxt = refline.groups()[0] # only one item in this tuple if reftxt: - print(f"{self.insp} *REF quoted text: '{reftxt}' in {survexblock.survexfile.path}") # only store it if not an empty string survexblock.ref_text = reftxt survexblock.save() @@ -1265,7 +1318,7 @@ class LoadingSurvex: if args == "": # naked '*data' which is relevant only for passages. Ignore. Continue with previous settings. return - # DEFAULT | NORMAL | CARTESIAN| NOSURVEY |PASSAGE | TOPOFIL | CYLPOLAR | DIVING + # DEFAULT | NORMAL | CARTESIAN| NOSURVEY |PASSAGE | TOPOFIL | CYLPOLAR | DIVING -- all ignored ls = args.lower().split() if ls[0] == "default": self.datastar = copy.deepcopy(self.datastardefault) @@ -1307,9 +1360,8 @@ class LoadingSurvex: # print(message,file=sys.stderr) # stash_data_issue(parser='survex', message=message) self.datastar["type"] = ls[0] - elif ls[0] == "cartesian": # We should not ignore this ?! Default for Germans ? + elif ls[0] == "cartesian": # Handled in the legs calc. # message = f" ! - *data {ls[0].upper()} blocks ignored. {survexblock.name}|{args}" - # print(message) # print(message,file=sys.stderr) # stash_data_issue(parser='survex', message=message) self.datastar["type"] = ls[0] @@ -2964,7 +3016,7 @@ def LoadSurvexBlocks(): print(" - Flushing survex Data Issues ") global dataissues - dataissues = [] + dataissues = set() DataIssue.objects.filter(parser="survex").delete() DataIssue.objects.filter(parser="xSvxDate").delete() DataIssue.objects.filter(parser="survexleg").delete() diff --git a/templates/statistics.html b/templates/statistics.html index 08bf6493f..0b5fced36 100644 --- a/templates/statistics.html +++ b/templates/statistics.html @@ -27,15 +27,15 @@ in an ODS-format spr legsbyexpo: ((expedition, {"success":success, "people": expoers, "tourists": tourists, "nsurvexlegs": legsyear, "survexleglength": survexleglength, "nblocks": nblocks})) --> - + {% for legs in legsbyexpo %} - - - + + + - + {% endfor %} diff --git a/templates/yearwallets.html b/templates/yearwallets.html index efa4416e6..baa3b6af2 100644 --- a/templates/yearwallets.html +++ b/templates/yearwallets.html @@ -9,6 +9,7 @@ plans and elevations. It also contains scans of centre-line survex output on whi hand-drawn passage sections are drawn. These hand-drawn passages will eventually be traced to produce Tunnel or Therion drawings and eventually the final complete cave survey.

Link to logbooks and calendar for {{year}}. +

Link to unwalleted survex files for {{year}}. {% include 'wallet_new.html' %}

YearSurvex
Survey
Blocks		Survex
Survey Legs		Survex
length(m)		CaversNon-caversmeters/caver
YearSurvex
Survey
Blocks		Survex
Survey
Legs		Survex
length(m)		CaversNon-caversmeters/caver
{{legs.0}}{{legs.1.nblocks}}{{legs.1.nsurvexlegs|rjust:"10"|floatformat:"0g"}} {{legs.1.survexleglength|floatformat:"0g"}}{{legs.1.nblocks}}{{legs.1.nsurvexlegs|rjust:"10"|floatformat:"0g"}}{% if legs.1.missing_length %}{% else %}{% endif %}{{legs.1.survexleglength|floatformat:"0g"}} {{legs.1.people|floatformat:"0g"}}{{legs.1.tourists|floatformat:"0g"}}{{legs.1.tourists|floatformat:"0g"}} {{legs.1.success|floatformat:"1g"}}