#!/usr/bin/env python import sys, os, operator, urllib, json, re, time # 2017 originally by Martin Green # 2018-08-27 edited Philip Sargent # 2019-03-02 extended to take command line argument of loser_dir and set mod time of index.html to be sane as json file # 2019-12-17 extra output of links to troggle-generated trip data loser_dir = "/home/expo/loser/" #loser_dir = "/mnt/d/CUCC-Expo/loser/" # when running on Win10/bash #loser_dir = "/media/philip/SD-huge/CUCC-Expo/loser/" # when running on xubuntu laptop 'barbie' if len(sys.argv) > 1 : if sys.argv[1] != "": loser_dir = sys.argv[1] print "Loser repo is assumed to be in: " + loser_dir html_base = "%(body)s" html_year_index = html_base % {"body": "

%(year)s surveys: wallets status

\n

List of trips: expedition/%(year)s - troggle-processed .svx files and logbook entries on server

\n

Persons

\n\n

Wallets

\n%(wallets)s
\n

Needing Scanning

\n\n

Website (Guidebook description) needing updating\n

\n\n"} html_year_person = "
  • %(person)s
    • \n" html_year_wallet_entry = "%(walletname)s %(cave)s %(name)s %(complaints)s\n" html_person_wallet_entry = "
  • %(walletname)s
    • \n" html_year_scanning_entry = "
  • %(walletname)s %(cave)s %(name)s
    • \n" html_wallet_file_entry = "
  • %(filename)s
    • \n" html_wallet_index = html_base % {"body": "

    %(title)s : %(cave)s : %(name)s

    \n

    Date: %(date)s

    People: %(people)s

    \n

    Cave Guidebook description - %(description_needed)s \n

    Survex file:
      Local (Windows): %(loser_dirw)s%(survex)s
      Local (Linux): %(loser_dir)s%(survex)s
      Server:            %(survex)s

    Wallet index for this year

    \n

    Issues

    \n%(complaints)s\n

    Files

    \n\n"} html_survex_required = {True: "Survex ", False: ""} html_plan_scanned = {True: "", False: "Plan "} html_elev_scanned = {True: "", False: "Elev "} html_description_written = {True: "", False: "Desc "} html_qms_written = {True: "", False: "QMs "} html_status = {True: "Issues: ", False: ""} html_person = html_base % {"body": "

    %(person)s

    Outstanding Wallets

    "} html_complaint_items = "
  • %(count)i %(complaint)s
    • " html_items = "
  • %s
    • " blank_json = { "cave": "", "date": "", "description url": "", "description written": False, "electronic survey": False, "elev drawn": False, "elev not required": False, "name": "", "people": [ "Unknown" ], "plan drawn": False, "plan not required": False, "qms written": False, "survex file": "", "survex not required": False, "website updated": False} #need to use wallets as a dict/tuple (id,cave,name) - not sure how. wallets = [] wallets_needing_scanning = set() website_needing_updating = set() people = {} #use dir this file is in to get current year path,year = os.path.split(os.path.dirname(os.path.realpath(__file__))) print "Year: " + year for item in sorted(os.listdir(".")): if os.path.isdir(item): files = [] for f in os.listdir(os.path.join(".", item)): if f not in ["contents.json", "contents.json~","index.html"] and os.path.isfile(os.path.join(".", item, f)): files.append(f) contents_path = os.path.join(".", item, "contents.json") # print "Trying to read file %s" % (contents_path) if not os.path.isfile(contents_path): print "Creating file %s from template" % (contents_path) json_file = open(contents_path, "w") json.dump(blank_json, json_file, sort_keys=True, indent = 1) json_file.close() # print "Reading file %s" % (contents_path) json_file = open(contents_path) #print json_file data = json.load(json_file) if not data["people"]: data["people"]=["NOBODY"] json_file.close() write_required = False try: wallet, cave, name = re.match("(\d\d\d\d#\d\d)-(.*) (.*)", item).groups() except: wallet, cave, name = "", "", "" #print data for k, v in blank_json.items(): if not data.has_key(k): if k == "cave": data[k] = cave elif k == "name": data[k] = name else: data[k] = v write_required = True #print write_required if write_required: print "Writing file %s" % (contents_path) json_file = open(contents_path, "w") json.dump(data, json_file, indent = 1) json_file.close() # Get modification time of contents.json # print("json last modified: %s" % time.ctime(os.path.getmtime(contents_path))) json_mtime = os.path.getmtime(contents_path) #make wallet descriptions #Survex survex_required = (data["survex not required"] and data["survex file"] == "") or \ not (not data["survex not required"] and os.path.isfile(os.path.join(loser_dir, data["survex file"]))) survex_complaint = "" if data["survex not required"] and data["survex file"] != "": survex_complaint = "Survex is not required and yet there is a survex file!" if not data["survex not required"] and data["survex file"] == "": survex_complaint = "A survex file is required, but has not been specified!" if not data["survex not required"] and not os.path.isfile(os.path.join(loser_dir, data["survex file"])): survex_complaint = "The specified survex file (%s) does not exist here!" % os.path.join(loser_dir, data["survex file"]) complaints = [] person_complaints = [] if survex_required: complaints.append(survex_complaint) person_complaints.append(survex_complaint) #Notes notes_scanned = reduce(operator.or_, [f.startswith("note") for f in files], False) if not notes_scanned: complaints.append("The notes needs scanning (no noteN.jpg file found)") wallets_needing_scanning.add(item) #Plan drawing required plan_scanned = reduce(operator.or_, [f.startswith("plan") for f in files], False) plan_drawing_required = not (plan_scanned or data["plan drawn"]) if plan_drawing_required: complaints.append("The plan needs drawing (no planN.jpg file found)") person_complaints.append(" plan(s) needs drawing (no planN.jpg file found)") if not plan_drawing_required and not plan_scanned: complaints.append("The plan needs scanning (no planN.jpg file found)") wallets_needing_scanning.add(item) #Elev drawing required elev_scanned = reduce(operator.or_, [f.startswith("elev") for f in files], False) elev_drawing_required = not (elev_scanned or data["elev drawn"]) if elev_drawing_required: complaints.append("The elev needs drawing (no elevN.jpg file found)") person_complaints.append(" elev(s) needs drawing (no elevN.jpg file found)") if not elev_drawing_required and not elev_scanned: complaints.append("The elev needs scanning (no elevN.jpg file found)") wallets_needing_scanning.add(item) #Description if not data["description written"]: complaints.append("The description needs writing") person_complaints.append(" description(s) needs writing") description_needed = "Not present, needs doing." #QMS if not data["qms written"]: complaints.append("The QMs needs writing") person_complaints.append(" set(s) of QMs needs writing") #Website if not data["website updated"]: complaints.append("The guidebook description on the website needs updating") website_needing_updating.add(item) #Electronic Surveys if not data["electronic survey"]: complaints.append("Tunnel / Therion files need drawing") if data["survex file"]: survex_description = data["survex file"] else: survex_description = "Not specified" wallet_index_file = open(os.path.join(item, "index.html"), "w") wallet_index_file.write(html_wallet_index % {"title": item, "cave": data["cave"], "name": data["name"], "date": data["date"], "people": reduce(operator.add, [" %s," % person for person in data["people"]], ""), "description": "http://expo.survex.com"+data["description url"], "description_needed": description_needed, "loser_dir": loser_dir, "loser_dirw": loser_dir[5].upper() + ':/' + loser_dir[7:], "survex": survex_description, "complaints": reduce(operator.add, ["

    " + complaint + "

    " for complaint in complaints], ""), "files": reduce(operator.add, [html_wallet_file_entry % {"fileurl": urllib.quote(f), "filename": f} for f in files], "")}) wallet_index_file.close() wallets.append((item, data["cave"], data["name"], survex_required, plan_scanned, elev_scanned, data["description written"], data["qms written"])) # Set modification time to be the same as that of contents.json index_file = item+"/index.html" os.utime(index_file, ( json_mtime,json_mtime)) #People for person in data["people"]: # delete all person.html as we are recreating all the ones that matter and old ones have old data if os.path.isfile(person + ".html"): os.remove(person + ".html") if person_complaints: for person in data["people"]: if not people.has_key(person): people[person] = [] people[person].append((item, person_complaints)) wallets.sort() website_needing_updating = list(website_needing_updating) website_needing_updating.sort() wallets_needing_scanning = list(wallets_needing_scanning) wallets_needing_scanning.sort() person_summary = [] for person, person_wallets in people.items(): complaints = reduce(operator.add, [complaints for wallet, complaints in person_wallets], []) complaints_summary = [] for complaint in set(complaints): complaints_summary.append((complaint, complaints.count(complaint))) person_summary.append((person, complaints_summary)) person_summary = dict(person_summary) year_index_file = open("index.html", "w") year_index_file.write(html_year_index % {"year": year, "persons": reduce(operator.add, [html_year_person % {"person": person, "complaints": reduce(operator.add, [html_complaint_items % {"complaint": complaint, "count": count} for complaint, count in complaints], "")} for person, complaints in person_summary.items()], ""), "needing scanning": reduce(operator.add, [html_year_scanning_entry % {"walletname": wallet, "cave": cave, "name": name, "walletindex": urllib.quote(wallet) + "/index.html"} for (wallet) in wallets_needing_scanning], ""), "website needing updating": reduce(operator.add, [html_year_scanning_entry % {"walletname": wallet, "cave": cave, "name": name, "walletindex": urllib.quote(wallet) + "/index.html"} for (wallet) in website_needing_updating], ""), "wallets": reduce(operator.add, [html_year_wallet_entry % {"walletname": wallet, "cave": cave, "name": name, "walletindex": urllib.quote(wallet) + "/index.html", "complaints": html_status[survex_required or not plan_scanned or not elev_scanned or description_written] + html_survex_required[survex_required] + html_plan_scanned[plan_scanned] + html_elev_scanned[elev_scanned] + html_description_written[description_written] + html_qms_written[qms_written] } for (wallet, cave, name, survex_required, plan_scanned, elev_scanned, description_written, qms_written) in wallets])}) year_index_file.close() for person, item_complaint_list in people.items(): person_file = open(person + ".html", "w") person_file.write(html_person % {"person": person, "wallets": reduce(operator.add, [html_person_wallet_entry % {"walletname": wallet, "walletindex": urllib.quote(wallet) + "/index.html", "complaints": reduce(operator.add, [html_items % complaint for complaint in complaints], "") } for wallet, complaints in item_complaint_list], "") }) person_file.close()