import csv, re, datetime, os, shutil from html.parser import HTMLParser from unidecode import unidecode from django.conf import settings from troggle.core.models.troggle import Expedition, Person, PersonExpedition from troggle.core.utils import save_carefully '''These functions do not match how the stand-alone folk script works. So the script produces an HTML file which has href links to pages in troggle which troggle does not think are right. The standalone script needs to be renedred defucnt, and all the parsing needs to be in troggle. Either that, or they should use the same code by importing a module. ''' def parseMugShotAndBlurb(personline, header, person): """create mugshot Photo instance""" mugShotFilename=personline[header["Mugshot"]] mugShotPath = os.path.join(settings.EXPOWEB, "folk", mugShotFilename) if mugShotPath[-3:]=='jpg': #if person just has an image, add it #saveMugShot(mugShotPath=mugShotPath, mugShotFilename=mugShotFilename, person=person) pass elif mugShotPath[-3:]=='htm': #if person has an html page, find the image(s) and add it. Also, add the text from the html page to the "blurb" field in his model instance. personPageOld=open(mugShotPath,'r').read() if not person.blurb: pblurb=re.search('.*.*", "", name) firstname = "" nickname = "" rawlastname = personline[header["Lastname"]].strip() matchlastname = re.match(r"^([\w&;\s]+)(?:\(([^)]*)\))?", rawlastname) lastname = matchlastname.group(1).strip() splitnick = re.match(r"^([\w&;\s]+)(?:\(([^)]*)\))?", name) fullname = splitnick.group(1) nickname = splitnick.group(2) or "" fullname = fullname.strip() names = fullname.split(' ') firstname = names[0] if len(names) == 1: lastname = "" if personline[header["VfHO member"]] =='': vfho = False else: vfho = True lookupAttribs={'first_name':firstname, 'last_name':(lastname or "")} nonLookupAttribs={'is_vfho':vfho, 'fullname':fullname} person, created = save_carefully(Person, lookupAttribs, nonLookupAttribs) parseMugShotAndBlurb(personline=personline, header=header, person=person) # make person expedition from table for year, attended in list(zip(headers, personline))[5:]: expedition = Expedition.objects.get(year=year) if attended == "1" or attended == "-1": lookupAttribs = {'person':person, 'expedition':expedition} nonLookupAttribs = {'nickname':nickname, 'is_guest':(personline[header["Guest"]] == "1")} save_carefully(PersonExpedition, lookupAttribs, nonLookupAttribs) # used in other referencing parser functions # expedition name lookup cached for speed (it's a very big list) Gpersonexpeditionnamelookup = { } def GetPersonExpeditionNameLookup(expedition): global Gpersonexpeditionnamelookup res = Gpersonexpeditionnamelookup.get(expedition.name) if res: return res res = { } duplicates = set() #print("Calculating GetPersonExpeditionNameLookup for " + expedition.year) personexpeditions = PersonExpedition.objects.filter(expedition=expedition) htmlparser = HTMLParser() for personexpedition in personexpeditions: possnames = [ ] f = unidecode(htmlparser.unescape(personexpedition.person.first_name.lower())) l = unidecode(htmlparser.unescape(personexpedition.person.last_name.lower())) full = unidecode(htmlparser.unescape(personexpedition.person.fullname.lower())) if l: possnames.append(f + " " + l) possnames.append(f + " " + l[0]) possnames.append(f + l[0]) possnames.append(f[0] + " " + l) possnames.append(f) if full not in possnames: possnames.append(full) if personexpedition.nickname not in possnames: possnames.append(personexpedition.nickname.lower()) if l: # This allows for nickname to be used for short name eg Phil # adding Phil Sargent to the list if str(personexpedition.nickname.lower() + " " + l) not in possnames: possnames.append(personexpedition.nickname.lower() + " " + l) if str(personexpedition.nickname.lower() + " " + l[0]) not in possnames: possnames.append(personexpedition.nickname.lower() + " " + l[0]) if str(personexpedition.nickname.lower() + l[0]) not in possnames: possnames.append(personexpedition.nickname.lower() + l[0]) for possname in possnames: if possname in res: duplicates.add(possname) else: res[possname] = personexpedition for possname in duplicates: del res[possname] Gpersonexpeditionnamelookup[expedition.name] = res return res