1#!/usr/bin/python3 2# 3# Send build performance test report emails 4# 5# Copyright (c) 2017, Intel Corporation. 6# 7# This program is free software; you can redistribute it and/or modify it 8# under the terms and conditions of the GNU General Public License, 9# version 2, as published by the Free Software Foundation. 10# 11# This program is distributed in the hope it will be useful, but WITHOUT 12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14# more details. 15# 16import argparse 17import base64 18import logging 19import os 20import pwd 21import re 22import shutil 23import smtplib 24import socket 25import subprocess 26import sys 27import tempfile 28from email.mime.image import MIMEImage 29from email.mime.multipart import MIMEMultipart 30from email.mime.text import MIMEText 31 32 33# Setup logging 34logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") 35log = logging.getLogger('oe-build-perf-report') 36 37 38# Find js scaper script 39SCRAPE_JS = os.path.join(os.path.dirname(__file__), '..', 'lib', 'build_perf', 40 'scrape-html-report.js') 41if not os.path.isfile(SCRAPE_JS): 42 log.error("Unableto find oe-build-perf-report-scrape.js") 43 sys.exit(1) 44 45 46class ReportError(Exception): 47 """Local errors""" 48 pass 49 50 51def check_utils(): 52 """Check that all needed utils are installed in the system""" 53 missing = [] 54 for cmd in ('phantomjs', 'optipng'): 55 if not shutil.which(cmd): 56 missing.append(cmd) 57 if missing: 58 log.error("The following tools are missing: %s", ' '.join(missing)) 59 sys.exit(1) 60 61 62def parse_args(argv): 63 """Parse command line arguments""" 64 description = """Email build perf test report""" 65 parser = argparse.ArgumentParser( 66 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 67 description=description) 68 69 parser.add_argument('--debug', '-d', action='store_true', 70 help="Verbose logging") 71 parser.add_argument('--quiet', '-q', action='store_true', 72 help="Only print errors") 73 parser.add_argument('--to', action='append', 74 help="Recipients of the email") 75 parser.add_argument('--cc', action='append', 76 help="Carbon copy recipients of the email") 77 parser.add_argument('--bcc', action='append', 78 help="Blind carbon copy recipients of the email") 79 parser.add_argument('--subject', default="Yocto build perf test report", 80 help="Email subject") 81 parser.add_argument('--outdir', '-o', 82 help="Store files in OUTDIR. Can be used to preserve " 83 "the email parts") 84 parser.add_argument('--text', 85 help="Plain text message") 86 parser.add_argument('--html', 87 help="HTML peport generated by oe-build-perf-report") 88 parser.add_argument('--phantomjs-args', action='append', 89 help="Extra command line arguments passed to PhantomJS") 90 91 args = parser.parse_args(argv) 92 93 if not args.html and not args.text: 94 parser.error("Please specify --html and/or --text") 95 96 return args 97 98 99def decode_png(infile, outfile): 100 """Parse/decode/optimize png data from a html element""" 101 with open(infile) as f: 102 raw_data = f.read() 103 104 # Grab raw base64 data 105 b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1) 106 b64_data = re.sub('">.+$', '', b64_data, 1) 107 108 # Replace file with proper decoded png 109 with open(outfile, 'wb') as f: 110 f.write(base64.b64decode(b64_data)) 111 112 subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT) 113 114 115def mangle_html_report(infile, outfile, pngs): 116 """Mangle html file into a email compatible format""" 117 paste = True 118 png_dir = os.path.dirname(outfile) 119 with open(infile) as f_in: 120 with open(outfile, 'w') as f_out: 121 for line in f_in.readlines(): 122 stripped = line.strip() 123 # Strip out scripts 124 if stripped == '<!--START-OF-SCRIPTS-->': 125 paste = False 126 elif stripped == '<!--END-OF-SCRIPTS-->': 127 paste = True 128 elif paste: 129 if re.match('^.+href="data:image/png;base64', stripped): 130 # Strip out encoded pngs (as they're huge in size) 131 continue 132 elif 'www.gstatic.com' in stripped: 133 # HACK: drop references to external static pages 134 continue 135 136 # Replace charts with <img> elements 137 match = re.match('<div id="(?P<id>\w+)"', stripped) 138 if match and match.group('id') in pngs: 139 f_out.write('<img src="cid:{}"\n'.format(match.group('id'))) 140 else: 141 f_out.write(line) 142 143 144def scrape_html_report(report, outdir, phantomjs_extra_args=None): 145 """Scrape html report into a format sendable by email""" 146 tmpdir = tempfile.mkdtemp(dir='.') 147 log.debug("Using tmpdir %s for phantomjs output", tmpdir) 148 149 if not os.path.isdir(outdir): 150 os.mkdir(outdir) 151 if os.path.splitext(report)[1] not in ('.html', '.htm'): 152 raise ReportError("Invalid file extension for report, needs to be " 153 "'.html' or '.htm'") 154 155 try: 156 log.info("Scraping HTML report with PhangomJS") 157 extra_args = phantomjs_extra_args if phantomjs_extra_args else [] 158 subprocess.check_output(['phantomjs', '--debug=true'] + extra_args + 159 [SCRAPE_JS, report, tmpdir], 160 stderr=subprocess.STDOUT) 161 162 pngs = [] 163 images = [] 164 for fname in os.listdir(tmpdir): 165 base, ext = os.path.splitext(fname) 166 if ext == '.png': 167 log.debug("Decoding %s", fname) 168 decode_png(os.path.join(tmpdir, fname), 169 os.path.join(outdir, fname)) 170 pngs.append(base) 171 images.append(fname) 172 elif ext in ('.html', '.htm'): 173 report_file = fname 174 else: 175 log.warning("Unknown file extension: '%s'", ext) 176 #shutil.move(os.path.join(tmpdir, fname), outdir) 177 178 log.debug("Mangling html report file %s", report_file) 179 mangle_html_report(os.path.join(tmpdir, report_file), 180 os.path.join(outdir, report_file), pngs) 181 return (os.path.join(outdir, report_file), 182 [os.path.join(outdir, i) for i in images]) 183 finally: 184 shutil.rmtree(tmpdir) 185 186def send_email(text_fn, html_fn, image_fns, subject, recipients, copy=[], 187 blind_copy=[]): 188 """Send email""" 189 # Generate email message 190 text_msg = html_msg = None 191 if text_fn: 192 with open(text_fn) as f: 193 text_msg = MIMEText("Yocto build performance test report.\n" + 194 f.read(), 'plain') 195 if html_fn: 196 html_msg = msg = MIMEMultipart('related') 197 with open(html_fn) as f: 198 html_msg.attach(MIMEText(f.read(), 'html')) 199 for img_fn in image_fns: 200 # Expect that content id is same as the filename 201 cid = os.path.splitext(os.path.basename(img_fn))[0] 202 with open(img_fn, 'rb') as f: 203 image_msg = MIMEImage(f.read()) 204 image_msg['Content-ID'] = '<{}>'.format(cid) 205 html_msg.attach(image_msg) 206 207 if text_msg and html_msg: 208 msg = MIMEMultipart('alternative') 209 msg.attach(text_msg) 210 msg.attach(html_msg) 211 elif text_msg: 212 msg = text_msg 213 elif html_msg: 214 msg = html_msg 215 else: 216 raise ReportError("Neither plain text nor html body specified") 217 218 pw_data = pwd.getpwuid(os.getuid()) 219 full_name = pw_data.pw_gecos.split(',')[0] 220 email = os.environ.get('EMAIL', 221 '{}@{}'.format(pw_data.pw_name, socket.getfqdn())) 222 msg['From'] = "{} <{}>".format(full_name, email) 223 msg['To'] = ', '.join(recipients) 224 if copy: 225 msg['Cc'] = ', '.join(copy) 226 if blind_copy: 227 msg['Bcc'] = ', '.join(blind_copy) 228 msg['Subject'] = subject 229 230 # Send email 231 with smtplib.SMTP('localhost') as smtp: 232 smtp.send_message(msg) 233 234 235def main(argv=None): 236 """Script entry point""" 237 args = parse_args(argv) 238 if args.quiet: 239 log.setLevel(logging.ERROR) 240 if args.debug: 241 log.setLevel(logging.DEBUG) 242 243 check_utils() 244 245 if args.outdir: 246 outdir = args.outdir 247 if not os.path.exists(outdir): 248 os.mkdir(outdir) 249 else: 250 outdir = tempfile.mkdtemp(dir='.') 251 252 try: 253 log.debug("Storing email parts in %s", outdir) 254 html_report = images = None 255 if args.html: 256 html_report, images = scrape_html_report(args.html, outdir, 257 args.phantomjs_args) 258 259 if args.to: 260 log.info("Sending email to %s", ', '.join(args.to)) 261 if args.cc: 262 log.info("Copying to %s", ', '.join(args.cc)) 263 if args.bcc: 264 log.info("Blind copying to %s", ', '.join(args.bcc)) 265 send_email(args.text, html_report, images, args.subject, 266 args.to, args.cc, args.bcc) 267 except subprocess.CalledProcessError as err: 268 log.error("%s, with output:\n%s", str(err), err.output.decode()) 269 return 1 270 except ReportError as err: 271 log.error(err) 272 return 1 273 finally: 274 if not args.outdir: 275 log.debug("Wiping %s", outdir) 276 shutil.rmtree(outdir) 277 278 return 0 279 280 281if __name__ == "__main__": 282 sys.exit(main()) 283