1#!/usr/bin/python3
2#
3# Send build performance test report emails
4#
5# Copyright (c) 2017, Intel Corporation.
6#
7# This program is free software; you can redistribute it and/or modify it
8# under the terms and conditions of the GNU General Public License,
9# version 2, as published by the Free Software Foundation.
10#
11# This program is distributed in the hope it will be useful, but WITHOUT
12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
14# more details.
15#
16import argparse
17import base64
18import logging
19import os
20import pwd
21import re
22import shutil
23import smtplib
24import socket
25import subprocess
26import sys
27import tempfile
28from email.mime.image import MIMEImage
29from email.mime.multipart import MIMEMultipart
30from email.mime.text import MIMEText
31
32
33# Setup logging
34logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
35log = logging.getLogger('oe-build-perf-report')
36
37
38# Find js scaper script
39SCRAPE_JS = os.path.join(os.path.dirname(__file__), '..', 'lib', 'build_perf',
40                         'scrape-html-report.js')
41if not os.path.isfile(SCRAPE_JS):
42    log.error("Unableto find oe-build-perf-report-scrape.js")
43    sys.exit(1)
44
45
46class ReportError(Exception):
47    """Local errors"""
48    pass
49
50
51def check_utils():
52    """Check that all needed utils are installed in the system"""
53    missing = []
54    for cmd in ('phantomjs', 'optipng'):
55        if not shutil.which(cmd):
56            missing.append(cmd)
57    if missing:
58        log.error("The following tools are missing: %s", ' '.join(missing))
59        sys.exit(1)
60
61
62def parse_args(argv):
63    """Parse command line arguments"""
64    description = """Email build perf test report"""
65    parser = argparse.ArgumentParser(
66        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
67        description=description)
68
69    parser.add_argument('--debug', '-d', action='store_true',
70                        help="Verbose logging")
71    parser.add_argument('--quiet', '-q', action='store_true',
72                        help="Only print errors")
73    parser.add_argument('--to', action='append',
74                        help="Recipients of the email")
75    parser.add_argument('--cc', action='append',
76                        help="Carbon copy recipients of the email")
77    parser.add_argument('--bcc', action='append',
78                        help="Blind carbon copy recipients of the email")
79    parser.add_argument('--subject', default="Yocto build perf test report",
80                        help="Email subject")
81    parser.add_argument('--outdir', '-o',
82                        help="Store files in OUTDIR. Can be used to preserve "
83                             "the email parts")
84    parser.add_argument('--text',
85                        help="Plain text message")
86    parser.add_argument('--html',
87                        help="HTML peport generated by oe-build-perf-report")
88    parser.add_argument('--phantomjs-args', action='append',
89                        help="Extra command line arguments passed to PhantomJS")
90
91    args = parser.parse_args(argv)
92
93    if not args.html and not args.text:
94        parser.error("Please specify --html and/or --text")
95
96    return args
97
98
99def decode_png(infile, outfile):
100    """Parse/decode/optimize png data from a html element"""
101    with open(infile) as f:
102        raw_data = f.read()
103
104    # Grab raw base64 data
105    b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1)
106    b64_data = re.sub('">.+$', '', b64_data, 1)
107
108    # Replace file with proper decoded png
109    with open(outfile, 'wb') as f:
110        f.write(base64.b64decode(b64_data))
111
112    subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT)
113
114
115def mangle_html_report(infile, outfile, pngs):
116    """Mangle html file into a email compatible format"""
117    paste = True
118    png_dir = os.path.dirname(outfile)
119    with open(infile) as f_in:
120        with open(outfile, 'w') as f_out:
121            for line in f_in.readlines():
122                stripped = line.strip()
123                # Strip out scripts
124                if stripped == '<!--START-OF-SCRIPTS-->':
125                    paste = False
126                elif stripped == '<!--END-OF-SCRIPTS-->':
127                    paste = True
128                elif paste:
129                    if re.match('^.+href="data:image/png;base64', stripped):
130                        # Strip out encoded pngs (as they're huge in size)
131                        continue
132                    elif 'www.gstatic.com' in stripped:
133                        # HACK: drop references to external static pages
134                        continue
135
136                    # Replace charts with <img> elements
137                    match = re.match('<div id="(?P<id>\w+)"', stripped)
138                    if match and match.group('id') in pngs:
139                        f_out.write('<img src="cid:{}"\n'.format(match.group('id')))
140                    else:
141                        f_out.write(line)
142
143
144def scrape_html_report(report, outdir, phantomjs_extra_args=None):
145    """Scrape html report into a format sendable by email"""
146    tmpdir = tempfile.mkdtemp(dir='.')
147    log.debug("Using tmpdir %s for phantomjs output", tmpdir)
148
149    if not os.path.isdir(outdir):
150        os.mkdir(outdir)
151    if os.path.splitext(report)[1] not in ('.html', '.htm'):
152        raise ReportError("Invalid file extension for report, needs to be "
153                          "'.html' or '.htm'")
154
155    try:
156        log.info("Scraping HTML report with PhangomJS")
157        extra_args = phantomjs_extra_args if phantomjs_extra_args else []
158        subprocess.check_output(['phantomjs', '--debug=true'] + extra_args +
159                                [SCRAPE_JS, report, tmpdir],
160                                stderr=subprocess.STDOUT)
161
162        pngs = []
163        images = []
164        for fname in os.listdir(tmpdir):
165            base, ext = os.path.splitext(fname)
166            if ext == '.png':
167                log.debug("Decoding %s", fname)
168                decode_png(os.path.join(tmpdir, fname),
169                           os.path.join(outdir, fname))
170                pngs.append(base)
171                images.append(fname)
172            elif ext in ('.html', '.htm'):
173                report_file = fname
174            else:
175                log.warning("Unknown file extension: '%s'", ext)
176                #shutil.move(os.path.join(tmpdir, fname), outdir)
177
178        log.debug("Mangling html report file %s", report_file)
179        mangle_html_report(os.path.join(tmpdir, report_file),
180                           os.path.join(outdir, report_file), pngs)
181        return (os.path.join(outdir, report_file),
182                [os.path.join(outdir, i) for i in images])
183    finally:
184        shutil.rmtree(tmpdir)
185
186def send_email(text_fn, html_fn, image_fns, subject, recipients, copy=[],
187               blind_copy=[]):
188    """Send email"""
189    # Generate email message
190    text_msg = html_msg = None
191    if text_fn:
192        with open(text_fn) as f:
193            text_msg = MIMEText("Yocto build performance test report.\n" +
194                                f.read(), 'plain')
195    if html_fn:
196        html_msg = msg = MIMEMultipart('related')
197        with open(html_fn) as f:
198            html_msg.attach(MIMEText(f.read(), 'html'))
199        for img_fn in image_fns:
200            # Expect that content id is same as the filename
201            cid = os.path.splitext(os.path.basename(img_fn))[0]
202            with open(img_fn, 'rb') as f:
203                image_msg = MIMEImage(f.read())
204            image_msg['Content-ID'] = '<{}>'.format(cid)
205            html_msg.attach(image_msg)
206
207    if text_msg and html_msg:
208        msg = MIMEMultipart('alternative')
209        msg.attach(text_msg)
210        msg.attach(html_msg)
211    elif text_msg:
212        msg = text_msg
213    elif html_msg:
214        msg = html_msg
215    else:
216        raise ReportError("Neither plain text nor html body specified")
217
218    pw_data = pwd.getpwuid(os.getuid())
219    full_name = pw_data.pw_gecos.split(',')[0]
220    email = os.environ.get('EMAIL',
221                           '{}@{}'.format(pw_data.pw_name, socket.getfqdn()))
222    msg['From'] = "{} <{}>".format(full_name, email)
223    msg['To'] = ', '.join(recipients)
224    if copy:
225        msg['Cc'] = ', '.join(copy)
226    if blind_copy:
227        msg['Bcc'] = ', '.join(blind_copy)
228    msg['Subject'] = subject
229
230    # Send email
231    with smtplib.SMTP('localhost') as smtp:
232        smtp.send_message(msg)
233
234
235def main(argv=None):
236    """Script entry point"""
237    args = parse_args(argv)
238    if args.quiet:
239        log.setLevel(logging.ERROR)
240    if args.debug:
241        log.setLevel(logging.DEBUG)
242
243    check_utils()
244
245    if args.outdir:
246        outdir = args.outdir
247        if not os.path.exists(outdir):
248            os.mkdir(outdir)
249    else:
250        outdir = tempfile.mkdtemp(dir='.')
251
252    try:
253        log.debug("Storing email parts in %s", outdir)
254        html_report = images = None
255        if args.html:
256            html_report, images = scrape_html_report(args.html, outdir,
257                                                     args.phantomjs_args)
258
259        if args.to:
260            log.info("Sending email to %s", ', '.join(args.to))
261            if args.cc:
262                log.info("Copying to %s", ', '.join(args.cc))
263            if args.bcc:
264                log.info("Blind copying to %s", ', '.join(args.bcc))
265            send_email(args.text, html_report, images, args.subject,
266                       args.to, args.cc, args.bcc)
267    except subprocess.CalledProcessError as err:
268        log.error("%s, with output:\n%s", str(err), err.output.decode())
269        return 1
270    except ReportError as err:
271        log.error(err)
272        return 1
273    finally:
274        if not args.outdir:
275            log.debug("Wiping %s", outdir)
276            shutil.rmtree(outdir)
277
278    return 0
279
280
281if __name__ == "__main__":
282    sys.exit(main())
283