1#!/usr/bin/env python3
2
3# Buildtools and buildtools extended installer helper script
4#
5# Copyright (C) 2017-2020 Intel Corporation
6#
7# SPDX-License-Identifier: GPL-2.0-only
8#
9#  NOTE: --with-extended-buildtools is on by default
10#
11#  Example usage (extended buildtools from milestone):
12#    (1) using --url and --filename
13#        $ install-buildtools \
14#          --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \
15#          --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh
16#    (2) using --base-url, --release, --installer-version and --build-date
17#        $ install-buildtools \
18#          --base-url http://downloads.yoctoproject.org/releases/yocto \
19#          --release yocto-3.1_M3 \
20#          --installer-version 3.0+snapshot
21#          --build-date 202000315
22#
23#  Example usage (standard buildtools from release):
24#    (3) using --url and --filename
25#        $ install-buildtools --without-extended-buildtools \
26#          --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \
27#          --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh
28#    (4) using --base-url, --release and --installer-version
29#        $ install-buildtools --without-extended-buildtools \
30#          --base-url http://downloads.yoctoproject.org/releases/yocto \
31#          --release yocto-3.0.2 \
32#          --installer-version 3.0.2
33#
34
35import argparse
36import logging
37import os
38import platform
39import re
40import shutil
41import shlex
42import stat
43import subprocess
44import sys
45import tempfile
46from urllib.parse import quote
47
48scripts_path = os.path.dirname(os.path.realpath(__file__))
49lib_path = scripts_path + '/lib'
50sys.path = sys.path + [lib_path]
51import scriptutils
52import scriptpath
53
54
55PROGNAME = 'install-buildtools'
56logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
57
58DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools')
59DEFAULT_BASE_URL = 'http://downloads.yoctoproject.org/releases/yocto'
60DEFAULT_RELEASE = 'yocto-4.1'
61DEFAULT_INSTALLER_VERSION = '4.1'
62DEFAULT_BUILDDATE = '202110XX'
63
64# Python version sanity check
65if not (sys.version_info.major == 3 and sys.version_info.minor >= 4):
66    logger.error("This script requires Python 3.4 or greater")
67    logger.error("You have Python %s.%s" %
68	  (sys.version_info.major, sys.version_info.minor))
69    sys.exit(1)
70
71# The following three functions are copied directly from
72# bitbake/lib/bb/utils.py, in order to allow this script
73# to run on versions of python earlier than what bitbake
74# supports (e.g. less than Python 3.5 for YP 3.1 release)
75
76def _hasher(method, filename):
77    import mmap
78
79    with open(filename, "rb") as f:
80        try:
81            with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
82                for chunk in iter(lambda: mm.read(8192), b''):
83                    method.update(chunk)
84        except ValueError:
85            # You can't mmap() an empty file so silence this exception
86            pass
87    return method.hexdigest()
88
89
90def md5_file(filename):
91    """
92    Return the hex string representation of the MD5 checksum of filename.
93    """
94    import hashlib
95    return _hasher(hashlib.md5(), filename)
96
97def sha256_file(filename):
98    """
99    Return the hex string representation of the 256-bit SHA checksum of
100    filename.
101    """
102    import hashlib
103    return _hasher(hashlib.sha256(), filename)
104
105
106def main():
107    global DEFAULT_INSTALL_DIR
108    global DEFAULT_BASE_URL
109    global DEFAULT_RELEASE
110    global DEFAULT_INSTALLER_VERSION
111    global DEFAULT_BUILDDATE
112    filename = ""
113    release = ""
114    buildtools_url = ""
115    install_dir = ""
116    arch = platform.machine()
117
118    parser = argparse.ArgumentParser(
119        description="Buildtools installation helper",
120        add_help=False)
121    parser.add_argument('-u', '--url',
122                        help='URL from where to fetch buildtools SDK installer, not '
123                             'including filename (optional)\n'
124                             'Requires --filename.',
125                        action='store')
126    parser.add_argument('-f', '--filename',
127                        help='filename for the buildtools SDK installer to be installed '
128                             '(optional)\nRequires --url',
129                        action='store')
130    parser.add_argument('-d', '--directory',
131                        default=DEFAULT_INSTALL_DIR,
132                        help='directory where buildtools SDK will be installed (optional)',
133                        action='store')
134    parser.add_argument('-r', '--release',
135                        default=DEFAULT_RELEASE,
136                        help='Yocto Project release string for SDK which will be '
137                             'installed (optional)',
138                        action='store')
139    parser.add_argument('-V', '--installer-version',
140                        default=DEFAULT_INSTALLER_VERSION,
141                        help='version string for the SDK to be installed (optional)',
142                        action='store')
143    parser.add_argument('-b', '--base-url',
144                        default=DEFAULT_BASE_URL,
145                        help='base URL from which to fetch SDK (optional)', action='store')
146    parser.add_argument('-t', '--build-date',
147                        default=DEFAULT_BUILDDATE,
148                        help='Build date of pre-release SDK (optional)', action='store')
149    group = parser.add_mutually_exclusive_group()
150    group.add_argument('--with-extended-buildtools', action='store_true',
151                       dest='with_extended_buildtools',
152                       default=True,
153                       help='enable extended buildtools tarball (on by default)')
154    group.add_argument('--without-extended-buildtools', action='store_false',
155                       dest='with_extended_buildtools',
156                       help='disable extended buildtools (traditional buildtools tarball)')
157    group.add_argument('--make-only', action='store_true',
158                       help='only install make tarball')
159    group = parser.add_mutually_exclusive_group()
160    group.add_argument('-c', '--check', help='enable checksum validation',
161                        default=True, action='store_true')
162    group.add_argument('-n', '--no-check', help='disable checksum validation',
163                        dest="check", action='store_false')
164    parser.add_argument('-D', '--debug', help='enable debug output',
165                        action='store_true')
166    parser.add_argument('-q', '--quiet', help='print only errors',
167                        action='store_true')
168
169    parser.add_argument('-h', '--help', action='help',
170                        default=argparse.SUPPRESS,
171                        help='show this help message and exit')
172
173    args = parser.parse_args()
174
175    if args.make_only:
176        args.with_extended_buildtools = False
177
178    if args.debug:
179        logger.setLevel(logging.DEBUG)
180    elif args.quiet:
181        logger.setLevel(logging.ERROR)
182
183    if args.url and args.filename:
184        logger.debug("--url and --filename detected. Ignoring --base-url "
185                     "--release --installer-version  arguments.")
186        filename = args.filename
187        buildtools_url = "%s/%s" % (args.url, filename)
188    else:
189        if args.base_url:
190            base_url = args.base_url
191        else:
192            base_url = DEFAULT_BASE_URL
193        if args.release:
194            # check if this is a pre-release "milestone" SDK
195            m = re.search(r"^(?P<distro>[a-zA-Z\-]+)(?P<version>[0-9.]+)(?P<milestone>_M[1-9])$",
196                          args.release)
197            logger.debug("milestone regex: %s" % m)
198            if m and m.group('milestone'):
199                logger.debug("release[distro]: %s" % m.group('distro'))
200                logger.debug("release[version]: %s" % m.group('version'))
201                logger.debug("release[milestone]: %s" % m.group('milestone'))
202                if not args.build_date:
203                    logger.error("Milestone installers require --build-date")
204                else:
205                    if args.make_only:
206                        filename = "%s-buildtools-make-nativesdk-standalone-%s-%s.sh" % (
207                            arch, args.installer_version, args.build_date)
208                    elif args.with_extended_buildtools:
209                        filename = "%s-buildtools-extended-nativesdk-standalone-%s-%s.sh" % (
210                            arch, args.installer_version, args.build_date)
211                    else:
212                        filename = "%s-buildtools-nativesdk-standalone-%s-%s.sh" % (
213                            arch, args.installer_version, args.build_date)
214                    safe_filename = quote(filename)
215                    buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename)
216            # regular release SDK
217            else:
218                if args.make_only:
219                    filename = "%s-buildtools-make-nativesdk-standalone-%s.sh" % (arch, args.installer_version)
220                if args.with_extended_buildtools:
221                    filename = "%s-buildtools-extended-nativesdk-standalone-%s.sh" % (arch, args.installer_version)
222                else:
223                    filename = "%s-buildtools-nativesdk-standalone-%s.sh" % (arch, args.installer_version)
224                safe_filename = quote(filename)
225                buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename)
226
227    tmpsdk_dir = tempfile.mkdtemp()
228    try:
229        # Fetch installer
230        logger.info("Fetching buildtools installer")
231        tmpbuildtools = os.path.join(tmpsdk_dir, filename)
232        ret = subprocess.call("wget -q -O %s %s" %
233                              (tmpbuildtools, buildtools_url), shell=True)
234        if ret != 0:
235            logger.error("Could not download file from %s" % buildtools_url)
236            return ret
237
238        # Verify checksum
239        if args.check:
240            logger.info("Fetching buildtools installer checksum")
241            checksum_type = ""
242            for checksum_type in ["md5sum", "sha256sum"]:
243                check_url = "{}.{}".format(buildtools_url, checksum_type)
244                checksum_filename = "{}.{}".format(filename, checksum_type)
245                tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename)
246                ret = subprocess.call("wget -q -O %s %s" %
247                                      (tmpbuildtools_checksum, check_url), shell=True)
248                if ret == 0:
249                    break
250            else:
251                if ret != 0:
252                    logger.error("Could not download file from %s" % check_url)
253                    return ret
254            regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s+(?P<path>.*/)?(?P<filename>.*)$")
255            with open(tmpbuildtools_checksum, 'rb') as f:
256                original = f.read()
257                m = re.search(regex, original.decode("utf-8"))
258                logger.debug("checksum regex match: %s" % m)
259                logger.debug("checksum: %s" % m.group('checksum'))
260                logger.debug("path: %s" % m.group('path'))
261                logger.debug("filename: %s" % m.group('filename'))
262                if filename != m.group('filename'):
263                    logger.error("Filename does not match name in checksum")
264                    return 1
265                checksum = m.group('checksum')
266            if checksum_type == "md5sum":
267                checksum_value = md5_file(tmpbuildtools)
268            else:
269                checksum_value = sha256_file(tmpbuildtools)
270            if checksum == checksum_value:
271                    logger.info("Checksum success")
272            else:
273                logger.error("Checksum %s expected. Actual checksum is %s." %
274                             (checksum, checksum_value))
275                return 1
276
277        # Make installer executable
278        logger.info("Making installer executable")
279        st = os.stat(tmpbuildtools)
280        os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC)
281        logger.debug(os.stat(tmpbuildtools))
282        if args.directory:
283            install_dir = args.directory
284            ret = subprocess.call("%s -d %s -y" %
285                                  (tmpbuildtools, install_dir), shell=True)
286        else:
287            install_dir = "/opt/poky/%s" % args.installer_version
288            ret = subprocess.call("%s -y" % tmpbuildtools, shell=True)
289        if ret != 0:
290            logger.error("Could not run buildtools installer")
291            return ret
292
293        # Setup the environment
294        logger.info("Setting up the environment")
295        regex = re.compile(r'^(?P<export>export )?(?P<env_var>[A-Z_]+)=(?P<env_val>.+)$')
296        with open("%s/environment-setup-%s-pokysdk-linux" %
297                  (install_dir, arch), 'rb') as f:
298            for line in f:
299                match = regex.search(line.decode('utf-8'))
300                logger.debug("export regex: %s" % match)
301                if match:
302                    env_var = match.group('env_var')
303                    logger.debug("env_var: %s" % env_var)
304                    env_val = match.group('env_val')
305                    logger.debug("env_val: %s" % env_val)
306                    os.environ[env_var] = env_val
307
308        # Test installation
309        logger.info("Testing installation")
310        tool = ""
311        m = re.search("extended", tmpbuildtools)
312        logger.debug("extended regex: %s" % m)
313        if args.with_extended_buildtools and not m:
314            logger.info("Ignoring --with-extended-buildtools as filename "
315                        "does not contain 'extended'")
316        if args.make_only:
317            tool = 'make'
318        elif args.with_extended_buildtools and m:
319            tool = 'gcc'
320        else:
321            tool = 'tar'
322        logger.debug("install_dir: %s" % install_dir)
323        cmd = shlex.split("/usr/bin/which %s" % tool)
324        logger.debug("cmd: %s" % cmd)
325        logger.debug("tool: %s" % tool)
326        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
327        output, errors = proc.communicate()
328        logger.debug("proc.args: %s" % proc.args)
329        logger.debug("proc.communicate(): output %s" % output)
330        logger.debug("proc.communicate(): errors %s" % errors)
331        which_tool = output.decode('utf-8')
332        logger.debug("which %s: %s" % (tool, which_tool))
333        ret = proc.returncode
334        if not which_tool.startswith(install_dir):
335            logger.error("Something went wrong: %s not found in %s" %
336                         (tool, install_dir))
337        if ret != 0:
338            logger.error("Something went wrong: installation failed")
339        else:
340            logger.info("Installation successful. Remember to source the "
341                        "environment setup script now and in any new session.")
342        return ret
343
344    finally:
345        # cleanup tmp directory
346        shutil.rmtree(tmpsdk_dir)
347
348
349if __name__ == '__main__':
350    try:
351        ret = main()
352    except Exception:
353        ret = 1
354        import traceback
355
356        traceback.print_exc()
357    sys.exit(ret)
358