1#!/usr/bin/env python3 2 3# Buildtools and buildtools extended installer helper script 4# 5# Copyright (C) 2017-2020 Intel Corporation 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9# NOTE: --with-extended-buildtools is on by default 10# 11# Example usage (extended buildtools from milestone): 12# (1) using --url and --filename 13# $ install-buildtools \ 14# --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \ 15# --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh 16# (2) using --base-url, --release, --installer-version and --build-date 17# $ install-buildtools \ 18# --base-url http://downloads.yoctoproject.org/releases/yocto \ 19# --release yocto-3.1_M3 \ 20# --installer-version 3.0+snapshot 21# --build-date 202000315 22# 23# Example usage (standard buildtools from release): 24# (3) using --url and --filename 25# $ install-buildtools --without-extended-buildtools \ 26# --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \ 27# --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh 28# (4) using --base-url, --release and --installer-version 29# $ install-buildtools --without-extended-buildtools \ 30# --base-url http://downloads.yoctoproject.org/releases/yocto \ 31# --release yocto-3.0.2 \ 32# --installer-version 3.0.2 33# 34 35import argparse 36import logging 37import os 38import platform 39import re 40import shutil 41import shlex 42import stat 43import subprocess 44import sys 45import tempfile 46from urllib.parse import quote 47 48scripts_path = os.path.dirname(os.path.realpath(__file__)) 49lib_path = scripts_path + '/lib' 50sys.path = sys.path + [lib_path] 51import scriptutils 52import scriptpath 53 54 55PROGNAME = 'install-buildtools' 56logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout) 57 58DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools') 59DEFAULT_BASE_URL = 'https://downloads.yoctoproject.org/releases/yocto' 60DEFAULT_RELEASE = 'yocto-5.2' 61DEFAULT_INSTALLER_VERSION = '5.2' 62DEFAULT_BUILDDATE = '202110XX' 63 64# Python version sanity check 65if not (sys.version_info.major == 3 and sys.version_info.minor >= 4): 66 logger.error("This script requires Python 3.4 or greater") 67 logger.error("You have Python %s.%s" % 68 (sys.version_info.major, sys.version_info.minor)) 69 sys.exit(1) 70 71# The following three functions are copied directly from 72# bitbake/lib/bb/utils.py, in order to allow this script 73# to run on versions of python earlier than what bitbake 74# supports (e.g. less than Python 3.5 for YP 3.1 release) 75 76def _hasher(method, filename): 77 import mmap 78 79 with open(filename, "rb") as f: 80 try: 81 with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: 82 for chunk in iter(lambda: mm.read(8192), b''): 83 method.update(chunk) 84 except ValueError: 85 # You can't mmap() an empty file so silence this exception 86 pass 87 return method.hexdigest() 88 89 90def md5_file(filename): 91 """ 92 Return the hex string representation of the MD5 checksum of filename. 93 """ 94 import hashlib 95 return _hasher(hashlib.md5(), filename) 96 97def sha256_file(filename): 98 """ 99 Return the hex string representation of the 256-bit SHA checksum of 100 filename. 101 """ 102 import hashlib 103 return _hasher(hashlib.sha256(), filename) 104 105def remove_quotes(var): 106 """ 107 If a variable starts and ends with double quotes, remove them. 108 Assumption: if a variable starts with double quotes, it must also 109 end with them. 110 """ 111 if var[0] == '"': 112 var = var[1:-1] 113 return var 114 115 116def main(): 117 global DEFAULT_INSTALL_DIR 118 global DEFAULT_BASE_URL 119 global DEFAULT_RELEASE 120 global DEFAULT_INSTALLER_VERSION 121 global DEFAULT_BUILDDATE 122 filename = "" 123 release = "" 124 buildtools_url = "" 125 install_dir = "" 126 arch = platform.machine() 127 128 parser = argparse.ArgumentParser( 129 description="Buildtools installation helper", 130 add_help=False, 131 formatter_class=argparse.RawTextHelpFormatter) 132 parser.add_argument('-u', '--url', 133 help='URL from where to fetch buildtools SDK installer, not ' 134 'including filename (optional)\n' 135 'Requires --filename.', 136 action='store') 137 parser.add_argument('-f', '--filename', 138 help='filename for the buildtools SDK installer to be installed ' 139 '(optional)\nRequires --url', 140 action='store') 141 parser.add_argument('-d', '--directory', 142 default=DEFAULT_INSTALL_DIR, 143 help='directory where buildtools SDK will be installed (optional)', 144 action='store') 145 parser.add_argument('--downloads-directory', 146 help='use this directory for tarball/checksum downloads and do not erase them (default is a temporary directory which is deleted after unpacking and installing the buildtools)', 147 action='store') 148 parser.add_argument('-r', '--release', 149 default=DEFAULT_RELEASE, 150 help='Yocto Project release string for SDK which will be ' 151 'installed (optional)', 152 action='store') 153 parser.add_argument('-V', '--installer-version', 154 default=DEFAULT_INSTALLER_VERSION, 155 help='version string for the SDK to be installed (optional)', 156 action='store') 157 parser.add_argument('-b', '--base-url', 158 default=DEFAULT_BASE_URL, 159 help='base URL from which to fetch SDK (optional)', action='store') 160 parser.add_argument('-t', '--build-date', 161 default=DEFAULT_BUILDDATE, 162 help='Build date of pre-release SDK (optional)', action='store') 163 group = parser.add_mutually_exclusive_group() 164 group.add_argument('--with-extended-buildtools', action='store_true', 165 dest='with_extended_buildtools', 166 default=True, 167 help='enable extended buildtools tarball (on by default)') 168 group.add_argument('--without-extended-buildtools', action='store_false', 169 dest='with_extended_buildtools', 170 help='disable extended buildtools (traditional buildtools tarball)') 171 group.add_argument('--make-only', action='store_true', 172 help='only install make tarball') 173 group = parser.add_mutually_exclusive_group() 174 group.add_argument('-c', '--check', help='enable checksum validation', 175 default=True, action='store_true') 176 group.add_argument('-n', '--no-check', help='disable checksum validation', 177 dest="check", action='store_false') 178 parser.add_argument('-D', '--debug', help='enable debug output', 179 action='store_true') 180 parser.add_argument('-q', '--quiet', help='print only errors', 181 action='store_true') 182 183 parser.add_argument('-h', '--help', action='help', 184 default=argparse.SUPPRESS, 185 help='show this help message and exit') 186 187 args = parser.parse_args() 188 189 if args.make_only: 190 args.with_extended_buildtools = False 191 192 if args.debug: 193 logger.setLevel(logging.DEBUG) 194 elif args.quiet: 195 logger.setLevel(logging.ERROR) 196 197 if args.url and args.filename: 198 logger.debug("--url and --filename detected. Ignoring --base-url " 199 "--release --installer-version arguments.") 200 filename = args.filename 201 buildtools_url = "%s/%s" % (args.url, filename) 202 else: 203 if args.base_url: 204 base_url = args.base_url 205 else: 206 base_url = DEFAULT_BASE_URL 207 if args.release: 208 # check if this is a pre-release "milestone" SDK 209 m = re.search(r"^(?P<distro>[a-zA-Z\-]+)(?P<version>[0-9.]+)(?P<milestone>_M[1-9])$", 210 args.release) 211 logger.debug("milestone regex: %s" % m) 212 if m and m.group('milestone'): 213 logger.debug("release[distro]: %s" % m.group('distro')) 214 logger.debug("release[version]: %s" % m.group('version')) 215 logger.debug("release[milestone]: %s" % m.group('milestone')) 216 if not args.build_date: 217 logger.error("Milestone installers require --build-date") 218 else: 219 if args.make_only: 220 filename = "%s-buildtools-make-nativesdk-standalone-%s-%s.sh" % ( 221 arch, args.installer_version, args.build_date) 222 elif args.with_extended_buildtools: 223 filename = "%s-buildtools-extended-nativesdk-standalone-%s-%s.sh" % ( 224 arch, args.installer_version, args.build_date) 225 else: 226 filename = "%s-buildtools-nativesdk-standalone-%s-%s.sh" % ( 227 arch, args.installer_version, args.build_date) 228 safe_filename = quote(filename) 229 buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename) 230 # regular release SDK 231 else: 232 if args.make_only: 233 filename = "%s-buildtools-make-nativesdk-standalone-%s.sh" % (arch, args.installer_version) 234 if args.with_extended_buildtools: 235 filename = "%s-buildtools-extended-nativesdk-standalone-%s.sh" % (arch, args.installer_version) 236 else: 237 filename = "%s-buildtools-nativesdk-standalone-%s.sh" % (arch, args.installer_version) 238 safe_filename = quote(filename) 239 buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename) 240 241 sdk_dir = args.downloads_directory or tempfile.mkdtemp() 242 os.makedirs(sdk_dir, exist_ok=True) 243 try: 244 # Fetch installer 245 logger.info("Fetching buildtools installer") 246 tmpbuildtools = os.path.join(sdk_dir, filename) 247 with open(os.path.join(sdk_dir, 'buildtools_url'), 'w') as f: 248 f.write(buildtools_url) 249 ret = subprocess.call("wget -q -O %s %s" % 250 (tmpbuildtools, buildtools_url), shell=True) 251 if ret != 0: 252 logger.error("Could not download file from %s" % buildtools_url) 253 return ret 254 255 # Verify checksum 256 if args.check: 257 logger.info("Fetching buildtools installer checksum") 258 checksum_type = "sha256sum" 259 checksum_url = "{}.{}".format(buildtools_url, checksum_type) 260 checksum_filename = "{}.{}".format(filename, checksum_type) 261 tmpbuildtools_checksum = os.path.join(sdk_dir, checksum_filename) 262 with open(os.path.join(sdk_dir, 'checksum_url'), 'w') as f: 263 f.write(checksum_url) 264 ret = subprocess.call("wget -q -O %s %s" % 265 (tmpbuildtools_checksum, checksum_url), shell=True) 266 if ret != 0: 267 logger.error("Could not download file from %s" % checksum_url) 268 return ret 269 regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s+(?P<path>.*/)?(?P<filename>.*)$") 270 with open(tmpbuildtools_checksum, 'rb') as f: 271 original = f.read() 272 m = re.search(regex, original.decode("utf-8")) 273 logger.debug("checksum regex match: %s" % m) 274 logger.debug("checksum: %s" % m.group('checksum')) 275 logger.debug("path: %s" % m.group('path')) 276 logger.debug("filename: %s" % m.group('filename')) 277 if filename != m.group('filename'): 278 logger.error("Filename does not match name in checksum") 279 return 1 280 checksum = m.group('checksum') 281 checksum_value = sha256_file(tmpbuildtools) 282 if checksum == checksum_value: 283 logger.info("Checksum success") 284 else: 285 logger.error("Checksum %s expected. Actual checksum is %s." % 286 (checksum, checksum_value)) 287 return 1 288 289 # Make installer executable 290 logger.info("Making installer executable") 291 st = os.stat(tmpbuildtools) 292 os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC) 293 logger.debug(os.stat(tmpbuildtools)) 294 if args.directory: 295 install_dir = os.path.abspath(args.directory) 296 ret = subprocess.call("%s -d %s -y" % 297 (tmpbuildtools, install_dir), shell=True) 298 else: 299 install_dir = "/opt/poky/%s" % args.installer_version 300 ret = subprocess.call("%s -y" % tmpbuildtools, shell=True) 301 if ret != 0: 302 logger.error("Could not run buildtools installer") 303 return ret 304 305 # Setup the environment 306 logger.info("Setting up the environment") 307 regex = re.compile(r'^(?P<export>export )?(?P<env_var>[A-Z_]+)=(?P<env_val>.+)$') 308 with open("%s/environment-setup-%s-pokysdk-linux" % 309 (install_dir, arch), 'rb') as f: 310 for line in f: 311 match = regex.search(line.decode('utf-8')) 312 logger.debug("export regex: %s" % match) 313 if match: 314 env_var = match.group('env_var') 315 logger.debug("env_var: %s" % env_var) 316 env_val = remove_quotes(match.group('env_val')) 317 logger.debug("env_val: %s" % env_val) 318 os.environ[env_var] = env_val 319 320 # Test installation 321 logger.info("Testing installation") 322 tool = "" 323 m = re.search("extended", tmpbuildtools) 324 logger.debug("extended regex: %s" % m) 325 if args.with_extended_buildtools and not m: 326 logger.info("Ignoring --with-extended-buildtools as filename " 327 "does not contain 'extended'") 328 if args.make_only: 329 tool = 'make' 330 elif args.with_extended_buildtools and m: 331 tool = 'gcc' 332 else: 333 tool = 'tar' 334 logger.debug("install_dir: %s" % install_dir) 335 cmd = shlex.split("/usr/bin/which %s" % tool) 336 logger.debug("cmd: %s" % cmd) 337 logger.debug("tool: %s" % tool) 338 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) 339 output, errors = proc.communicate() 340 logger.debug("proc.args: %s" % proc.args) 341 logger.debug("proc.communicate(): output %s" % output) 342 logger.debug("proc.communicate(): errors %s" % errors) 343 which_tool = output.decode('utf-8') 344 logger.debug("which %s: %s" % (tool, which_tool)) 345 ret = proc.returncode 346 if not which_tool.startswith(install_dir): 347 logger.error("Something went wrong: %s not found in %s" % 348 (tool, install_dir)) 349 if ret != 0: 350 logger.error("Something went wrong: installation failed") 351 else: 352 logger.info("Installation successful. Remember to source the " 353 "environment setup script now and in any new session.") 354 return ret 355 356 finally: 357 # cleanup tmp directory 358 if not args.downloads_directory: 359 shutil.rmtree(sdk_dir) 360 361 362if __name__ == '__main__': 363 try: 364 ret = main() 365 except Exception: 366 ret = 1 367 import traceback 368 369 traceback.print_exc() 370 sys.exit(ret) 371