1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2012 The Chromium OS Authors. 3# 4 5import re 6import glob 7from HTMLParser import HTMLParser 8import os 9import sys 10import tempfile 11import urllib2 12 13import bsettings 14import command 15import terminal 16 17(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, 18 PRIORITY_CALC) = range(4) 19 20# Simple class to collect links from a page 21class MyHTMLParser(HTMLParser): 22 def __init__(self, arch): 23 """Create a new parser 24 25 After the parser runs, self.links will be set to a list of the links 26 to .xz archives found in the page, and self.arch_link will be set to 27 the one for the given architecture (or None if not found). 28 29 Args: 30 arch: Architecture to search for 31 """ 32 HTMLParser.__init__(self) 33 self.arch_link = None 34 self.links = [] 35 self.re_arch = re.compile('[-_]%s-' % arch) 36 37 def handle_starttag(self, tag, attrs): 38 if tag == 'a': 39 for tag, value in attrs: 40 if tag == 'href': 41 if value and value.endswith('.xz'): 42 self.links.append(value) 43 if self.re_arch.search(value): 44 self.arch_link = value 45 46 47class Toolchain: 48 """A single toolchain 49 50 Public members: 51 gcc: Full path to C compiler 52 path: Directory path containing C compiler 53 cross: Cross compile string, e.g. 'arm-linux-' 54 arch: Architecture of toolchain as determined from the first 55 component of the filename. E.g. arm-linux-gcc becomes arm 56 priority: Toolchain priority (0=highest, 20=lowest) 57 override_toolchain: Toolchain to use for sandbox, overriding the normal 58 one 59 """ 60 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, 61 arch=None, override_toolchain=None): 62 """Create a new toolchain object. 63 64 Args: 65 fname: Filename of the gcc component 66 test: True to run the toolchain to test it 67 verbose: True to print out the information 68 priority: Priority to use for this toolchain, or PRIORITY_CALC to 69 calculate it 70 """ 71 self.gcc = fname 72 self.path = os.path.dirname(fname) 73 self.override_toolchain = override_toolchain 74 75 # Find the CROSS_COMPILE prefix to use for U-Boot. For example, 76 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. 77 basename = os.path.basename(fname) 78 pos = basename.rfind('-') 79 self.cross = basename[:pos + 1] if pos != -1 else '' 80 81 # The architecture is the first part of the name 82 pos = self.cross.find('-') 83 if arch: 84 self.arch = arch 85 else: 86 self.arch = self.cross[:pos] if pos != -1 else 'sandbox' 87 if self.arch == 'sandbox' and override_toolchain: 88 self.gcc = override_toolchain 89 90 env = self.MakeEnvironment(False) 91 92 # As a basic sanity check, run the C compiler with --version 93 cmd = [fname, '--version'] 94 if priority == PRIORITY_CALC: 95 self.priority = self.GetPriority(fname) 96 else: 97 self.priority = priority 98 if test: 99 result = command.RunPipe([cmd], capture=True, env=env, 100 raise_on_error=False) 101 self.ok = result.return_code == 0 102 if verbose: 103 print 'Tool chain test: ', 104 if self.ok: 105 print "OK, arch='%s', priority %d" % (self.arch, 106 self.priority) 107 else: 108 print 'BAD' 109 print 'Command: ', cmd 110 print result.stdout 111 print result.stderr 112 else: 113 self.ok = True 114 115 def GetPriority(self, fname): 116 """Return the priority of the toolchain. 117 118 Toolchains are ranked according to their suitability by their 119 filename prefix. 120 121 Args: 122 fname: Filename of toolchain 123 Returns: 124 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. 125 """ 126 priority_list = ['-elf', '-unknown-linux-gnu', '-linux', 127 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux', 128 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi', 129 '-linux-gnueabihf', '-le-linux', '-uclinux'] 130 for prio in range(len(priority_list)): 131 if priority_list[prio] in fname: 132 return PRIORITY_CALC + prio 133 return PRIORITY_CALC + prio 134 135 def GetWrapper(self, show_warning=True): 136 """Get toolchain wrapper from the setting file. 137 """ 138 value = '' 139 for name, value in bsettings.GetItems('toolchain-wrapper'): 140 if not value: 141 print "Warning: Wrapper not found" 142 if value: 143 value = value + ' ' 144 145 return value 146 147 def MakeEnvironment(self, full_path): 148 """Returns an environment for using the toolchain. 149 150 Thie takes the current environment and adds CROSS_COMPILE so that 151 the tool chain will operate correctly. This also disables localized 152 output and possibly unicode encoded output of all build tools by 153 adding LC_ALL=C. 154 155 Args: 156 full_path: Return the full path in CROSS_COMPILE and don't set 157 PATH 158 Returns: 159 Dict containing the environemnt to use. This is based on the current 160 environment, with changes as needed to CROSS_COMPILE, PATH and 161 LC_ALL. 162 """ 163 env = dict(os.environ) 164 wrapper = self.GetWrapper() 165 166 if self.override_toolchain: 167 # We'll use MakeArgs() to provide this 168 pass 169 elif full_path: 170 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross) 171 else: 172 env['CROSS_COMPILE'] = wrapper + self.cross 173 env['PATH'] = self.path + ':' + env['PATH'] 174 175 env['LC_ALL'] = 'C' 176 177 return env 178 179 def MakeArgs(self): 180 """Create the 'make' arguments for a toolchain 181 182 This is only used when the toolchain is being overridden. Since the 183 U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the 184 environment (and MakeEnvironment()) to override these values. This 185 function returns the arguments to accomplish this. 186 187 Returns: 188 List of arguments to pass to 'make' 189 """ 190 if self.override_toolchain: 191 return ['HOSTCC=%s' % self.override_toolchain, 192 'CC=%s' % self.override_toolchain] 193 return [] 194 195 196class Toolchains: 197 """Manage a list of toolchains for building U-Boot 198 199 We select one toolchain for each architecture type 200 201 Public members: 202 toolchains: Dict of Toolchain objects, keyed by architecture name 203 prefixes: Dict of prefixes to check, keyed by architecture. This can 204 be a full path and toolchain prefix, for example 205 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of 206 something on the search path, for example 207 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. 208 paths: List of paths to check for toolchains (may contain wildcards) 209 """ 210 211 def __init__(self, override_toolchain=None): 212 self.toolchains = {} 213 self.prefixes = {} 214 self.paths = [] 215 self.override_toolchain = override_toolchain 216 self._make_flags = dict(bsettings.GetItems('make-flags')) 217 218 def GetPathList(self, show_warning=True): 219 """Get a list of available toolchain paths 220 221 Args: 222 show_warning: True to show a warning if there are no tool chains. 223 224 Returns: 225 List of strings, each a path to a toolchain mentioned in the 226 [toolchain] section of the settings file. 227 """ 228 toolchains = bsettings.GetItems('toolchain') 229 if show_warning and not toolchains: 230 print ("Warning: No tool chains. Please run 'buildman " 231 "--fetch-arch all' to download all available toolchains, or " 232 "add a [toolchain] section to your buildman config file " 233 "%s. See README for details" % 234 bsettings.config_fname) 235 236 paths = [] 237 for name, value in toolchains: 238 if '*' in value: 239 paths += glob.glob(value) 240 else: 241 paths.append(value) 242 return paths 243 244 def GetSettings(self, show_warning=True): 245 """Get toolchain settings from the settings file. 246 247 Args: 248 show_warning: True to show a warning if there are no tool chains. 249 """ 250 self.prefixes = bsettings.GetItems('toolchain-prefix') 251 self.paths += self.GetPathList(show_warning) 252 253 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, 254 arch=None): 255 """Add a toolchain to our list 256 257 We select the given toolchain as our preferred one for its 258 architecture if it is a higher priority than the others. 259 260 Args: 261 fname: Filename of toolchain's gcc driver 262 test: True to run the toolchain to test it 263 priority: Priority to use for this toolchain 264 arch: Toolchain architecture, or None if not known 265 """ 266 toolchain = Toolchain(fname, test, verbose, priority, arch, 267 self.override_toolchain) 268 add_it = toolchain.ok 269 if toolchain.arch in self.toolchains: 270 add_it = (toolchain.priority < 271 self.toolchains[toolchain.arch].priority) 272 if add_it: 273 self.toolchains[toolchain.arch] = toolchain 274 elif verbose: 275 print ("Toolchain '%s' at priority %d will be ignored because " 276 "another toolchain for arch '%s' has priority %d" % 277 (toolchain.gcc, toolchain.priority, toolchain.arch, 278 self.toolchains[toolchain.arch].priority)) 279 280 def ScanPath(self, path, verbose): 281 """Scan a path for a valid toolchain 282 283 Args: 284 path: Path to scan 285 verbose: True to print out progress information 286 Returns: 287 Filename of C compiler if found, else None 288 """ 289 fnames = [] 290 for subdir in ['.', 'bin', 'usr/bin']: 291 dirname = os.path.join(path, subdir) 292 if verbose: print " - looking in '%s'" % dirname 293 for fname in glob.glob(dirname + '/*gcc'): 294 if verbose: print " - found '%s'" % fname 295 fnames.append(fname) 296 return fnames 297 298 def ScanPathEnv(self, fname): 299 """Scan the PATH environment variable for a given filename. 300 301 Args: 302 fname: Filename to scan for 303 Returns: 304 List of matching pathanames, or [] if none 305 """ 306 pathname_list = [] 307 for path in os.environ["PATH"].split(os.pathsep): 308 path = path.strip('"') 309 pathname = os.path.join(path, fname) 310 if os.path.exists(pathname): 311 pathname_list.append(pathname) 312 return pathname_list 313 314 def Scan(self, verbose): 315 """Scan for available toolchains and select the best for each arch. 316 317 We look for all the toolchains we can file, figure out the 318 architecture for each, and whether it works. Then we select the 319 highest priority toolchain for each arch. 320 321 Args: 322 verbose: True to print out progress information 323 """ 324 if verbose: print 'Scanning for tool chains' 325 for name, value in self.prefixes: 326 if verbose: print " - scanning prefix '%s'" % value 327 if os.path.exists(value): 328 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) 329 continue 330 fname = value + 'gcc' 331 if os.path.exists(fname): 332 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) 333 continue 334 fname_list = self.ScanPathEnv(fname) 335 for f in fname_list: 336 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) 337 if not fname_list: 338 raise ValueError, ("No tool chain found for prefix '%s'" % 339 value) 340 for path in self.paths: 341 if verbose: print " - scanning path '%s'" % path 342 fnames = self.ScanPath(path, verbose) 343 for fname in fnames: 344 self.Add(fname, True, verbose) 345 346 def List(self): 347 """List out the selected toolchains for each architecture""" 348 col = terminal.Color() 349 print col.Color(col.BLUE, 'List of available toolchains (%d):' % 350 len(self.toolchains)) 351 if len(self.toolchains): 352 for key, value in sorted(self.toolchains.iteritems()): 353 print '%-10s: %s' % (key, value.gcc) 354 else: 355 print 'None' 356 357 def Select(self, arch): 358 """Returns the toolchain for a given architecture 359 360 Args: 361 args: Name of architecture (e.g. 'arm', 'ppc_8xx') 362 363 returns: 364 toolchain object, or None if none found 365 """ 366 for tag, value in bsettings.GetItems('toolchain-alias'): 367 if arch == tag: 368 for alias in value.split(): 369 if alias in self.toolchains: 370 return self.toolchains[alias] 371 372 if not arch in self.toolchains: 373 raise ValueError, ("No tool chain found for arch '%s'" % arch) 374 return self.toolchains[arch] 375 376 def ResolveReferences(self, var_dict, args): 377 """Resolve variable references in a string 378 379 This converts ${blah} within the string to the value of blah. 380 This function works recursively. 381 382 Args: 383 var_dict: Dictionary containing variables and their values 384 args: String containing make arguments 385 Returns: 386 Resolved string 387 388 >>> bsettings.Setup() 389 >>> tcs = Toolchains() 390 >>> tcs.Add('fred', False) 391 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ 392 'second' : '2nd'} 393 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') 394 'this=OBLIQUE_set' 395 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') 396 'this=OBLIQUE_setfi2ndrstnd' 397 """ 398 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') 399 400 while True: 401 m = re_var.search(args) 402 if not m: 403 break 404 lookup = m.group(0)[2:-1] 405 value = var_dict.get(lookup, '') 406 args = args[:m.start(0)] + value + args[m.end(0):] 407 return args 408 409 def GetMakeArguments(self, board): 410 """Returns 'make' arguments for a given board 411 412 The flags are in a section called 'make-flags'. Flags are named 413 after the target they represent, for example snapper9260=TESTING=1 414 will pass TESTING=1 to make when building the snapper9260 board. 415 416 References to other boards can be added in the string also. For 417 example: 418 419 [make-flags] 420 at91-boards=ENABLE_AT91_TEST=1 421 snapper9260=${at91-boards} BUILD_TAG=442 422 snapper9g45=${at91-boards} BUILD_TAG=443 423 424 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 425 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. 426 427 A special 'target' variable is set to the board target. 428 429 Args: 430 board: Board object for the board to check. 431 Returns: 432 'make' flags for that board, or '' if none 433 """ 434 self._make_flags['target'] = board.target 435 arg_str = self.ResolveReferences(self._make_flags, 436 self._make_flags.get(board.target, '')) 437 args = arg_str.split(' ') 438 i = 0 439 while i < len(args): 440 if not args[i]: 441 del args[i] 442 else: 443 i += 1 444 return args 445 446 def LocateArchUrl(self, fetch_arch): 447 """Find a toolchain available online 448 449 Look in standard places for available toolchains. At present the 450 only standard place is at kernel.org. 451 452 Args: 453 arch: Architecture to look for, or 'list' for all 454 Returns: 455 If fetch_arch is 'list', a tuple: 456 Machine architecture (e.g. x86_64) 457 List of toolchains 458 else 459 URL containing this toolchain, if avaialble, else None 460 """ 461 arch = command.OutputOneLine('uname', '-m') 462 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' 463 versions = ['7.3.0', '6.4.0', '4.9.4'] 464 links = [] 465 for version in versions: 466 url = '%s/%s/%s/' % (base, arch, version) 467 print 'Checking: %s' % url 468 response = urllib2.urlopen(url) 469 html = response.read() 470 parser = MyHTMLParser(fetch_arch) 471 parser.feed(html) 472 if fetch_arch == 'list': 473 links += parser.links 474 elif parser.arch_link: 475 return url + parser.arch_link 476 if fetch_arch == 'list': 477 return arch, links 478 return None 479 480 def Download(self, url): 481 """Download a file to a temporary directory 482 483 Args: 484 url: URL to download 485 Returns: 486 Tuple: 487 Temporary directory name 488 Full path to the downloaded archive file in that directory, 489 or None if there was an error while downloading 490 """ 491 print 'Downloading: %s' % url 492 leaf = url.split('/')[-1] 493 tmpdir = tempfile.mkdtemp('.buildman') 494 response = urllib2.urlopen(url) 495 fname = os.path.join(tmpdir, leaf) 496 fd = open(fname, 'wb') 497 meta = response.info() 498 size = int(meta.getheaders('Content-Length')[0]) 499 done = 0 500 block_size = 1 << 16 501 status = '' 502 503 # Read the file in chunks and show progress as we go 504 while True: 505 buffer = response.read(block_size) 506 if not buffer: 507 print chr(8) * (len(status) + 1), '\r', 508 break 509 510 done += len(buffer) 511 fd.write(buffer) 512 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024, 513 done * 100 / size) 514 status = status + chr(8) * (len(status) + 1) 515 print status, 516 sys.stdout.flush() 517 fd.close() 518 if done != size: 519 print 'Error, failed to download' 520 os.remove(fname) 521 fname = None 522 return tmpdir, fname 523 524 def Unpack(self, fname, dest): 525 """Unpack a tar file 526 527 Args: 528 fname: Filename to unpack 529 dest: Destination directory 530 Returns: 531 Directory name of the first entry in the archive, without the 532 trailing / 533 """ 534 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) 535 dirs = stdout.splitlines()[1].split('/')[:2] 536 return '/'.join(dirs) 537 538 def TestSettingsHasPath(self, path): 539 """Check if buildman will find this toolchain 540 541 Returns: 542 True if the path is in settings, False if not 543 """ 544 paths = self.GetPathList(False) 545 return path in paths 546 547 def ListArchs(self): 548 """List architectures with available toolchains to download""" 549 host_arch, archives = self.LocateArchUrl('list') 550 re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*') 551 arch_set = set() 552 for archive in archives: 553 # Remove the host architecture from the start 554 arch = re_arch.match(archive[len(host_arch):]) 555 if arch: 556 if arch.group(1) != '2.0' and arch.group(1) != '64': 557 arch_set.add(arch.group(1)) 558 return sorted(arch_set) 559 560 def FetchAndInstall(self, arch): 561 """Fetch and install a new toolchain 562 563 arch: 564 Architecture to fetch, or 'list' to list 565 """ 566 # Fist get the URL for this architecture 567 col = terminal.Color() 568 print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch) 569 url = self.LocateArchUrl(arch) 570 if not url: 571 print ("Cannot find toolchain for arch '%s' - use 'list' to list" % 572 arch) 573 return 2 574 home = os.environ['HOME'] 575 dest = os.path.join(home, '.buildman-toolchains') 576 if not os.path.exists(dest): 577 os.mkdir(dest) 578 579 # Download the tar file for this toolchain and unpack it 580 tmpdir, tarfile = self.Download(url) 581 if not tarfile: 582 return 1 583 print col.Color(col.GREEN, 'Unpacking to: %s' % dest), 584 sys.stdout.flush() 585 path = self.Unpack(tarfile, dest) 586 os.remove(tarfile) 587 os.rmdir(tmpdir) 588 print 589 590 # Check that the toolchain works 591 print col.Color(col.GREEN, 'Testing') 592 dirpath = os.path.join(dest, path) 593 compiler_fname_list = self.ScanPath(dirpath, True) 594 if not compiler_fname_list: 595 print 'Could not locate C compiler - fetch failed.' 596 return 1 597 if len(compiler_fname_list) != 1: 598 print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' % 599 ', '.join(compiler_fname_list)) 600 toolchain = Toolchain(compiler_fname_list[0], True, True) 601 602 # Make sure that it will be found by buildman 603 if not self.TestSettingsHasPath(dirpath): 604 print ("Adding 'download' to config file '%s'" % 605 bsettings.config_fname) 606 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest) 607 return 0 608