1# Copyright (c) 2012 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import re 7import glob 8from HTMLParser import HTMLParser 9import os 10import sys 11import tempfile 12import urllib2 13 14import bsettings 15import command 16import terminal 17 18(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, 19 PRIORITY_CALC) = range(4) 20 21# Simple class to collect links from a page 22class MyHTMLParser(HTMLParser): 23 def __init__(self, arch): 24 """Create a new parser 25 26 After the parser runs, self.links will be set to a list of the links 27 to .xz archives found in the page, and self.arch_link will be set to 28 the one for the given architecture (or None if not found). 29 30 Args: 31 arch: Architecture to search for 32 """ 33 HTMLParser.__init__(self) 34 self.arch_link = None 35 self.links = [] 36 self._match = '_%s-' % arch 37 38 def handle_starttag(self, tag, attrs): 39 if tag == 'a': 40 for tag, value in attrs: 41 if tag == 'href': 42 if value and value.endswith('.xz'): 43 self.links.append(value) 44 if self._match in value: 45 self.arch_link = value 46 47 48class Toolchain: 49 """A single toolchain 50 51 Public members: 52 gcc: Full path to C compiler 53 path: Directory path containing C compiler 54 cross: Cross compile string, e.g. 'arm-linux-' 55 arch: Architecture of toolchain as determined from the first 56 component of the filename. E.g. arm-linux-gcc becomes arm 57 priority: Toolchain priority (0=highest, 20=lowest) 58 """ 59 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, 60 arch=None): 61 """Create a new toolchain object. 62 63 Args: 64 fname: Filename of the gcc component 65 test: True to run the toolchain to test it 66 verbose: True to print out the information 67 priority: Priority to use for this toolchain, or PRIORITY_CALC to 68 calculate it 69 """ 70 self.gcc = fname 71 self.path = os.path.dirname(fname) 72 73 # Find the CROSS_COMPILE prefix to use for U-Boot. For example, 74 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. 75 basename = os.path.basename(fname) 76 pos = basename.rfind('-') 77 self.cross = basename[:pos + 1] if pos != -1 else '' 78 79 # The architecture is the first part of the name 80 pos = self.cross.find('-') 81 if arch: 82 self.arch = arch 83 else: 84 self.arch = self.cross[:pos] if pos != -1 else 'sandbox' 85 86 env = self.MakeEnvironment(False) 87 88 # As a basic sanity check, run the C compiler with --version 89 cmd = [fname, '--version'] 90 if priority == PRIORITY_CALC: 91 self.priority = self.GetPriority(fname) 92 else: 93 self.priority = priority 94 if test: 95 result = command.RunPipe([cmd], capture=True, env=env, 96 raise_on_error=False) 97 self.ok = result.return_code == 0 98 if verbose: 99 print 'Tool chain test: ', 100 if self.ok: 101 print "OK, arch='%s', priority %d" % (self.arch, 102 self.priority) 103 else: 104 print 'BAD' 105 print 'Command: ', cmd 106 print result.stdout 107 print result.stderr 108 else: 109 self.ok = True 110 111 def GetPriority(self, fname): 112 """Return the priority of the toolchain. 113 114 Toolchains are ranked according to their suitability by their 115 filename prefix. 116 117 Args: 118 fname: Filename of toolchain 119 Returns: 120 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. 121 """ 122 priority_list = ['-elf', '-unknown-linux-gnu', '-linux', 123 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux', 124 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi', 125 '-linux-gnueabihf', '-le-linux', '-uclinux'] 126 for prio in range(len(priority_list)): 127 if priority_list[prio] in fname: 128 return PRIORITY_CALC + prio 129 return PRIORITY_CALC + prio 130 131 def GetWrapper(self, show_warning=True): 132 """Get toolchain wrapper from the setting file. 133 """ 134 value = '' 135 for name, value in bsettings.GetItems('toolchain-wrapper'): 136 if not value: 137 print "Warning: Wrapper not found" 138 if value: 139 value = value + ' ' 140 141 return value 142 143 def MakeEnvironment(self, full_path): 144 """Returns an environment for using the toolchain. 145 146 Thie takes the current environment and adds CROSS_COMPILE so that 147 the tool chain will operate correctly. 148 149 Args: 150 full_path: Return the full path in CROSS_COMPILE and don't set 151 PATH 152 """ 153 env = dict(os.environ) 154 wrapper = self.GetWrapper() 155 156 if full_path: 157 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross) 158 else: 159 env['CROSS_COMPILE'] = wrapper + self.cross 160 env['PATH'] = self.path + ':' + env['PATH'] 161 162 return env 163 164 165class Toolchains: 166 """Manage a list of toolchains for building U-Boot 167 168 We select one toolchain for each architecture type 169 170 Public members: 171 toolchains: Dict of Toolchain objects, keyed by architecture name 172 prefixes: Dict of prefixes to check, keyed by architecture. This can 173 be a full path and toolchain prefix, for example 174 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of 175 something on the search path, for example 176 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. 177 paths: List of paths to check for toolchains (may contain wildcards) 178 """ 179 180 def __init__(self): 181 self.toolchains = {} 182 self.prefixes = {} 183 self.paths = [] 184 self._make_flags = dict(bsettings.GetItems('make-flags')) 185 186 def GetPathList(self, show_warning=True): 187 """Get a list of available toolchain paths 188 189 Args: 190 show_warning: True to show a warning if there are no tool chains. 191 192 Returns: 193 List of strings, each a path to a toolchain mentioned in the 194 [toolchain] section of the settings file. 195 """ 196 toolchains = bsettings.GetItems('toolchain') 197 if show_warning and not toolchains: 198 print ("Warning: No tool chains. Please run 'buildman " 199 "--fetch-arch all' to download all available toolchains, or " 200 "add a [toolchain] section to your buildman config file " 201 "%s. See README for details" % 202 bsettings.config_fname) 203 204 paths = [] 205 for name, value in toolchains: 206 if '*' in value: 207 paths += glob.glob(value) 208 else: 209 paths.append(value) 210 return paths 211 212 def GetSettings(self, show_warning=True): 213 """Get toolchain settings from the settings file. 214 215 Args: 216 show_warning: True to show a warning if there are no tool chains. 217 """ 218 self.prefixes = bsettings.GetItems('toolchain-prefix') 219 self.paths += self.GetPathList(show_warning) 220 221 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, 222 arch=None): 223 """Add a toolchain to our list 224 225 We select the given toolchain as our preferred one for its 226 architecture if it is a higher priority than the others. 227 228 Args: 229 fname: Filename of toolchain's gcc driver 230 test: True to run the toolchain to test it 231 priority: Priority to use for this toolchain 232 arch: Toolchain architecture, or None if not known 233 """ 234 toolchain = Toolchain(fname, test, verbose, priority, arch) 235 add_it = toolchain.ok 236 if toolchain.arch in self.toolchains: 237 add_it = (toolchain.priority < 238 self.toolchains[toolchain.arch].priority) 239 if add_it: 240 self.toolchains[toolchain.arch] = toolchain 241 elif verbose: 242 print ("Toolchain '%s' at priority %d will be ignored because " 243 "another toolchain for arch '%s' has priority %d" % 244 (toolchain.gcc, toolchain.priority, toolchain.arch, 245 self.toolchains[toolchain.arch].priority)) 246 247 def ScanPath(self, path, verbose): 248 """Scan a path for a valid toolchain 249 250 Args: 251 path: Path to scan 252 verbose: True to print out progress information 253 Returns: 254 Filename of C compiler if found, else None 255 """ 256 fnames = [] 257 for subdir in ['.', 'bin', 'usr/bin']: 258 dirname = os.path.join(path, subdir) 259 if verbose: print " - looking in '%s'" % dirname 260 for fname in glob.glob(dirname + '/*gcc'): 261 if verbose: print " - found '%s'" % fname 262 fnames.append(fname) 263 return fnames 264 265 def ScanPathEnv(self, fname): 266 """Scan the PATH environment variable for a given filename. 267 268 Args: 269 fname: Filename to scan for 270 Returns: 271 List of matching pathanames, or [] if none 272 """ 273 pathname_list = [] 274 for path in os.environ["PATH"].split(os.pathsep): 275 path = path.strip('"') 276 pathname = os.path.join(path, fname) 277 if os.path.exists(pathname): 278 pathname_list.append(pathname) 279 return pathname_list 280 281 def Scan(self, verbose): 282 """Scan for available toolchains and select the best for each arch. 283 284 We look for all the toolchains we can file, figure out the 285 architecture for each, and whether it works. Then we select the 286 highest priority toolchain for each arch. 287 288 Args: 289 verbose: True to print out progress information 290 """ 291 if verbose: print 'Scanning for tool chains' 292 for name, value in self.prefixes: 293 if verbose: print " - scanning prefix '%s'" % value 294 if os.path.exists(value): 295 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) 296 continue 297 fname = value + 'gcc' 298 if os.path.exists(fname): 299 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) 300 continue 301 fname_list = self.ScanPathEnv(fname) 302 for f in fname_list: 303 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) 304 if not fname_list: 305 raise ValueError, ("No tool chain found for prefix '%s'" % 306 value) 307 for path in self.paths: 308 if verbose: print " - scanning path '%s'" % path 309 fnames = self.ScanPath(path, verbose) 310 for fname in fnames: 311 self.Add(fname, True, verbose) 312 313 def List(self): 314 """List out the selected toolchains for each architecture""" 315 col = terminal.Color() 316 print col.Color(col.BLUE, 'List of available toolchains (%d):' % 317 len(self.toolchains)) 318 if len(self.toolchains): 319 for key, value in sorted(self.toolchains.iteritems()): 320 print '%-10s: %s' % (key, value.gcc) 321 else: 322 print 'None' 323 324 def Select(self, arch): 325 """Returns the toolchain for a given architecture 326 327 Args: 328 args: Name of architecture (e.g. 'arm', 'ppc_8xx') 329 330 returns: 331 toolchain object, or None if none found 332 """ 333 for tag, value in bsettings.GetItems('toolchain-alias'): 334 if arch == tag: 335 for alias in value.split(): 336 if alias in self.toolchains: 337 return self.toolchains[alias] 338 339 if not arch in self.toolchains: 340 raise ValueError, ("No tool chain found for arch '%s'" % arch) 341 return self.toolchains[arch] 342 343 def ResolveReferences(self, var_dict, args): 344 """Resolve variable references in a string 345 346 This converts ${blah} within the string to the value of blah. 347 This function works recursively. 348 349 Args: 350 var_dict: Dictionary containing variables and their values 351 args: String containing make arguments 352 Returns: 353 Resolved string 354 355 >>> bsettings.Setup() 356 >>> tcs = Toolchains() 357 >>> tcs.Add('fred', False) 358 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ 359 'second' : '2nd'} 360 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') 361 'this=OBLIQUE_set' 362 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') 363 'this=OBLIQUE_setfi2ndrstnd' 364 """ 365 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') 366 367 while True: 368 m = re_var.search(args) 369 if not m: 370 break 371 lookup = m.group(0)[2:-1] 372 value = var_dict.get(lookup, '') 373 args = args[:m.start(0)] + value + args[m.end(0):] 374 return args 375 376 def GetMakeArguments(self, board): 377 """Returns 'make' arguments for a given board 378 379 The flags are in a section called 'make-flags'. Flags are named 380 after the target they represent, for example snapper9260=TESTING=1 381 will pass TESTING=1 to make when building the snapper9260 board. 382 383 References to other boards can be added in the string also. For 384 example: 385 386 [make-flags] 387 at91-boards=ENABLE_AT91_TEST=1 388 snapper9260=${at91-boards} BUILD_TAG=442 389 snapper9g45=${at91-boards} BUILD_TAG=443 390 391 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 392 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. 393 394 A special 'target' variable is set to the board target. 395 396 Args: 397 board: Board object for the board to check. 398 Returns: 399 'make' flags for that board, or '' if none 400 """ 401 self._make_flags['target'] = board.target 402 arg_str = self.ResolveReferences(self._make_flags, 403 self._make_flags.get(board.target, '')) 404 args = arg_str.split(' ') 405 i = 0 406 while i < len(args): 407 if not args[i]: 408 del args[i] 409 else: 410 i += 1 411 return args 412 413 def LocateArchUrl(self, fetch_arch): 414 """Find a toolchain available online 415 416 Look in standard places for available toolchains. At present the 417 only standard place is at kernel.org. 418 419 Args: 420 arch: Architecture to look for, or 'list' for all 421 Returns: 422 If fetch_arch is 'list', a tuple: 423 Machine architecture (e.g. x86_64) 424 List of toolchains 425 else 426 URL containing this toolchain, if avaialble, else None 427 """ 428 arch = command.OutputOneLine('uname', '-m') 429 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' 430 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4'] 431 links = [] 432 for version in versions: 433 url = '%s/%s/%s/' % (base, arch, version) 434 print 'Checking: %s' % url 435 response = urllib2.urlopen(url) 436 html = response.read() 437 parser = MyHTMLParser(fetch_arch) 438 parser.feed(html) 439 if fetch_arch == 'list': 440 links += parser.links 441 elif parser.arch_link: 442 return url + parser.arch_link 443 if fetch_arch == 'list': 444 return arch, links 445 return None 446 447 def Download(self, url): 448 """Download a file to a temporary directory 449 450 Args: 451 url: URL to download 452 Returns: 453 Tuple: 454 Temporary directory name 455 Full path to the downloaded archive file in that directory, 456 or None if there was an error while downloading 457 """ 458 print 'Downloading: %s' % url 459 leaf = url.split('/')[-1] 460 tmpdir = tempfile.mkdtemp('.buildman') 461 response = urllib2.urlopen(url) 462 fname = os.path.join(tmpdir, leaf) 463 fd = open(fname, 'wb') 464 meta = response.info() 465 size = int(meta.getheaders('Content-Length')[0]) 466 done = 0 467 block_size = 1 << 16 468 status = '' 469 470 # Read the file in chunks and show progress as we go 471 while True: 472 buffer = response.read(block_size) 473 if not buffer: 474 print chr(8) * (len(status) + 1), '\r', 475 break 476 477 done += len(buffer) 478 fd.write(buffer) 479 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024, 480 done * 100 / size) 481 status = status + chr(8) * (len(status) + 1) 482 print status, 483 sys.stdout.flush() 484 fd.close() 485 if done != size: 486 print 'Error, failed to download' 487 os.remove(fname) 488 fname = None 489 return tmpdir, fname 490 491 def Unpack(self, fname, dest): 492 """Unpack a tar file 493 494 Args: 495 fname: Filename to unpack 496 dest: Destination directory 497 Returns: 498 Directory name of the first entry in the archive, without the 499 trailing / 500 """ 501 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) 502 return stdout.splitlines()[0][:-1] 503 504 def TestSettingsHasPath(self, path): 505 """Check if buildman will find this toolchain 506 507 Returns: 508 True if the path is in settings, False if not 509 """ 510 paths = self.GetPathList(False) 511 return path in paths 512 513 def ListArchs(self): 514 """List architectures with available toolchains to download""" 515 host_arch, archives = self.LocateArchUrl('list') 516 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') 517 arch_set = set() 518 for archive in archives: 519 # Remove the host architecture from the start 520 arch = re_arch.match(archive[len(host_arch):]) 521 if arch: 522 arch_set.add(arch.group(1)) 523 return sorted(arch_set) 524 525 def FetchAndInstall(self, arch): 526 """Fetch and install a new toolchain 527 528 arch: 529 Architecture to fetch, or 'list' to list 530 """ 531 # Fist get the URL for this architecture 532 col = terminal.Color() 533 print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch) 534 url = self.LocateArchUrl(arch) 535 if not url: 536 print ("Cannot find toolchain for arch '%s' - use 'list' to list" % 537 arch) 538 return 2 539 home = os.environ['HOME'] 540 dest = os.path.join(home, '.buildman-toolchains') 541 if not os.path.exists(dest): 542 os.mkdir(dest) 543 544 # Download the tar file for this toolchain and unpack it 545 tmpdir, tarfile = self.Download(url) 546 if not tarfile: 547 return 1 548 print col.Color(col.GREEN, 'Unpacking to: %s' % dest), 549 sys.stdout.flush() 550 path = self.Unpack(tarfile, dest) 551 os.remove(tarfile) 552 os.rmdir(tmpdir) 553 print 554 555 # Check that the toolchain works 556 print col.Color(col.GREEN, 'Testing') 557 dirpath = os.path.join(dest, path) 558 compiler_fname_list = self.ScanPath(dirpath, True) 559 if not compiler_fname_list: 560 print 'Could not locate C compiler - fetch failed.' 561 return 1 562 if len(compiler_fname_list) != 1: 563 print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' % 564 ', '.join(compiler_fname_list)) 565 toolchain = Toolchain(compiler_fname_list[0], True, True) 566 567 # Make sure that it will be found by buildman 568 if not self.TestSettingsHasPath(dirpath): 569 print ("Adding 'download' to config file '%s'" % 570 bsettings.config_fname) 571 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest) 572 return 0 573