1# Copyright (c) 2012 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import re 7import glob 8from HTMLParser import HTMLParser 9import os 10import sys 11import tempfile 12import urllib2 13 14import bsettings 15import command 16 17(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, 18 PRIORITY_CALC) = range(4) 19 20# Simple class to collect links from a page 21class MyHTMLParser(HTMLParser): 22 def __init__(self, arch): 23 """Create a new parser 24 25 After the parser runs, self.links will be set to a list of the links 26 to .xz archives found in the page, and self.arch_link will be set to 27 the one for the given architecture (or None if not found). 28 29 Args: 30 arch: Architecture to search for 31 """ 32 HTMLParser.__init__(self) 33 self.arch_link = None 34 self.links = [] 35 self._match = '_%s-' % arch 36 37 def handle_starttag(self, tag, attrs): 38 if tag == 'a': 39 for tag, value in attrs: 40 if tag == 'href': 41 if value and value.endswith('.xz'): 42 self.links.append(value) 43 if self._match in value: 44 self.arch_link = value 45 46 47class Toolchain: 48 """A single toolchain 49 50 Public members: 51 gcc: Full path to C compiler 52 path: Directory path containing C compiler 53 cross: Cross compile string, e.g. 'arm-linux-' 54 arch: Architecture of toolchain as determined from the first 55 component of the filename. E.g. arm-linux-gcc becomes arm 56 priority: Toolchain priority (0=highest, 20=lowest) 57 """ 58 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, 59 arch=None): 60 """Create a new toolchain object. 61 62 Args: 63 fname: Filename of the gcc component 64 test: True to run the toolchain to test it 65 verbose: True to print out the information 66 priority: Priority to use for this toolchain, or PRIORITY_CALC to 67 calculate it 68 """ 69 self.gcc = fname 70 self.path = os.path.dirname(fname) 71 72 # Find the CROSS_COMPILE prefix to use for U-Boot. For example, 73 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. 74 basename = os.path.basename(fname) 75 pos = basename.rfind('-') 76 self.cross = basename[:pos + 1] if pos != -1 else '' 77 78 # The architecture is the first part of the name 79 pos = self.cross.find('-') 80 if arch: 81 self.arch = arch 82 else: 83 self.arch = self.cross[:pos] if pos != -1 else 'sandbox' 84 85 env = self.MakeEnvironment(False) 86 87 # As a basic sanity check, run the C compiler with --version 88 cmd = [fname, '--version'] 89 if priority == PRIORITY_CALC: 90 self.priority = self.GetPriority(fname) 91 else: 92 self.priority = priority 93 if test: 94 result = command.RunPipe([cmd], capture=True, env=env, 95 raise_on_error=False) 96 self.ok = result.return_code == 0 97 if verbose: 98 print 'Tool chain test: ', 99 if self.ok: 100 print "OK, arch='%s', priority %d" % (self.arch, 101 self.priority) 102 else: 103 print 'BAD' 104 print 'Command: ', cmd 105 print result.stdout 106 print result.stderr 107 else: 108 self.ok = True 109 110 def GetPriority(self, fname): 111 """Return the priority of the toolchain. 112 113 Toolchains are ranked according to their suitability by their 114 filename prefix. 115 116 Args: 117 fname: Filename of toolchain 118 Returns: 119 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. 120 """ 121 priority_list = ['-elf', '-unknown-linux-gnu', '-linux', 122 '-none-linux-gnueabi', '-uclinux', '-none-eabi', 123 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux'] 124 for prio in range(len(priority_list)): 125 if priority_list[prio] in fname: 126 return PRIORITY_CALC + prio 127 return PRIORITY_CALC + prio 128 129 def MakeEnvironment(self, full_path): 130 """Returns an environment for using the toolchain. 131 132 Thie takes the current environment and adds CROSS_COMPILE so that 133 the tool chain will operate correctly. 134 135 Args: 136 full_path: Return the full path in CROSS_COMPILE and don't set 137 PATH 138 """ 139 env = dict(os.environ) 140 if full_path: 141 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross) 142 else: 143 env['CROSS_COMPILE'] = self.cross 144 env['PATH'] = self.path + ':' + env['PATH'] 145 146 return env 147 148 149class Toolchains: 150 """Manage a list of toolchains for building U-Boot 151 152 We select one toolchain for each architecture type 153 154 Public members: 155 toolchains: Dict of Toolchain objects, keyed by architecture name 156 prefixes: Dict of prefixes to check, keyed by architecture. This can 157 be a full path and toolchain prefix, for example 158 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of 159 something on the search path, for example 160 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. 161 paths: List of paths to check for toolchains (may contain wildcards) 162 """ 163 164 def __init__(self): 165 self.toolchains = {} 166 self.prefixes = {} 167 self.paths = [] 168 self._make_flags = dict(bsettings.GetItems('make-flags')) 169 170 def GetPathList(self): 171 """Get a list of available toolchain paths 172 173 Returns: 174 List of strings, each a path to a toolchain mentioned in the 175 [toolchain] section of the settings file. 176 """ 177 toolchains = bsettings.GetItems('toolchain') 178 if not toolchains: 179 print ('Warning: No tool chains - please add a [toolchain] section' 180 ' to your buildman config file %s. See README for details' % 181 bsettings.config_fname) 182 183 paths = [] 184 for name, value in toolchains: 185 if '*' in value: 186 paths += glob.glob(value) 187 else: 188 paths.append(value) 189 return paths 190 191 def GetSettings(self): 192 self.prefixes = bsettings.GetItems('toolchain-prefix') 193 self.paths += self.GetPathList() 194 195 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, 196 arch=None): 197 """Add a toolchain to our list 198 199 We select the given toolchain as our preferred one for its 200 architecture if it is a higher priority than the others. 201 202 Args: 203 fname: Filename of toolchain's gcc driver 204 test: True to run the toolchain to test it 205 priority: Priority to use for this toolchain 206 arch: Toolchain architecture, or None if not known 207 """ 208 toolchain = Toolchain(fname, test, verbose, priority, arch) 209 add_it = toolchain.ok 210 if toolchain.arch in self.toolchains: 211 add_it = (toolchain.priority < 212 self.toolchains[toolchain.arch].priority) 213 if add_it: 214 self.toolchains[toolchain.arch] = toolchain 215 elif verbose: 216 print ("Toolchain '%s' at priority %d will be ignored because " 217 "another toolchain for arch '%s' has priority %d" % 218 (toolchain.gcc, toolchain.priority, toolchain.arch, 219 self.toolchains[toolchain.arch].priority)) 220 221 def ScanPath(self, path, verbose): 222 """Scan a path for a valid toolchain 223 224 Args: 225 path: Path to scan 226 verbose: True to print out progress information 227 Returns: 228 Filename of C compiler if found, else None 229 """ 230 fnames = [] 231 for subdir in ['.', 'bin', 'usr/bin']: 232 dirname = os.path.join(path, subdir) 233 if verbose: print " - looking in '%s'" % dirname 234 for fname in glob.glob(dirname + '/*gcc'): 235 if verbose: print " - found '%s'" % fname 236 fnames.append(fname) 237 return fnames 238 239 def ScanPathEnv(self, fname): 240 """Scan the PATH environment variable for a given filename. 241 242 Args: 243 fname: Filename to scan for 244 Returns: 245 List of matching pathanames, or [] if none 246 """ 247 pathname_list = [] 248 for path in os.environ["PATH"].split(os.pathsep): 249 path = path.strip('"') 250 pathname = os.path.join(path, fname) 251 if os.path.exists(pathname): 252 pathname_list.append(pathname) 253 return pathname_list 254 255 def Scan(self, verbose): 256 """Scan for available toolchains and select the best for each arch. 257 258 We look for all the toolchains we can file, figure out the 259 architecture for each, and whether it works. Then we select the 260 highest priority toolchain for each arch. 261 262 Args: 263 verbose: True to print out progress information 264 """ 265 if verbose: print 'Scanning for tool chains' 266 for name, value in self.prefixes: 267 if verbose: print " - scanning prefix '%s'" % value 268 if os.path.exists(value): 269 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) 270 continue 271 fname = value + 'gcc' 272 if os.path.exists(fname): 273 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) 274 continue 275 fname_list = self.ScanPathEnv(fname) 276 for f in fname_list: 277 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) 278 if not fname_list: 279 raise ValueError, ("No tool chain found for prefix '%s'" % 280 value) 281 for path in self.paths: 282 if verbose: print " - scanning path '%s'" % path 283 fnames = self.ScanPath(path, verbose) 284 for fname in fnames: 285 self.Add(fname, True, verbose) 286 287 def List(self): 288 """List out the selected toolchains for each architecture""" 289 print 'List of available toolchains (%d):' % len(self.toolchains) 290 if len(self.toolchains): 291 for key, value in sorted(self.toolchains.iteritems()): 292 print '%-10s: %s' % (key, value.gcc) 293 else: 294 print 'None' 295 296 def Select(self, arch): 297 """Returns the toolchain for a given architecture 298 299 Args: 300 args: Name of architecture (e.g. 'arm', 'ppc_8xx') 301 302 returns: 303 toolchain object, or None if none found 304 """ 305 for tag, value in bsettings.GetItems('toolchain-alias'): 306 if arch == tag: 307 for alias in value.split(): 308 if alias in self.toolchains: 309 return self.toolchains[alias] 310 311 if not arch in self.toolchains: 312 raise ValueError, ("No tool chain found for arch '%s'" % arch) 313 return self.toolchains[arch] 314 315 def ResolveReferences(self, var_dict, args): 316 """Resolve variable references in a string 317 318 This converts ${blah} within the string to the value of blah. 319 This function works recursively. 320 321 Args: 322 var_dict: Dictionary containing variables and their values 323 args: String containing make arguments 324 Returns: 325 Resolved string 326 327 >>> bsettings.Setup() 328 >>> tcs = Toolchains() 329 >>> tcs.Add('fred', False) 330 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ 331 'second' : '2nd'} 332 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') 333 'this=OBLIQUE_set' 334 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') 335 'this=OBLIQUE_setfi2ndrstnd' 336 """ 337 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') 338 339 while True: 340 m = re_var.search(args) 341 if not m: 342 break 343 lookup = m.group(0)[2:-1] 344 value = var_dict.get(lookup, '') 345 args = args[:m.start(0)] + value + args[m.end(0):] 346 return args 347 348 def GetMakeArguments(self, board): 349 """Returns 'make' arguments for a given board 350 351 The flags are in a section called 'make-flags'. Flags are named 352 after the target they represent, for example snapper9260=TESTING=1 353 will pass TESTING=1 to make when building the snapper9260 board. 354 355 References to other boards can be added in the string also. For 356 example: 357 358 [make-flags] 359 at91-boards=ENABLE_AT91_TEST=1 360 snapper9260=${at91-boards} BUILD_TAG=442 361 snapper9g45=${at91-boards} BUILD_TAG=443 362 363 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 364 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. 365 366 A special 'target' variable is set to the board target. 367 368 Args: 369 board: Board object for the board to check. 370 Returns: 371 'make' flags for that board, or '' if none 372 """ 373 self._make_flags['target'] = board.target 374 arg_str = self.ResolveReferences(self._make_flags, 375 self._make_flags.get(board.target, '')) 376 args = arg_str.split(' ') 377 i = 0 378 while i < len(args): 379 if not args[i]: 380 del args[i] 381 else: 382 i += 1 383 return args 384 385 def LocateArchUrl(self, fetch_arch): 386 """Find a toolchain available online 387 388 Look in standard places for available toolchains. At present the 389 only standard place is at kernel.org. 390 391 Args: 392 arch: Architecture to look for, or 'list' for all 393 Returns: 394 If fetch_arch is 'list', a tuple: 395 Machine architecture (e.g. x86_64) 396 List of toolchains 397 else 398 URL containing this toolchain, if avaialble, else None 399 """ 400 arch = command.OutputOneLine('uname', '-m') 401 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' 402 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4'] 403 links = [] 404 for version in versions: 405 url = '%s/%s/%s/' % (base, arch, version) 406 print 'Checking: %s' % url 407 response = urllib2.urlopen(url) 408 html = response.read() 409 parser = MyHTMLParser(fetch_arch) 410 parser.feed(html) 411 if fetch_arch == 'list': 412 links += parser.links 413 elif parser.arch_link: 414 return url + parser.arch_link 415 if fetch_arch == 'list': 416 return arch, links 417 return None 418 419 def Download(self, url): 420 """Download a file to a temporary directory 421 422 Args: 423 url: URL to download 424 Returns: 425 Tuple: 426 Temporary directory name 427 Full path to the downloaded archive file in that directory, 428 or None if there was an error while downloading 429 """ 430 print 'Downloading: %s' % url 431 leaf = url.split('/')[-1] 432 tmpdir = tempfile.mkdtemp('.buildman') 433 response = urllib2.urlopen(url) 434 fname = os.path.join(tmpdir, leaf) 435 fd = open(fname, 'wb') 436 meta = response.info() 437 size = int(meta.getheaders('Content-Length')[0]) 438 done = 0 439 block_size = 1 << 16 440 status = '' 441 442 # Read the file in chunks and show progress as we go 443 while True: 444 buffer = response.read(block_size) 445 if not buffer: 446 print chr(8) * (len(status) + 1), '\r', 447 break 448 449 done += len(buffer) 450 fd.write(buffer) 451 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024, 452 done * 100 / size) 453 status = status + chr(8) * (len(status) + 1) 454 print status, 455 sys.stdout.flush() 456 fd.close() 457 if done != size: 458 print 'Error, failed to download' 459 os.remove(fname) 460 fname = None 461 return tmpdir, fname 462 463 def Unpack(self, fname, dest): 464 """Unpack a tar file 465 466 Args: 467 fname: Filename to unpack 468 dest: Destination directory 469 Returns: 470 Directory name of the first entry in the archive, without the 471 trailing / 472 """ 473 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) 474 return stdout.splitlines()[0][:-1] 475 476 def TestSettingsHasPath(self, path): 477 """Check if builmand will find this toolchain 478 479 Returns: 480 True if the path is in settings, False if not 481 """ 482 paths = self.GetPathList() 483 return path in paths 484 485 def ListArchs(self): 486 """List architectures with available toolchains to download""" 487 host_arch, archives = self.LocateArchUrl('list') 488 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') 489 arch_set = set() 490 for archive in archives: 491 # Remove the host architecture from the start 492 arch = re_arch.match(archive[len(host_arch):]) 493 if arch: 494 arch_set.add(arch.group(1)) 495 return sorted(arch_set) 496 497 def FetchAndInstall(self, arch): 498 """Fetch and install a new toolchain 499 500 arch: 501 Architecture to fetch, or 'list' to list 502 """ 503 # Fist get the URL for this architecture 504 url = self.LocateArchUrl(arch) 505 if not url: 506 print ("Cannot find toolchain for arch '%s' - use 'list' to list" % 507 arch) 508 return 2 509 home = os.environ['HOME'] 510 dest = os.path.join(home, '.buildman-toolchains') 511 if not os.path.exists(dest): 512 os.mkdir(dest) 513 514 # Download the tar file for this toolchain and unpack it 515 tmpdir, tarfile = self.Download(url) 516 if not tarfile: 517 return 1 518 print 'Unpacking to: %s' % dest, 519 sys.stdout.flush() 520 path = self.Unpack(tarfile, dest) 521 os.remove(tarfile) 522 os.rmdir(tmpdir) 523 print 524 525 # Check that the toolchain works 526 print 'Testing' 527 dirpath = os.path.join(dest, path) 528 compiler_fname_list = self.ScanPath(dirpath, True) 529 if not compiler_fname_list: 530 print 'Could not locate C compiler - fetch failed.' 531 return 1 532 if len(compiler_fname_list) != 1: 533 print ('Internal error, ambiguous toolchains: %s' % 534 (', '.join(compiler_fname))) 535 return 1 536 toolchain = Toolchain(compiler_fname_list[0], True, True) 537 538 # Make sure that it will be found by buildman 539 if not self.TestSettingsHasPath(dirpath): 540 print ("Adding 'download' to config file '%s'" % 541 bsettings.config_fname) 542 tools_dir = os.path.dirname(dirpath) 543 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir) 544 return 0 545