1# Copyright (c) 2012 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import re 7import glob 8from HTMLParser import HTMLParser 9import os 10import sys 11import tempfile 12import urllib2 13 14import bsettings 15import command 16 17PRIORITY_CALC = 0 18 19# Simple class to collect links from a page 20class MyHTMLParser(HTMLParser): 21 def __init__(self, arch): 22 """Create a new parser 23 24 After the parser runs, self.links will be set to a list of the links 25 to .xz archives found in the page, and self.arch_link will be set to 26 the one for the given architecture (or None if not found). 27 28 Args: 29 arch: Architecture to search for 30 """ 31 HTMLParser.__init__(self) 32 self.arch_link = None 33 self.links = [] 34 self._match = '_%s-' % arch 35 36 def handle_starttag(self, tag, attrs): 37 if tag == 'a': 38 for tag, value in attrs: 39 if tag == 'href': 40 if value and value.endswith('.xz'): 41 self.links.append(value) 42 if self._match in value: 43 self.arch_link = value 44 45 46class Toolchain: 47 """A single toolchain 48 49 Public members: 50 gcc: Full path to C compiler 51 path: Directory path containing C compiler 52 cross: Cross compile string, e.g. 'arm-linux-' 53 arch: Architecture of toolchain as determined from the first 54 component of the filename. E.g. arm-linux-gcc becomes arm 55 priority: Toolchain priority (0=highest, 20=lowest) 56 """ 57 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, 58 arch=None): 59 """Create a new toolchain object. 60 61 Args: 62 fname: Filename of the gcc component 63 test: True to run the toolchain to test it 64 verbose: True to print out the information 65 priority: Priority to use for this toolchain, or PRIORITY_CALC to 66 calculate it 67 """ 68 self.gcc = fname 69 self.path = os.path.dirname(fname) 70 71 # Find the CROSS_COMPILE prefix to use for U-Boot. For example, 72 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. 73 basename = os.path.basename(fname) 74 pos = basename.rfind('-') 75 self.cross = basename[:pos + 1] if pos != -1 else '' 76 77 # The architecture is the first part of the name 78 pos = self.cross.find('-') 79 if arch: 80 self.arch = arch 81 else: 82 self.arch = self.cross[:pos] if pos != -1 else 'sandbox' 83 84 env = self.MakeEnvironment(False) 85 86 # As a basic sanity check, run the C compiler with --version 87 cmd = [fname, '--version'] 88 if priority == PRIORITY_CALC: 89 self.priority = self.GetPriority(fname) 90 else: 91 self.priority = priority 92 if test: 93 result = command.RunPipe([cmd], capture=True, env=env, 94 raise_on_error=False) 95 self.ok = result.return_code == 0 96 if verbose: 97 print 'Tool chain test: ', 98 if self.ok: 99 print "OK, arch='%s', priority %d" % (self.arch, 100 self.priority) 101 else: 102 print 'BAD' 103 print 'Command: ', cmd 104 print result.stdout 105 print result.stderr 106 else: 107 self.ok = True 108 109 def GetPriority(self, fname): 110 """Return the priority of the toolchain. 111 112 Toolchains are ranked according to their suitability by their 113 filename prefix. 114 115 Args: 116 fname: Filename of toolchain 117 Returns: 118 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. 119 """ 120 priority_list = ['-elf', '-unknown-linux-gnu', '-linux', 121 '-none-linux-gnueabi', '-uclinux', '-none-eabi', 122 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux'] 123 for prio in range(len(priority_list)): 124 if priority_list[prio] in fname: 125 return PRIORITY_CALC + prio 126 return PRIORITY_CALC + prio 127 128 def MakeEnvironment(self, full_path): 129 """Returns an environment for using the toolchain. 130 131 Thie takes the current environment and adds CROSS_COMPILE so that 132 the tool chain will operate correctly. 133 134 Args: 135 full_path: Return the full path in CROSS_COMPILE and don't set 136 PATH 137 """ 138 env = dict(os.environ) 139 if full_path: 140 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross) 141 else: 142 env['CROSS_COMPILE'] = self.cross 143 env['PATH'] = self.path + ':' + env['PATH'] 144 145 return env 146 147 148class Toolchains: 149 """Manage a list of toolchains for building U-Boot 150 151 We select one toolchain for each architecture type 152 153 Public members: 154 toolchains: Dict of Toolchain objects, keyed by architecture name 155 paths: List of paths to check for toolchains (may contain wildcards) 156 """ 157 158 def __init__(self): 159 self.toolchains = {} 160 self.paths = [] 161 self._make_flags = dict(bsettings.GetItems('make-flags')) 162 163 def GetPathList(self): 164 """Get a list of available toolchain paths 165 166 Returns: 167 List of strings, each a path to a toolchain mentioned in the 168 [toolchain] section of the settings file. 169 """ 170 toolchains = bsettings.GetItems('toolchain') 171 if not toolchains: 172 print ('Warning: No tool chains - please add a [toolchain] section' 173 ' to your buildman config file %s. See README for details' % 174 bsettings.config_fname) 175 176 paths = [] 177 for name, value in toolchains: 178 if '*' in value: 179 paths += glob.glob(value) 180 else: 181 paths.append(value) 182 return paths 183 184 def GetSettings(self): 185 self.paths += self.GetPathList() 186 187 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, 188 arch=None): 189 """Add a toolchain to our list 190 191 We select the given toolchain as our preferred one for its 192 architecture if it is a higher priority than the others. 193 194 Args: 195 fname: Filename of toolchain's gcc driver 196 test: True to run the toolchain to test it 197 priority: Priority to use for this toolchain 198 arch: Toolchain architecture, or None if not known 199 """ 200 toolchain = Toolchain(fname, test, verbose, priority, arch) 201 add_it = toolchain.ok 202 if toolchain.arch in self.toolchains: 203 add_it = (toolchain.priority < 204 self.toolchains[toolchain.arch].priority) 205 if add_it: 206 self.toolchains[toolchain.arch] = toolchain 207 elif verbose: 208 print ("Toolchain '%s' at priority %d will be ignored because " 209 "another toolchain for arch '%s' has priority %d" % 210 (toolchain.gcc, toolchain.priority, toolchain.arch, 211 self.toolchains[toolchain.arch].priority)) 212 213 def ScanPath(self, path, verbose): 214 """Scan a path for a valid toolchain 215 216 Args: 217 path: Path to scan 218 verbose: True to print out progress information 219 Returns: 220 Filename of C compiler if found, else None 221 """ 222 fnames = [] 223 for subdir in ['.', 'bin', 'usr/bin']: 224 dirname = os.path.join(path, subdir) 225 if verbose: print " - looking in '%s'" % dirname 226 for fname in glob.glob(dirname + '/*gcc'): 227 if verbose: print " - found '%s'" % fname 228 fnames.append(fname) 229 return fnames 230 231 232 def Scan(self, verbose): 233 """Scan for available toolchains and select the best for each arch. 234 235 We look for all the toolchains we can file, figure out the 236 architecture for each, and whether it works. Then we select the 237 highest priority toolchain for each arch. 238 239 Args: 240 verbose: True to print out progress information 241 """ 242 if verbose: print 'Scanning for tool chains' 243 for path in self.paths: 244 if verbose: print " - scanning path '%s'" % path 245 fnames = self.ScanPath(path, verbose) 246 for fname in fnames: 247 self.Add(fname, True, verbose) 248 249 def List(self): 250 """List out the selected toolchains for each architecture""" 251 print 'List of available toolchains (%d):' % len(self.toolchains) 252 if len(self.toolchains): 253 for key, value in sorted(self.toolchains.iteritems()): 254 print '%-10s: %s' % (key, value.gcc) 255 else: 256 print 'None' 257 258 def Select(self, arch): 259 """Returns the toolchain for a given architecture 260 261 Args: 262 args: Name of architecture (e.g. 'arm', 'ppc_8xx') 263 264 returns: 265 toolchain object, or None if none found 266 """ 267 for tag, value in bsettings.GetItems('toolchain-alias'): 268 if arch == tag: 269 for alias in value.split(): 270 if alias in self.toolchains: 271 return self.toolchains[alias] 272 273 if not arch in self.toolchains: 274 raise ValueError, ("No tool chain found for arch '%s'" % arch) 275 return self.toolchains[arch] 276 277 def ResolveReferences(self, var_dict, args): 278 """Resolve variable references in a string 279 280 This converts ${blah} within the string to the value of blah. 281 This function works recursively. 282 283 Args: 284 var_dict: Dictionary containing variables and their values 285 args: String containing make arguments 286 Returns: 287 Resolved string 288 289 >>> bsettings.Setup() 290 >>> tcs = Toolchains() 291 >>> tcs.Add('fred', False) 292 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ 293 'second' : '2nd'} 294 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') 295 'this=OBLIQUE_set' 296 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') 297 'this=OBLIQUE_setfi2ndrstnd' 298 """ 299 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') 300 301 while True: 302 m = re_var.search(args) 303 if not m: 304 break 305 lookup = m.group(0)[2:-1] 306 value = var_dict.get(lookup, '') 307 args = args[:m.start(0)] + value + args[m.end(0):] 308 return args 309 310 def GetMakeArguments(self, board): 311 """Returns 'make' arguments for a given board 312 313 The flags are in a section called 'make-flags'. Flags are named 314 after the target they represent, for example snapper9260=TESTING=1 315 will pass TESTING=1 to make when building the snapper9260 board. 316 317 References to other boards can be added in the string also. For 318 example: 319 320 [make-flags] 321 at91-boards=ENABLE_AT91_TEST=1 322 snapper9260=${at91-boards} BUILD_TAG=442 323 snapper9g45=${at91-boards} BUILD_TAG=443 324 325 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 326 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. 327 328 A special 'target' variable is set to the board target. 329 330 Args: 331 board: Board object for the board to check. 332 Returns: 333 'make' flags for that board, or '' if none 334 """ 335 self._make_flags['target'] = board.target 336 arg_str = self.ResolveReferences(self._make_flags, 337 self._make_flags.get(board.target, '')) 338 args = arg_str.split(' ') 339 i = 0 340 while i < len(args): 341 if not args[i]: 342 del args[i] 343 else: 344 i += 1 345 return args 346 347 def LocateArchUrl(self, fetch_arch): 348 """Find a toolchain available online 349 350 Look in standard places for available toolchains. At present the 351 only standard place is at kernel.org. 352 353 Args: 354 arch: Architecture to look for, or 'list' for all 355 Returns: 356 If fetch_arch is 'list', a tuple: 357 Machine architecture (e.g. x86_64) 358 List of toolchains 359 else 360 URL containing this toolchain, if avaialble, else None 361 """ 362 arch = command.OutputOneLine('uname', '-m') 363 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' 364 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4'] 365 links = [] 366 for version in versions: 367 url = '%s/%s/%s/' % (base, arch, version) 368 print 'Checking: %s' % url 369 response = urllib2.urlopen(url) 370 html = response.read() 371 parser = MyHTMLParser(fetch_arch) 372 parser.feed(html) 373 if fetch_arch == 'list': 374 links += parser.links 375 elif parser.arch_link: 376 return url + parser.arch_link 377 if fetch_arch == 'list': 378 return arch, links 379 return None 380 381 def Download(self, url): 382 """Download a file to a temporary directory 383 384 Args: 385 url: URL to download 386 Returns: 387 Tuple: 388 Temporary directory name 389 Full path to the downloaded archive file in that directory, 390 or None if there was an error while downloading 391 """ 392 print 'Downloading: %s' % url 393 leaf = url.split('/')[-1] 394 tmpdir = tempfile.mkdtemp('.buildman') 395 response = urllib2.urlopen(url) 396 fname = os.path.join(tmpdir, leaf) 397 fd = open(fname, 'wb') 398 meta = response.info() 399 size = int(meta.getheaders('Content-Length')[0]) 400 done = 0 401 block_size = 1 << 16 402 status = '' 403 404 # Read the file in chunks and show progress as we go 405 while True: 406 buffer = response.read(block_size) 407 if not buffer: 408 print chr(8) * (len(status) + 1), '\r', 409 break 410 411 done += len(buffer) 412 fd.write(buffer) 413 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024, 414 done * 100 / size) 415 status = status + chr(8) * (len(status) + 1) 416 print status, 417 sys.stdout.flush() 418 fd.close() 419 if done != size: 420 print 'Error, failed to download' 421 os.remove(fname) 422 fname = None 423 return tmpdir, fname 424 425 def Unpack(self, fname, dest): 426 """Unpack a tar file 427 428 Args: 429 fname: Filename to unpack 430 dest: Destination directory 431 Returns: 432 Directory name of the first entry in the archive, without the 433 trailing / 434 """ 435 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) 436 return stdout.splitlines()[0][:-1] 437 438 def TestSettingsHasPath(self, path): 439 """Check if builmand will find this toolchain 440 441 Returns: 442 True if the path is in settings, False if not 443 """ 444 paths = self.GetPathList() 445 return path in paths 446 447 def ListArchs(self): 448 """List architectures with available toolchains to download""" 449 host_arch, archives = self.LocateArchUrl('list') 450 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') 451 arch_set = set() 452 for archive in archives: 453 # Remove the host architecture from the start 454 arch = re_arch.match(archive[len(host_arch):]) 455 if arch: 456 arch_set.add(arch.group(1)) 457 return sorted(arch_set) 458 459 def FetchAndInstall(self, arch): 460 """Fetch and install a new toolchain 461 462 arch: 463 Architecture to fetch, or 'list' to list 464 """ 465 # Fist get the URL for this architecture 466 url = self.LocateArchUrl(arch) 467 if not url: 468 print ("Cannot find toolchain for arch '%s' - use 'list' to list" % 469 arch) 470 return 2 471 home = os.environ['HOME'] 472 dest = os.path.join(home, '.buildman-toolchains') 473 if not os.path.exists(dest): 474 os.mkdir(dest) 475 476 # Download the tar file for this toolchain and unpack it 477 tmpdir, tarfile = self.Download(url) 478 if not tarfile: 479 return 1 480 print 'Unpacking to: %s' % dest, 481 sys.stdout.flush() 482 path = self.Unpack(tarfile, dest) 483 os.remove(tarfile) 484 os.rmdir(tmpdir) 485 print 486 487 # Check that the toolchain works 488 print 'Testing' 489 dirpath = os.path.join(dest, path) 490 compiler_fname_list = self.ScanPath(dirpath, True) 491 if not compiler_fname_list: 492 print 'Could not locate C compiler - fetch failed.' 493 return 1 494 if len(compiler_fname_list) != 1: 495 print ('Internal error, ambiguous toolchains: %s' % 496 (', '.join(compiler_fname))) 497 return 1 498 toolchain = Toolchain(compiler_fname_list[0], True, True) 499 500 # Make sure that it will be found by buildman 501 if not self.TestSettingsHasPath(dirpath): 502 print ("Adding 'download' to config file '%s'" % 503 bsettings.config_fname) 504 tools_dir = os.path.dirname(dirpath) 505 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir) 506 return 0 507