1# Copyright (c) 2012 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import re 7import glob 8from HTMLParser import HTMLParser 9import os 10import sys 11import tempfile 12import urllib2 13 14import bsettings 15import command 16 17# Simple class to collect links from a page 18class MyHTMLParser(HTMLParser): 19 def __init__(self, arch): 20 """Create a new parser 21 22 After the parser runs, self.links will be set to a list of the links 23 to .xz archives found in the page, and self.arch_link will be set to 24 the one for the given architecture (or None if not found). 25 26 Args: 27 arch: Architecture to search for 28 """ 29 HTMLParser.__init__(self) 30 self.arch_link = None 31 self.links = [] 32 self._match = '_%s-' % arch 33 34 def handle_starttag(self, tag, attrs): 35 if tag == 'a': 36 for tag, value in attrs: 37 if tag == 'href': 38 if value and value.endswith('.xz'): 39 self.links.append(value) 40 if self._match in value: 41 self.arch_link = value 42 43 44class Toolchain: 45 """A single toolchain 46 47 Public members: 48 gcc: Full path to C compiler 49 path: Directory path containing C compiler 50 cross: Cross compile string, e.g. 'arm-linux-' 51 arch: Architecture of toolchain as determined from the first 52 component of the filename. E.g. arm-linux-gcc becomes arm 53 """ 54 def __init__(self, fname, test, verbose=False): 55 """Create a new toolchain object. 56 57 Args: 58 fname: Filename of the gcc component 59 test: True to run the toolchain to test it 60 """ 61 self.gcc = fname 62 self.path = os.path.dirname(fname) 63 64 # Find the CROSS_COMPILE prefix to use for U-Boot. For example, 65 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. 66 basename = os.path.basename(fname) 67 pos = basename.rfind('-') 68 self.cross = basename[:pos + 1] if pos != -1 else '' 69 70 # The architecture is the first part of the name 71 pos = self.cross.find('-') 72 self.arch = self.cross[:pos] if pos != -1 else 'sandbox' 73 74 env = self.MakeEnvironment(False) 75 76 # As a basic sanity check, run the C compiler with --version 77 cmd = [fname, '--version'] 78 if test: 79 result = command.RunPipe([cmd], capture=True, env=env, 80 raise_on_error=False) 81 self.ok = result.return_code == 0 82 if verbose: 83 print 'Tool chain test: ', 84 if self.ok: 85 print 'OK' 86 else: 87 print 'BAD' 88 print 'Command: ', cmd 89 print result.stdout 90 print result.stderr 91 else: 92 self.ok = True 93 self.priority = self.GetPriority(fname) 94 95 def GetPriority(self, fname): 96 """Return the priority of the toolchain. 97 98 Toolchains are ranked according to their suitability by their 99 filename prefix. 100 101 Args: 102 fname: Filename of toolchain 103 Returns: 104 Priority of toolchain, 0=highest, 20=lowest. 105 """ 106 priority_list = ['-elf', '-unknown-linux-gnu', '-linux', 107 '-none-linux-gnueabi', '-uclinux', '-none-eabi', 108 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux'] 109 for prio in range(len(priority_list)): 110 if priority_list[prio] in fname: 111 return prio 112 return prio 113 114 def MakeEnvironment(self, full_path): 115 """Returns an environment for using the toolchain. 116 117 Thie takes the current environment and adds CROSS_COMPILE so that 118 the tool chain will operate correctly. 119 120 Args: 121 full_path: Return the full path in CROSS_COMPILE and don't set 122 PATH 123 """ 124 env = dict(os.environ) 125 if full_path: 126 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross) 127 else: 128 env['CROSS_COMPILE'] = self.cross 129 env['PATH'] = self.path + ':' + env['PATH'] 130 131 return env 132 133 134class Toolchains: 135 """Manage a list of toolchains for building U-Boot 136 137 We select one toolchain for each architecture type 138 139 Public members: 140 toolchains: Dict of Toolchain objects, keyed by architecture name 141 paths: List of paths to check for toolchains (may contain wildcards) 142 """ 143 144 def __init__(self): 145 self.toolchains = {} 146 self.paths = [] 147 self._make_flags = dict(bsettings.GetItems('make-flags')) 148 149 def GetPathList(self): 150 """Get a list of available toolchain paths 151 152 Returns: 153 List of strings, each a path to a toolchain mentioned in the 154 [toolchain] section of the settings file. 155 """ 156 toolchains = bsettings.GetItems('toolchain') 157 if not toolchains: 158 print ("Warning: No tool chains - please add a [toolchain] section" 159 " to your buildman config file %s. See README for details" % 160 bsettings.config_fname) 161 162 paths = [] 163 for name, value in toolchains: 164 if '*' in value: 165 paths += glob.glob(value) 166 else: 167 paths.append(value) 168 return paths 169 170 def GetSettings(self): 171 self.paths += self.GetPathList() 172 173 def Add(self, fname, test=True, verbose=False): 174 """Add a toolchain to our list 175 176 We select the given toolchain as our preferred one for its 177 architecture if it is a higher priority than the others. 178 179 Args: 180 fname: Filename of toolchain's gcc driver 181 test: True to run the toolchain to test it 182 """ 183 toolchain = Toolchain(fname, test, verbose) 184 add_it = toolchain.ok 185 if toolchain.arch in self.toolchains: 186 add_it = (toolchain.priority < 187 self.toolchains[toolchain.arch].priority) 188 if add_it: 189 self.toolchains[toolchain.arch] = toolchain 190 191 def ScanPath(self, path, verbose): 192 """Scan a path for a valid toolchain 193 194 Args: 195 path: Path to scan 196 verbose: True to print out progress information 197 Returns: 198 Filename of C compiler if found, else None 199 """ 200 fnames = [] 201 for subdir in ['.', 'bin', 'usr/bin']: 202 dirname = os.path.join(path, subdir) 203 if verbose: print " - looking in '%s'" % dirname 204 for fname in glob.glob(dirname + '/*gcc'): 205 if verbose: print " - found '%s'" % fname 206 fnames.append(fname) 207 return fnames 208 209 210 def Scan(self, verbose): 211 """Scan for available toolchains and select the best for each arch. 212 213 We look for all the toolchains we can file, figure out the 214 architecture for each, and whether it works. Then we select the 215 highest priority toolchain for each arch. 216 217 Args: 218 verbose: True to print out progress information 219 """ 220 if verbose: print 'Scanning for tool chains' 221 for path in self.paths: 222 if verbose: print " - scanning path '%s'" % path 223 fnames = self.ScanPath(path, verbose) 224 for fname in fnames: 225 self.Add(fname, True, verbose) 226 227 def List(self): 228 """List out the selected toolchains for each architecture""" 229 print 'List of available toolchains (%d):' % len(self.toolchains) 230 if len(self.toolchains): 231 for key, value in sorted(self.toolchains.iteritems()): 232 print '%-10s: %s' % (key, value.gcc) 233 else: 234 print 'None' 235 236 def Select(self, arch): 237 """Returns the toolchain for a given architecture 238 239 Args: 240 args: Name of architecture (e.g. 'arm', 'ppc_8xx') 241 242 returns: 243 toolchain object, or None if none found 244 """ 245 for tag, value in bsettings.GetItems('toolchain-alias'): 246 if arch == tag: 247 for alias in value.split(): 248 if alias in self.toolchains: 249 return self.toolchains[alias] 250 251 if not arch in self.toolchains: 252 raise ValueError, ("No tool chain found for arch '%s'" % arch) 253 return self.toolchains[arch] 254 255 def ResolveReferences(self, var_dict, args): 256 """Resolve variable references in a string 257 258 This converts ${blah} within the string to the value of blah. 259 This function works recursively. 260 261 Args: 262 var_dict: Dictionary containing variables and their values 263 args: String containing make arguments 264 Returns: 265 Resolved string 266 267 >>> bsettings.Setup() 268 >>> tcs = Toolchains() 269 >>> tcs.Add('fred', False) 270 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ 271 'second' : '2nd'} 272 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') 273 'this=OBLIQUE_set' 274 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') 275 'this=OBLIQUE_setfi2ndrstnd' 276 """ 277 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') 278 279 while True: 280 m = re_var.search(args) 281 if not m: 282 break 283 lookup = m.group(0)[2:-1] 284 value = var_dict.get(lookup, '') 285 args = args[:m.start(0)] + value + args[m.end(0):] 286 return args 287 288 def GetMakeArguments(self, board): 289 """Returns 'make' arguments for a given board 290 291 The flags are in a section called 'make-flags'. Flags are named 292 after the target they represent, for example snapper9260=TESTING=1 293 will pass TESTING=1 to make when building the snapper9260 board. 294 295 References to other boards can be added in the string also. For 296 example: 297 298 [make-flags] 299 at91-boards=ENABLE_AT91_TEST=1 300 snapper9260=${at91-boards} BUILD_TAG=442 301 snapper9g45=${at91-boards} BUILD_TAG=443 302 303 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 304 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. 305 306 A special 'target' variable is set to the board target. 307 308 Args: 309 board: Board object for the board to check. 310 Returns: 311 'make' flags for that board, or '' if none 312 """ 313 self._make_flags['target'] = board.target 314 arg_str = self.ResolveReferences(self._make_flags, 315 self._make_flags.get(board.target, '')) 316 args = arg_str.split(' ') 317 i = 0 318 while i < len(args): 319 if not args[i]: 320 del args[i] 321 else: 322 i += 1 323 return args 324 325 def LocateArchUrl(self, fetch_arch): 326 """Find a toolchain available online 327 328 Look in standard places for available toolchains. At present the 329 only standard place is at kernel.org. 330 331 Args: 332 arch: Architecture to look for, or 'list' for all 333 Returns: 334 If fetch_arch is 'list', a tuple: 335 Machine architecture (e.g. x86_64) 336 List of toolchains 337 else 338 URL containing this toolchain, if avaialble, else None 339 """ 340 arch = command.OutputOneLine('uname', '-m') 341 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' 342 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4'] 343 links = [] 344 for version in versions: 345 url = '%s/%s/%s/' % (base, arch, version) 346 print 'Checking: %s' % url 347 response = urllib2.urlopen(url) 348 html = response.read() 349 parser = MyHTMLParser(fetch_arch) 350 parser.feed(html) 351 if fetch_arch == 'list': 352 links += parser.links 353 elif parser.arch_link: 354 return url + parser.arch_link 355 if fetch_arch == 'list': 356 return arch, links 357 return None 358 359 def Download(self, url): 360 """Download a file to a temporary directory 361 362 Args: 363 url: URL to download 364 Returns: 365 Tuple: 366 Temporary directory name 367 Full path to the downloaded archive file in that directory, 368 or None if there was an error while downloading 369 """ 370 print "Downloading: %s" % url 371 leaf = url.split('/')[-1] 372 tmpdir = tempfile.mkdtemp('.buildman') 373 response = urllib2.urlopen(url) 374 fname = os.path.join(tmpdir, leaf) 375 fd = open(fname, 'wb') 376 meta = response.info() 377 size = int(meta.getheaders("Content-Length")[0]) 378 done = 0 379 block_size = 1 << 16 380 status = '' 381 382 # Read the file in chunks and show progress as we go 383 while True: 384 buffer = response.read(block_size) 385 if not buffer: 386 print chr(8) * (len(status) + 1), '\r', 387 break 388 389 done += len(buffer) 390 fd.write(buffer) 391 status = r"%10d MiB [%3d%%]" % (done / 1024 / 1024, 392 done * 100 / size) 393 status = status + chr(8) * (len(status) + 1) 394 print status, 395 sys.stdout.flush() 396 fd.close() 397 if done != size: 398 print 'Error, failed to download' 399 os.remove(fname) 400 fname = None 401 return tmpdir, fname 402 403 def Unpack(self, fname, dest): 404 """Unpack a tar file 405 406 Args: 407 fname: Filename to unpack 408 dest: Destination directory 409 Returns: 410 Directory name of the first entry in the archive, without the 411 trailing / 412 """ 413 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) 414 return stdout.splitlines()[0][:-1] 415 416 def TestSettingsHasPath(self, path): 417 """Check if builmand will find this toolchain 418 419 Returns: 420 True if the path is in settings, False if not 421 """ 422 paths = self.GetPathList() 423 return path in paths 424 425 def ListArchs(self): 426 """List architectures with available toolchains to download""" 427 host_arch, archives = self.LocateArchUrl('list') 428 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') 429 arch_set = set() 430 for archive in archives: 431 # Remove the host architecture from the start 432 arch = re_arch.match(archive[len(host_arch):]) 433 if arch: 434 arch_set.add(arch.group(1)) 435 return sorted(arch_set) 436 437 def FetchAndInstall(self, arch): 438 """Fetch and install a new toolchain 439 440 arch: 441 Architecture to fetch, or 'list' to list 442 """ 443 # Fist get the URL for this architecture 444 url = self.LocateArchUrl(arch) 445 if not url: 446 print ("Cannot find toolchain for arch '%s' - use 'list' to list" % 447 arch) 448 return 2 449 home = os.environ['HOME'] 450 dest = os.path.join(home, '.buildman-toolchains') 451 if not os.path.exists(dest): 452 os.mkdir(dest) 453 454 # Download the tar file for this toolchain and unpack it 455 tmpdir, tarfile = self.Download(url) 456 if not tarfile: 457 return 1 458 print 'Unpacking to: %s' % dest, 459 sys.stdout.flush() 460 path = self.Unpack(tarfile, dest) 461 os.remove(tarfile) 462 os.rmdir(tmpdir) 463 print 464 465 # Check that the toolchain works 466 print 'Testing' 467 dirpath = os.path.join(dest, path) 468 compiler_fname_list = self.ScanPath(dirpath, True) 469 if not compiler_fname_list: 470 print 'Could not locate C compiler - fetch failed.' 471 return 1 472 if len(compiler_fname_list) != 1: 473 print ('Internal error, ambiguous toolchains: %s' % 474 (', '.join(compiler_fname))) 475 return 1 476 toolchain = Toolchain(compiler_fname_list[0], True, True) 477 478 # Make sure that it will be found by buildman 479 if not self.TestSettingsHasPath(dirpath): 480 print ("Adding 'download' to config file '%s'" % 481 bsettings.config_fname) 482 tools_dir = os.path.dirname(dirpath) 483 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir) 484 return 0 485