xref: /openbmc/u-boot/tools/buildman/toolchain.py (revision 95de1e2f) 
1# Copyright (c) 2012 The Chromium OS Authors.
2#
3# SPDX-License-Identifier:	GPL-2.0+
4#
5
6import re
7import glob
8from HTMLParser import HTMLParser
9import os
10import sys
11import tempfile
12import urllib2
13
14import bsettings
15import command
16
17# Simple class to collect links from a page
18class MyHTMLParser(HTMLParser):
19    def __init__(self, arch):
20        """Create a new parser
21
22        After the parser runs, self.links will be set to a list of the links
23        to .xz archives found in the page, and self.arch_link will be set to
24        the one for the given architecture (or None if not found).
25
26        Args:
27            arch: Architecture to search for
28        """
29        HTMLParser.__init__(self)
30        self.arch_link = None
31        self.links = []
32        self._match = '_%s-' % arch
33
34    def handle_starttag(self, tag, attrs):
35        if tag == 'a':
36            for tag, value in attrs:
37                if tag == 'href':
38                    if value and value.endswith('.xz'):
39                        self.links.append(value)
40                        if self._match in value:
41                            self.arch_link = value
42
43
44class Toolchain:
45    """A single toolchain
46
47    Public members:
48        gcc: Full path to C compiler
49        path: Directory path containing C compiler
50        cross: Cross compile string, e.g. 'arm-linux-'
51        arch: Architecture of toolchain as determined from the first
52                component of the filename. E.g. arm-linux-gcc becomes arm
53    """
54    def __init__(self, fname, test, verbose=False):
55        """Create a new toolchain object.
56
57        Args:
58            fname: Filename of the gcc component
59            test: True to run the toolchain to test it
60        """
61        self.gcc = fname
62        self.path = os.path.dirname(fname)
63
64        # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
65        # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
66        basename = os.path.basename(fname)
67        pos = basename.rfind('-')
68        self.cross = basename[:pos + 1] if pos != -1 else ''
69
70        # The architecture is the first part of the name
71        pos = self.cross.find('-')
72        self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
73
74        env = self.MakeEnvironment(False)
75
76        # As a basic sanity check, run the C compiler with --version
77        cmd = [fname, '--version']
78        if test:
79            result = command.RunPipe([cmd], capture=True, env=env,
80                                     raise_on_error=False)
81            self.ok = result.return_code == 0
82            if verbose:
83                print 'Tool chain test: ',
84                if self.ok:
85                    print 'OK'
86                else:
87                    print 'BAD'
88                    print 'Command: ', cmd
89                    print result.stdout
90                    print result.stderr
91        else:
92            self.ok = True
93        self.priority = self.GetPriority(fname)
94
95    def GetPriority(self, fname):
96        """Return the priority of the toolchain.
97
98        Toolchains are ranked according to their suitability by their
99        filename prefix.
100
101        Args:
102            fname: Filename of toolchain
103        Returns:
104            Priority of toolchain, 0=highest, 20=lowest.
105        """
106        priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
107            '-none-linux-gnueabi', '-uclinux', '-none-eabi',
108            '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
109        for prio in range(len(priority_list)):
110            if priority_list[prio] in fname:
111                return prio
112        return prio
113
114    def MakeEnvironment(self, full_path):
115        """Returns an environment for using the toolchain.
116
117        Thie takes the current environment and adds CROSS_COMPILE so that
118        the tool chain will operate correctly.
119
120        Args:
121            full_path: Return the full path in CROSS_COMPILE and don't set
122                PATH
123        """
124        env = dict(os.environ)
125        if full_path:
126            env['CROSS_COMPILE'] = os.path.join(self.path, self.cross)
127        else:
128            env['CROSS_COMPILE'] = self.cross
129            env['PATH'] = self.path + ':' + env['PATH']
130
131        return env
132
133
134class Toolchains:
135    """Manage a list of toolchains for building U-Boot
136
137    We select one toolchain for each architecture type
138
139    Public members:
140        toolchains: Dict of Toolchain objects, keyed by architecture name
141        paths: List of paths to check for toolchains (may contain wildcards)
142    """
143
144    def __init__(self):
145        self.toolchains = {}
146        self.paths = []
147        self._make_flags = dict(bsettings.GetItems('make-flags'))
148
149    def GetPathList(self):
150        """Get a list of available toolchain paths
151
152        Returns:
153            List of strings, each a path to a toolchain mentioned in the
154            [toolchain] section of the settings file.
155        """
156        toolchains = bsettings.GetItems('toolchain')
157        if not toolchains:
158            print ("Warning: No tool chains - please add a [toolchain] section"
159                 " to your buildman config file %s. See README for details" %
160                 bsettings.config_fname)
161
162        paths = []
163        for name, value in toolchains:
164            if '*' in value:
165                paths += glob.glob(value)
166            else:
167                paths.append(value)
168        return paths
169
170    def GetSettings(self):
171      self.paths += self.GetPathList()
172
173    def Add(self, fname, test=True, verbose=False):
174        """Add a toolchain to our list
175
176        We select the given toolchain as our preferred one for its
177        architecture if it is a higher priority than the others.
178
179        Args:
180            fname: Filename of toolchain's gcc driver
181            test: True to run the toolchain to test it
182        """
183        toolchain = Toolchain(fname, test, verbose)
184        add_it = toolchain.ok
185        if toolchain.arch in self.toolchains:
186            add_it = (toolchain.priority <
187                        self.toolchains[toolchain.arch].priority)
188        if add_it:
189            self.toolchains[toolchain.arch] = toolchain
190
191    def ScanPath(self, path, verbose):
192        """Scan a path for a valid toolchain
193
194        Args:
195            path: Path to scan
196            verbose: True to print out progress information
197        Returns:
198            Filename of C compiler if found, else None
199        """
200        fnames = []
201        for subdir in ['.', 'bin', 'usr/bin']:
202            dirname = os.path.join(path, subdir)
203            if verbose: print "      - looking in '%s'" % dirname
204            for fname in glob.glob(dirname + '/*gcc'):
205                if verbose: print "         - found '%s'" % fname
206                fnames.append(fname)
207        return fnames
208
209
210    def Scan(self, verbose):
211        """Scan for available toolchains and select the best for each arch.
212
213        We look for all the toolchains we can file, figure out the
214        architecture for each, and whether it works. Then we select the
215        highest priority toolchain for each arch.
216
217        Args:
218            verbose: True to print out progress information
219        """
220        if verbose: print 'Scanning for tool chains'
221        for path in self.paths:
222            if verbose: print "   - scanning path '%s'" % path
223            fnames = self.ScanPath(path, verbose)
224            for fname in fnames:
225                self.Add(fname, True, verbose)
226
227    def List(self):
228        """List out the selected toolchains for each architecture"""
229        print 'List of available toolchains (%d):' % len(self.toolchains)
230        if len(self.toolchains):
231            for key, value in sorted(self.toolchains.iteritems()):
232                print '%-10s: %s' % (key, value.gcc)
233        else:
234            print 'None'
235
236    def Select(self, arch):
237        """Returns the toolchain for a given architecture
238
239        Args:
240            args: Name of architecture (e.g. 'arm', 'ppc_8xx')
241
242        returns:
243            toolchain object, or None if none found
244        """
245        for tag, value in bsettings.GetItems('toolchain-alias'):
246            if arch == tag:
247                for alias in value.split():
248                    if alias in self.toolchains:
249                        return self.toolchains[alias]
250
251        if not arch in self.toolchains:
252            raise ValueError, ("No tool chain found for arch '%s'" % arch)
253        return self.toolchains[arch]
254
255    def ResolveReferences(self, var_dict, args):
256        """Resolve variable references in a string
257
258        This converts ${blah} within the string to the value of blah.
259        This function works recursively.
260
261        Args:
262            var_dict: Dictionary containing variables and their values
263            args: String containing make arguments
264        Returns:
265            Resolved string
266
267        >>> bsettings.Setup()
268        >>> tcs = Toolchains()
269        >>> tcs.Add('fred', False)
270        >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
271                        'second' : '2nd'}
272        >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
273        'this=OBLIQUE_set'
274        >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
275        'this=OBLIQUE_setfi2ndrstnd'
276        """
277        re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
278
279        while True:
280            m = re_var.search(args)
281            if not m:
282                break
283            lookup = m.group(0)[2:-1]
284            value = var_dict.get(lookup, '')
285            args = args[:m.start(0)] + value + args[m.end(0):]
286        return args
287
288    def GetMakeArguments(self, board):
289        """Returns 'make' arguments for a given board
290
291        The flags are in a section called 'make-flags'. Flags are named
292        after the target they represent, for example snapper9260=TESTING=1
293        will pass TESTING=1 to make when building the snapper9260 board.
294
295        References to other boards can be added in the string also. For
296        example:
297
298        [make-flags]
299        at91-boards=ENABLE_AT91_TEST=1
300        snapper9260=${at91-boards} BUILD_TAG=442
301        snapper9g45=${at91-boards} BUILD_TAG=443
302
303        This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
304        and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
305
306        A special 'target' variable is set to the board target.
307
308        Args:
309            board: Board object for the board to check.
310        Returns:
311            'make' flags for that board, or '' if none
312        """
313        self._make_flags['target'] = board.target
314        arg_str = self.ResolveReferences(self._make_flags,
315                           self._make_flags.get(board.target, ''))
316        args = arg_str.split(' ')
317        i = 0
318        while i < len(args):
319            if not args[i]:
320                del args[i]
321            else:
322                i += 1
323        return args
324
325    def LocateArchUrl(self, fetch_arch):
326        """Find a toolchain available online
327
328        Look in standard places for available toolchains. At present the
329        only standard place is at kernel.org.
330
331        Args:
332            arch: Architecture to look for, or 'list' for all
333        Returns:
334            If fetch_arch is 'list', a tuple:
335                Machine architecture (e.g. x86_64)
336                List of toolchains
337            else
338                URL containing this toolchain, if avaialble, else None
339        """
340        arch = command.OutputOneLine('uname', '-m')
341        base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
342        versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
343        links = []
344        for version in versions:
345            url = '%s/%s/%s/' % (base, arch, version)
346            print 'Checking: %s' % url
347            response = urllib2.urlopen(url)
348            html = response.read()
349            parser = MyHTMLParser(fetch_arch)
350            parser.feed(html)
351            if fetch_arch == 'list':
352                links += parser.links
353            elif parser.arch_link:
354                return url + parser.arch_link
355        if fetch_arch == 'list':
356            return arch, links
357        return None
358
359    def Download(self, url):
360        """Download a file to a temporary directory
361
362        Args:
363            url: URL to download
364        Returns:
365            Tuple:
366                Temporary directory name
367                Full path to the downloaded archive file in that directory,
368                    or None if there was an error while downloading
369        """
370        print "Downloading: %s" % url
371        leaf = url.split('/')[-1]
372        tmpdir = tempfile.mkdtemp('.buildman')
373        response = urllib2.urlopen(url)
374        fname = os.path.join(tmpdir, leaf)
375        fd = open(fname, 'wb')
376        meta = response.info()
377        size = int(meta.getheaders("Content-Length")[0])
378        done = 0
379        block_size = 1 << 16
380        status = ''
381
382        # Read the file in chunks and show progress as we go
383        while True:
384            buffer = response.read(block_size)
385            if not buffer:
386                print chr(8) * (len(status) + 1), '\r',
387                break
388
389            done += len(buffer)
390            fd.write(buffer)
391            status = r"%10d MiB  [%3d%%]" % (done / 1024 / 1024,
392                                             done * 100 / size)
393            status = status + chr(8) * (len(status) + 1)
394            print status,
395            sys.stdout.flush()
396        fd.close()
397        if done != size:
398            print 'Error, failed to download'
399            os.remove(fname)
400            fname = None
401        return tmpdir, fname
402
403    def Unpack(self, fname, dest):
404        """Unpack a tar file
405
406        Args:
407            fname: Filename to unpack
408            dest: Destination directory
409        Returns:
410            Directory name of the first entry in the archive, without the
411            trailing /
412        """
413        stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
414        return stdout.splitlines()[0][:-1]
415
416    def TestSettingsHasPath(self, path):
417        """Check if builmand will find this toolchain
418
419        Returns:
420            True if the path is in settings, False if not
421        """
422        paths = self.GetPathList()
423        return path in paths
424
425    def ListArchs(self):
426        """List architectures with available toolchains to download"""
427        host_arch, archives = self.LocateArchUrl('list')
428        re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
429        arch_set = set()
430        for archive in archives:
431            # Remove the host architecture from the start
432            arch = re_arch.match(archive[len(host_arch):])
433            if arch:
434                arch_set.add(arch.group(1))
435        return sorted(arch_set)
436
437    def FetchAndInstall(self, arch):
438        """Fetch and install a new toolchain
439
440        arch:
441            Architecture to fetch, or 'list' to list
442        """
443        # Fist get the URL for this architecture
444        url = self.LocateArchUrl(arch)
445        if not url:
446            print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
447                   arch)
448            return 2
449        home = os.environ['HOME']
450        dest = os.path.join(home, '.buildman-toolchains')
451        if not os.path.exists(dest):
452            os.mkdir(dest)
453
454        # Download the tar file for this toolchain and unpack it
455        tmpdir, tarfile = self.Download(url)
456        if not tarfile:
457            return 1
458        print 'Unpacking to: %s' % dest,
459        sys.stdout.flush()
460        path = self.Unpack(tarfile, dest)
461        os.remove(tarfile)
462        os.rmdir(tmpdir)
463        print
464
465        # Check that the toolchain works
466        print 'Testing'
467        dirpath = os.path.join(dest, path)
468        compiler_fname_list = self.ScanPath(dirpath, True)
469        if not compiler_fname_list:
470            print 'Could not locate C compiler - fetch failed.'
471            return 1
472        if len(compiler_fname_list) != 1:
473            print ('Internal error, ambiguous toolchains: %s' %
474                   (', '.join(compiler_fname)))
475            return 1
476        toolchain = Toolchain(compiler_fname_list[0], True, True)
477
478        # Make sure that it will be found by buildman
479        if not self.TestSettingsHasPath(dirpath):
480            print ("Adding 'download' to config file '%s'" %
481                   bsettings.config_fname)
482            tools_dir = os.path.dirname(dirpath)
483            bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir)
484        return 0
485