xref: /openbmc/u-boot/tools/buildman/toolchain.py (revision d928664f) 
1# Copyright (c) 2012 The Chromium OS Authors.
2#
3# SPDX-License-Identifier:	GPL-2.0+
4#
5
6import re
7import glob
8from HTMLParser import HTMLParser
9import os
10import sys
11import tempfile
12import urllib2
13
14import bsettings
15import command
16
17# Simple class to collect links from a page
18class MyHTMLParser(HTMLParser):
19    def __init__(self, arch):
20        """Create a new parser
21
22        After the parser runs, self.links will be set to a list of the links
23        to .xz archives found in the page, and self.arch_link will be set to
24        the one for the given architecture (or None if not found).
25
26        Args:
27            arch: Architecture to search for
28        """
29        HTMLParser.__init__(self)
30        self.arch_link = None
31        self.links = []
32        self._match = '_%s-' % arch
33
34    def handle_starttag(self, tag, attrs):
35        if tag == 'a':
36            for tag, value in attrs:
37                if tag == 'href':
38                    if value and value.endswith('.xz'):
39                        self.links.append(value)
40                        if self._match in value:
41                            self.arch_link = value
42
43
44class Toolchain:
45    """A single toolchain
46
47    Public members:
48        gcc: Full path to C compiler
49        path: Directory path containing C compiler
50        cross: Cross compile string, e.g. 'arm-linux-'
51        arch: Architecture of toolchain as determined from the first
52                component of the filename. E.g. arm-linux-gcc becomes arm
53    """
54    def __init__(self, fname, test, verbose=False):
55        """Create a new toolchain object.
56
57        Args:
58            fname: Filename of the gcc component
59            test: True to run the toolchain to test it
60        """
61        self.gcc = fname
62        self.path = os.path.dirname(fname)
63
64        # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
65        # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
66        basename = os.path.basename(fname)
67        pos = basename.rfind('-')
68        self.cross = basename[:pos + 1] if pos != -1 else ''
69
70        # The architecture is the first part of the name
71        pos = self.cross.find('-')
72        self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
73
74        env = self.MakeEnvironment(False)
75
76        # As a basic sanity check, run the C compiler with --version
77        cmd = [fname, '--version']
78        if test:
79            result = command.RunPipe([cmd], capture=True, env=env,
80                                     raise_on_error=False)
81            self.ok = result.return_code == 0
82            if verbose:
83                print 'Tool chain test: ',
84                if self.ok:
85                    print 'OK'
86                else:
87                    print 'BAD'
88                    print 'Command: ', cmd
89                    print result.stdout
90                    print result.stderr
91        else:
92            self.ok = True
93        self.priority = self.GetPriority(fname)
94
95    def GetPriority(self, fname):
96        """Return the priority of the toolchain.
97
98        Toolchains are ranked according to their suitability by their
99        filename prefix.
100
101        Args:
102            fname: Filename of toolchain
103        Returns:
104            Priority of toolchain, 0=highest, 20=lowest.
105        """
106        priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
107            '-none-linux-gnueabi', '-uclinux', '-none-eabi',
108            '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
109        for prio in range(len(priority_list)):
110            if priority_list[prio] in fname:
111                return prio
112        return prio
113
114    def MakeEnvironment(self, full_path):
115        """Returns an environment for using the toolchain.
116
117        Thie takes the current environment and adds CROSS_COMPILE so that
118        the tool chain will operate correctly.
119
120        Args:
121            full_path: Return the full path in CROSS_COMPILE and don't set
122                PATH
123        """
124        env = dict(os.environ)
125        if full_path:
126            env['CROSS_COMPILE'] = os.path.join(self.path, self.cross)
127        else:
128            env['CROSS_COMPILE'] = self.cross
129            env['PATH'] = self.path + ':' + env['PATH']
130
131        return env
132
133
134class Toolchains:
135    """Manage a list of toolchains for building U-Boot
136
137    We select one toolchain for each architecture type
138
139    Public members:
140        toolchains: Dict of Toolchain objects, keyed by architecture name
141        paths: List of paths to check for toolchains (may contain wildcards)
142    """
143
144    def __init__(self):
145        self.toolchains = {}
146        self.paths = []
147        self._make_flags = dict(bsettings.GetItems('make-flags'))
148
149    def GetPathList(self):
150        """Get a list of available toolchain paths
151
152        Returns:
153            List of strings, each a path to a toolchain mentioned in the
154            [toolchain] section of the settings file.
155        """
156        toolchains = bsettings.GetItems('toolchain')
157        if not toolchains:
158            print ("Warning: No tool chains - please add a [toolchain] section"
159                 " to your buildman config file %s. See README for details" %
160                 bsettings.config_fname)
161
162        paths = []
163        for name, value in toolchains:
164            if '*' in value:
165                paths += glob.glob(value)
166            else:
167                paths.append(value)
168        return paths
169
170    def GetSettings(self):
171      self.paths += self.GetPathList()
172
173    def Add(self, fname, test=True, verbose=False):
174        """Add a toolchain to our list
175
176        We select the given toolchain as our preferred one for its
177        architecture if it is a higher priority than the others.
178
179        Args:
180            fname: Filename of toolchain's gcc driver
181            test: True to run the toolchain to test it
182        """
183        toolchain = Toolchain(fname, test, verbose)
184        add_it = toolchain.ok
185        if toolchain.arch in self.toolchains:
186            add_it = (toolchain.priority <
187                        self.toolchains[toolchain.arch].priority)
188        if add_it:
189            self.toolchains[toolchain.arch] = toolchain
190
191    def ScanPath(self, path, verbose):
192        """Scan a path for a valid toolchain
193
194        Args:
195            path: Path to scan
196            verbose: True to print out progress information
197        Returns:
198            Filename of C compiler if found, else None
199        """
200        for subdir in ['.', 'bin', 'usr/bin']:
201            dirname = os.path.join(path, subdir)
202            if verbose: print "      - looking in '%s'" % dirname
203            for fname in glob.glob(dirname + '/*gcc'):
204                if verbose: print "         - found '%s'" % fname
205                return fname
206        return None
207
208
209    def Scan(self, verbose):
210        """Scan for available toolchains and select the best for each arch.
211
212        We look for all the toolchains we can file, figure out the
213        architecture for each, and whether it works. Then we select the
214        highest priority toolchain for each arch.
215
216        Args:
217            verbose: True to print out progress information
218        """
219        if verbose: print 'Scanning for tool chains'
220        for path in self.paths:
221            if verbose: print "   - scanning path '%s'" % path
222            fname = self.ScanPath(path, verbose)
223            if fname:
224                self.Add(fname, True, verbose)
225
226    def List(self):
227        """List out the selected toolchains for each architecture"""
228        print 'List of available toolchains (%d):' % len(self.toolchains)
229        if len(self.toolchains):
230            for key, value in sorted(self.toolchains.iteritems()):
231                print '%-10s: %s' % (key, value.gcc)
232        else:
233            print 'None'
234
235    def Select(self, arch):
236        """Returns the toolchain for a given architecture
237
238        Args:
239            args: Name of architecture (e.g. 'arm', 'ppc_8xx')
240
241        returns:
242            toolchain object, or None if none found
243        """
244        for tag, value in bsettings.GetItems('toolchain-alias'):
245            if arch == tag:
246                for alias in value.split():
247                    if alias in self.toolchains:
248                        return self.toolchains[alias]
249
250        if not arch in self.toolchains:
251            raise ValueError, ("No tool chain found for arch '%s'" % arch)
252        return self.toolchains[arch]
253
254    def ResolveReferences(self, var_dict, args):
255        """Resolve variable references in a string
256
257        This converts ${blah} within the string to the value of blah.
258        This function works recursively.
259
260        Args:
261            var_dict: Dictionary containing variables and their values
262            args: String containing make arguments
263        Returns:
264            Resolved string
265
266        >>> bsettings.Setup()
267        >>> tcs = Toolchains()
268        >>> tcs.Add('fred', False)
269        >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
270                        'second' : '2nd'}
271        >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
272        'this=OBLIQUE_set'
273        >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
274        'this=OBLIQUE_setfi2ndrstnd'
275        """
276        re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
277
278        while True:
279            m = re_var.search(args)
280            if not m:
281                break
282            lookup = m.group(0)[2:-1]
283            value = var_dict.get(lookup, '')
284            args = args[:m.start(0)] + value + args[m.end(0):]
285        return args
286
287    def GetMakeArguments(self, board):
288        """Returns 'make' arguments for a given board
289
290        The flags are in a section called 'make-flags'. Flags are named
291        after the target they represent, for example snapper9260=TESTING=1
292        will pass TESTING=1 to make when building the snapper9260 board.
293
294        References to other boards can be added in the string also. For
295        example:
296
297        [make-flags]
298        at91-boards=ENABLE_AT91_TEST=1
299        snapper9260=${at91-boards} BUILD_TAG=442
300        snapper9g45=${at91-boards} BUILD_TAG=443
301
302        This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
303        and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
304
305        A special 'target' variable is set to the board target.
306
307        Args:
308            board: Board object for the board to check.
309        Returns:
310            'make' flags for that board, or '' if none
311        """
312        self._make_flags['target'] = board.target
313        arg_str = self.ResolveReferences(self._make_flags,
314                           self._make_flags.get(board.target, ''))
315        args = arg_str.split(' ')
316        i = 0
317        while i < len(args):
318            if not args[i]:
319                del args[i]
320            else:
321                i += 1
322        return args
323
324    def LocateArchUrl(self, fetch_arch):
325        """Find a toolchain available online
326
327        Look in standard places for available toolchains. At present the
328        only standard place is at kernel.org.
329
330        Args:
331            arch: Architecture to look for, or 'list' for all
332        Returns:
333            If fetch_arch is 'list', a tuple:
334                Machine architecture (e.g. x86_64)
335                List of toolchains
336            else
337                URL containing this toolchain, if avaialble, else None
338        """
339        arch = command.OutputOneLine('uname', '-m')
340        base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
341        versions = ['4.6.3', '4.6.2', '4.5.1', '4.2.4']
342        links = []
343        for version in versions:
344            url = '%s/%s/%s/' % (base, arch, version)
345            print 'Checking: %s' % url
346            response = urllib2.urlopen(url)
347            html = response.read()
348            parser = MyHTMLParser(fetch_arch)
349            parser.feed(html)
350            if fetch_arch == 'list':
351                links += parser.links
352            elif parser.arch_link:
353                return url + parser.arch_link
354        if fetch_arch == 'list':
355            return arch, links
356        return None
357
358    def Download(self, url):
359        """Download a file to a temporary directory
360
361        Args:
362            url: URL to download
363        Returns:
364            Tuple:
365                Temporary directory name
366                Full path to the downloaded archive file in that directory,
367                    or None if there was an error while downloading
368        """
369        print "Downloading: %s" % url
370        leaf = url.split('/')[-1]
371        tmpdir = tempfile.mkdtemp('.buildman')
372        response = urllib2.urlopen(url)
373        fname = os.path.join(tmpdir, leaf)
374        fd = open(fname, 'wb')
375        meta = response.info()
376        size = int(meta.getheaders("Content-Length")[0])
377        done = 0
378        block_size = 1 << 16
379        status = ''
380
381        # Read the file in chunks and show progress as we go
382        while True:
383            buffer = response.read(block_size)
384            if not buffer:
385                print chr(8) * (len(status) + 1), '\r',
386                break
387
388            done += len(buffer)
389            fd.write(buffer)
390            status = r"%10d MiB  [%3d%%]" % (done / 1024 / 1024,
391                                             done * 100 / size)
392            status = status + chr(8) * (len(status) + 1)
393            print status,
394            sys.stdout.flush()
395        fd.close()
396        if done != size:
397            print 'Error, failed to download'
398            os.remove(fname)
399            fname = None
400        return tmpdir, fname
401
402    def Unpack(self, fname, dest):
403        """Unpack a tar file
404
405        Args:
406            fname: Filename to unpack
407            dest: Destination directory
408        Returns:
409            Directory name of the first entry in the archive, without the
410            trailing /
411        """
412        stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
413        return stdout.splitlines()[0][:-1]
414
415    def TestSettingsHasPath(self, path):
416        """Check if builmand will find this toolchain
417
418        Returns:
419            True if the path is in settings, False if not
420        """
421        paths = self.GetPathList()
422        return path in paths
423
424    def ListArchs(self):
425        """List architectures with available toolchains to download"""
426        host_arch, archives = self.LocateArchUrl('list')
427        re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
428        arch_set = set()
429        for archive in archives:
430            # Remove the host architecture from the start
431            arch = re_arch.match(archive[len(host_arch):])
432            if arch:
433                arch_set.add(arch.group(1))
434        return sorted(arch_set)
435
436    def FetchAndInstall(self, arch):
437        """Fetch and install a new toolchain
438
439        arch:
440            Architecture to fetch, or 'list' to list
441        """
442        # Fist get the URL for this architecture
443        url = self.LocateArchUrl(arch)
444        if not url:
445            print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
446                   arch)
447            return 2
448        home = os.environ['HOME']
449        dest = os.path.join(home, '.buildman-toolchains')
450        if not os.path.exists(dest):
451            os.mkdir(dest)
452
453        # Download the tar file for this toolchain and unpack it
454        tmpdir, tarfile = self.Download(url)
455        if not tarfile:
456            return 1
457        print 'Unpacking to: %s' % dest,
458        sys.stdout.flush()
459        path = self.Unpack(tarfile, dest)
460        os.remove(tarfile)
461        os.rmdir(tmpdir)
462        print
463
464        # Check that the toolchain works
465        print 'Testing'
466        dirpath = os.path.join(dest, path)
467        compiler_fname = self.ScanPath(dirpath, True)
468        if not compiler_fname:
469            print 'Could not locate C compiler - fetch failed.'
470            return 1
471        toolchain = Toolchain(compiler_fname, True, True)
472
473        # Make sure that it will be found by buildman
474        if not self.TestSettingsHasPath(dirpath):
475            print ("Adding 'download' to config file '%s'" %
476                   bsettings.config_fname)
477            tools_dir = os.path.dirname(dirpath)
478            bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir)
479        return 0
480