1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2014 Google, Inc
3#
4
5import errno
6import glob
7import os
8import shutil
9import sys
10import threading
11
12import command
13import gitutil
14
15RETURN_CODE_RETRY = -1
16
17def Mkdir(dirname, parents = False):
18    """Make a directory if it doesn't already exist.
19
20    Args:
21        dirname: Directory to create
22    """
23    try:
24        if parents:
25            os.makedirs(dirname)
26        else:
27            os.mkdir(dirname)
28    except OSError as err:
29        if err.errno == errno.EEXIST:
30            if os.path.realpath('.') == os.path.realpath(dirname):
31                print "Cannot create the current working directory '%s'!" % dirname
32                sys.exit(1)
33            pass
34        else:
35            raise
36
37class BuilderJob:
38    """Holds information about a job to be performed by a thread
39
40    Members:
41        board: Board object to build
42        commits: List of commit options to build.
43    """
44    def __init__(self):
45        self.board = None
46        self.commits = []
47
48
49class ResultThread(threading.Thread):
50    """This thread processes results from builder threads.
51
52    It simply passes the results on to the builder. There is only one
53    result thread, and this helps to serialise the build output.
54    """
55    def __init__(self, builder):
56        """Set up a new result thread
57
58        Args:
59            builder: Builder which will be sent each result
60        """
61        threading.Thread.__init__(self)
62        self.builder = builder
63
64    def run(self):
65        """Called to start up the result thread.
66
67        We collect the next result job and pass it on to the build.
68        """
69        while True:
70            result = self.builder.out_queue.get()
71            self.builder.ProcessResult(result)
72            self.builder.out_queue.task_done()
73
74
75class BuilderThread(threading.Thread):
76    """This thread builds U-Boot for a particular board.
77
78    An input queue provides each new job. We run 'make' to build U-Boot
79    and then pass the results on to the output queue.
80
81    Members:
82        builder: The builder which contains information we might need
83        thread_num: Our thread number (0-n-1), used to decide on a
84                temporary directory
85    """
86    def __init__(self, builder, thread_num, incremental, per_board_out_dir):
87        """Set up a new builder thread"""
88        threading.Thread.__init__(self)
89        self.builder = builder
90        self.thread_num = thread_num
91        self.incremental = incremental
92        self.per_board_out_dir = per_board_out_dir
93
94    def Make(self, commit, brd, stage, cwd, *args, **kwargs):
95        """Run 'make' on a particular commit and board.
96
97        The source code will already be checked out, so the 'commit'
98        argument is only for information.
99
100        Args:
101            commit: Commit object that is being built
102            brd: Board object that is being built
103            stage: Stage of the build. Valid stages are:
104                        mrproper - can be called to clean source
105                        config - called to configure for a board
106                        build - the main make invocation - it does the build
107            args: A list of arguments to pass to 'make'
108            kwargs: A list of keyword arguments to pass to command.RunPipe()
109
110        Returns:
111            CommandResult object
112        """
113        return self.builder.do_make(commit, brd, stage, cwd, *args,
114                **kwargs)
115
116    def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
117                  force_build, force_build_failures):
118        """Build a particular commit.
119
120        If the build is already done, and we are not forcing a build, we skip
121        the build and just return the previously-saved results.
122
123        Args:
124            commit_upto: Commit number to build (0...n-1)
125            brd: Board object to build
126            work_dir: Directory to which the source will be checked out
127            do_config: True to run a make <board>_defconfig on the source
128            config_only: Only configure the source, do not build it
129            force_build: Force a build even if one was previously done
130            force_build_failures: Force a bulid if the previous result showed
131                failure
132
133        Returns:
134            tuple containing:
135                - CommandResult object containing the results of the build
136                - boolean indicating whether 'make config' is still needed
137        """
138        # Create a default result - it will be overwritte by the call to
139        # self.Make() below, in the event that we do a build.
140        result = command.CommandResult()
141        result.return_code = 0
142        if self.builder.in_tree:
143            out_dir = work_dir
144        else:
145            if self.per_board_out_dir:
146                out_rel_dir = os.path.join('..', brd.target)
147            else:
148                out_rel_dir = 'build'
149            out_dir = os.path.join(work_dir, out_rel_dir)
150
151        # Check if the job was already completed last time
152        done_file = self.builder.GetDoneFile(commit_upto, brd.target)
153        result.already_done = os.path.exists(done_file)
154        will_build = (force_build or force_build_failures or
155            not result.already_done)
156        if result.already_done:
157            # Get the return code from that build and use it
158            with open(done_file, 'r') as fd:
159                try:
160                    result.return_code = int(fd.readline())
161                except ValueError:
162                    # The file may be empty due to running out of disk space.
163                    # Try a rebuild
164                    result.return_code = RETURN_CODE_RETRY
165
166            # Check the signal that the build needs to be retried
167            if result.return_code == RETURN_CODE_RETRY:
168                will_build = True
169            elif will_build:
170                err_file = self.builder.GetErrFile(commit_upto, brd.target)
171                if os.path.exists(err_file) and os.stat(err_file).st_size:
172                    result.stderr = 'bad'
173                elif not force_build:
174                    # The build passed, so no need to build it again
175                    will_build = False
176
177        if will_build:
178            # We are going to have to build it. First, get a toolchain
179            if not self.toolchain:
180                try:
181                    self.toolchain = self.builder.toolchains.Select(brd.arch)
182                except ValueError as err:
183                    result.return_code = 10
184                    result.stdout = ''
185                    result.stderr = str(err)
186                    # TODO(sjg@chromium.org): This gets swallowed, but needs
187                    # to be reported.
188
189            if self.toolchain:
190                # Checkout the right commit
191                if self.builder.commits:
192                    commit = self.builder.commits[commit_upto]
193                    if self.builder.checkout:
194                        git_dir = os.path.join(work_dir, '.git')
195                        gitutil.Checkout(commit.hash, git_dir, work_dir,
196                                         force=True)
197                else:
198                    commit = 'current'
199
200                # Set up the environment and command line
201                env = self.toolchain.MakeEnvironment(self.builder.full_path)
202                Mkdir(out_dir)
203                args = []
204                cwd = work_dir
205                src_dir = os.path.realpath(work_dir)
206                if not self.builder.in_tree:
207                    if commit_upto is None:
208                        # In this case we are building in the original source
209                        # directory (i.e. the current directory where buildman
210                        # is invoked. The output directory is set to this
211                        # thread's selected work directory.
212                        #
213                        # Symlinks can confuse U-Boot's Makefile since
214                        # we may use '..' in our path, so remove them.
215                        out_dir = os.path.realpath(out_dir)
216                        args.append('O=%s' % out_dir)
217                        cwd = None
218                        src_dir = os.getcwd()
219                    else:
220                        args.append('O=%s' % out_rel_dir)
221                if self.builder.verbose_build:
222                    args.append('V=1')
223                else:
224                    args.append('-s')
225                if self.builder.num_jobs is not None:
226                    args.extend(['-j', str(self.builder.num_jobs)])
227                if self.builder.warnings_as_errors:
228                    args.append('KCFLAGS=-Werror')
229                config_args = ['%s_defconfig' % brd.target]
230                config_out = ''
231                args.extend(self.builder.toolchains.GetMakeArguments(brd))
232                args.extend(self.toolchain.MakeArgs())
233
234                # If we need to reconfigure, do that now
235                if do_config:
236                    config_out = ''
237                    if not self.incremental:
238                        result = self.Make(commit, brd, 'mrproper', cwd,
239                                'mrproper', *args, env=env)
240                        config_out += result.combined
241                    result = self.Make(commit, brd, 'config', cwd,
242                            *(args + config_args), env=env)
243                    config_out += result.combined
244                    do_config = False   # No need to configure next time
245                if result.return_code == 0:
246                    if config_only:
247                        args.append('cfg')
248                    result = self.Make(commit, brd, 'build', cwd, *args,
249                            env=env)
250                result.stderr = result.stderr.replace(src_dir + '/', '')
251                if self.builder.verbose_build:
252                    result.stdout = config_out + result.stdout
253            else:
254                result.return_code = 1
255                result.stderr = 'No tool chain for %s\n' % brd.arch
256            result.already_done = False
257
258        result.toolchain = self.toolchain
259        result.brd = brd
260        result.commit_upto = commit_upto
261        result.out_dir = out_dir
262        return result, do_config
263
264    def _WriteResult(self, result, keep_outputs):
265        """Write a built result to the output directory.
266
267        Args:
268            result: CommandResult object containing result to write
269            keep_outputs: True to store the output binaries, False
270                to delete them
271        """
272        # Fatal error
273        if result.return_code < 0:
274            return
275
276        # If we think this might have been aborted with Ctrl-C, record the
277        # failure but not that we are 'done' with this board. A retry may fix
278        # it.
279        maybe_aborted =  result.stderr and 'No child processes' in result.stderr
280
281        if result.already_done:
282            return
283
284        # Write the output and stderr
285        output_dir = self.builder._GetOutputDir(result.commit_upto)
286        Mkdir(output_dir)
287        build_dir = self.builder.GetBuildDir(result.commit_upto,
288                result.brd.target)
289        Mkdir(build_dir)
290
291        outfile = os.path.join(build_dir, 'log')
292        with open(outfile, 'w') as fd:
293            if result.stdout:
294                # We don't want unicode characters in log files
295                fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
296
297        errfile = self.builder.GetErrFile(result.commit_upto,
298                result.brd.target)
299        if result.stderr:
300            with open(errfile, 'w') as fd:
301                # We don't want unicode characters in log files
302                fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
303        elif os.path.exists(errfile):
304            os.remove(errfile)
305
306        if result.toolchain:
307            # Write the build result and toolchain information.
308            done_file = self.builder.GetDoneFile(result.commit_upto,
309                    result.brd.target)
310            with open(done_file, 'w') as fd:
311                if maybe_aborted:
312                    # Special code to indicate we need to retry
313                    fd.write('%s' % RETURN_CODE_RETRY)
314                else:
315                    fd.write('%s' % result.return_code)
316            with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
317                print >>fd, 'gcc', result.toolchain.gcc
318                print >>fd, 'path', result.toolchain.path
319                print >>fd, 'cross', result.toolchain.cross
320                print >>fd, 'arch', result.toolchain.arch
321                fd.write('%s' % result.return_code)
322
323            # Write out the image and function size information and an objdump
324            env = result.toolchain.MakeEnvironment(self.builder.full_path)
325            with open(os.path.join(build_dir, 'env'), 'w') as fd:
326                for var in sorted(env.keys()):
327                    print >>fd, '%s="%s"' % (var, env[var])
328            lines = []
329            for fname in ['u-boot', 'spl/u-boot-spl']:
330                cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
331                nm_result = command.RunPipe([cmd], capture=True,
332                        capture_stderr=True, cwd=result.out_dir,
333                        raise_on_error=False, env=env)
334                if nm_result.stdout:
335                    nm = self.builder.GetFuncSizesFile(result.commit_upto,
336                                    result.brd.target, fname)
337                    with open(nm, 'w') as fd:
338                        print >>fd, nm_result.stdout,
339
340                cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
341                dump_result = command.RunPipe([cmd], capture=True,
342                        capture_stderr=True, cwd=result.out_dir,
343                        raise_on_error=False, env=env)
344                rodata_size = ''
345                if dump_result.stdout:
346                    objdump = self.builder.GetObjdumpFile(result.commit_upto,
347                                    result.brd.target, fname)
348                    with open(objdump, 'w') as fd:
349                        print >>fd, dump_result.stdout,
350                    for line in dump_result.stdout.splitlines():
351                        fields = line.split()
352                        if len(fields) > 5 and fields[1] == '.rodata':
353                            rodata_size = fields[2]
354
355                cmd = ['%ssize' % self.toolchain.cross, fname]
356                size_result = command.RunPipe([cmd], capture=True,
357                        capture_stderr=True, cwd=result.out_dir,
358                        raise_on_error=False, env=env)
359                if size_result.stdout:
360                    lines.append(size_result.stdout.splitlines()[1] + ' ' +
361                                 rodata_size)
362
363            # Extract the environment from U-Boot and dump it out
364            cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary',
365                   '-j', '.rodata.default_environment',
366                   'env/built-in.o', 'uboot.env']
367            command.RunPipe([cmd], capture=True,
368                            capture_stderr=True, cwd=result.out_dir,
369                            raise_on_error=False, env=env)
370            ubootenv = os.path.join(result.out_dir, 'uboot.env')
371            self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
372
373            # Write out the image sizes file. This is similar to the output
374            # of binutil's 'size' utility, but it omits the header line and
375            # adds an additional hex value at the end of each line for the
376            # rodata size
377            if len(lines):
378                sizes = self.builder.GetSizesFile(result.commit_upto,
379                                result.brd.target)
380                with open(sizes, 'w') as fd:
381                    print >>fd, '\n'.join(lines)
382
383        # Write out the configuration files, with a special case for SPL
384        for dirname in ['', 'spl', 'tpl']:
385            self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
386                'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
387                'include/autoconf.mk', 'include/generated/autoconf.h'])
388
389        # Now write the actual build output
390        if keep_outputs:
391            self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
392                '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
393                'spl/u-boot-spl*'])
394
395    def CopyFiles(self, out_dir, build_dir, dirname, patterns):
396        """Copy files from the build directory to the output.
397
398        Args:
399            out_dir: Path to output directory containing the files
400            build_dir: Place to copy the files
401            dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
402            patterns: A list of filenames (strings) to copy, each relative
403               to the build directory
404        """
405        for pattern in patterns:
406            file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
407            for fname in file_list:
408                target = os.path.basename(fname)
409                if dirname:
410                    base, ext = os.path.splitext(target)
411                    if ext:
412                        target = '%s-%s%s' % (base, dirname, ext)
413                shutil.copy(fname, os.path.join(build_dir, target))
414
415    def RunJob(self, job):
416        """Run a single job
417
418        A job consists of a building a list of commits for a particular board.
419
420        Args:
421            job: Job to build
422        """
423        brd = job.board
424        work_dir = self.builder.GetThreadDir(self.thread_num)
425        self.toolchain = None
426        if job.commits:
427            # Run 'make board_defconfig' on the first commit
428            do_config = True
429            commit_upto  = 0
430            force_build = False
431            for commit_upto in range(0, len(job.commits), job.step):
432                result, request_config = self.RunCommit(commit_upto, brd,
433                        work_dir, do_config, self.builder.config_only,
434                        force_build or self.builder.force_build,
435                        self.builder.force_build_failures)
436                failed = result.return_code or result.stderr
437                did_config = do_config
438                if failed and not do_config:
439                    # If our incremental build failed, try building again
440                    # with a reconfig.
441                    if self.builder.force_config_on_failure:
442                        result, request_config = self.RunCommit(commit_upto,
443                            brd, work_dir, True, False, True, False)
444                        did_config = True
445                if not self.builder.force_reconfig:
446                    do_config = request_config
447
448                # If we built that commit, then config is done. But if we got
449                # an warning, reconfig next time to force it to build the same
450                # files that created warnings this time. Otherwise an
451                # incremental build may not build the same file, and we will
452                # think that the warning has gone away.
453                # We could avoid this by using -Werror everywhere...
454                # For errors, the problem doesn't happen, since presumably
455                # the build stopped and didn't generate output, so will retry
456                # that file next time. So we could detect warnings and deal
457                # with them specially here. For now, we just reconfigure if
458                # anything goes work.
459                # Of course this is substantially slower if there are build
460                # errors/warnings (e.g. 2-3x slower even if only 10% of builds
461                # have problems).
462                if (failed and not result.already_done and not did_config and
463                        self.builder.force_config_on_failure):
464                    # If this build failed, try the next one with a
465                    # reconfigure.
466                    # Sometimes if the board_config.h file changes it can mess
467                    # with dependencies, and we get:
468                    # make: *** No rule to make target `include/autoconf.mk',
469                    #     needed by `depend'.
470                    do_config = True
471                    force_build = True
472                else:
473                    force_build = False
474                    if self.builder.force_config_on_failure:
475                        if failed:
476                            do_config = True
477                    result.commit_upto = commit_upto
478                    if result.return_code < 0:
479                        raise ValueError('Interrupt')
480
481                # We have the build results, so output the result
482                self._WriteResult(result, job.keep_outputs)
483                self.builder.out_queue.put(result)
484        else:
485            # Just build the currently checked-out build
486            result, request_config = self.RunCommit(None, brd, work_dir, True,
487                        self.builder.config_only, True,
488                        self.builder.force_build_failures)
489            result.commit_upto = 0
490            self._WriteResult(result, job.keep_outputs)
491            self.builder.out_queue.put(result)
492
493    def run(self):
494        """Our thread's run function
495
496        This thread picks a job from the queue, runs it, and then goes to the
497        next job.
498        """
499        while True:
500            job = self.builder.queue.get()
501            self.RunJob(job)
502            self.builder.queue.task_done()
503