1# Copyright (c) 2014 Google, Inc
2#
3# SPDX-License-Identifier:      GPL-2.0+
4#
5
6import errno
7import glob
8import os
9import shutil
10import threading
11
12import command
13import gitutil
14
15def Mkdir(dirname, parents = False):
16    """Make a directory if it doesn't already exist.
17
18    Args:
19        dirname: Directory to create
20    """
21    try:
22        if parents:
23            os.makedirs(dirname)
24        else:
25            os.mkdir(dirname)
26    except OSError as err:
27        if err.errno == errno.EEXIST:
28            pass
29        else:
30            raise
31
32class BuilderJob:
33    """Holds information about a job to be performed by a thread
34
35    Members:
36        board: Board object to build
37        commits: List of commit options to build.
38    """
39    def __init__(self):
40        self.board = None
41        self.commits = []
42
43
44class ResultThread(threading.Thread):
45    """This thread processes results from builder threads.
46
47    It simply passes the results on to the builder. There is only one
48    result thread, and this helps to serialise the build output.
49    """
50    def __init__(self, builder):
51        """Set up a new result thread
52
53        Args:
54            builder: Builder which will be sent each result
55        """
56        threading.Thread.__init__(self)
57        self.builder = builder
58
59    def run(self):
60        """Called to start up the result thread.
61
62        We collect the next result job and pass it on to the build.
63        """
64        while True:
65            result = self.builder.out_queue.get()
66            self.builder.ProcessResult(result)
67            self.builder.out_queue.task_done()
68
69
70class BuilderThread(threading.Thread):
71    """This thread builds U-Boot for a particular board.
72
73    An input queue provides each new job. We run 'make' to build U-Boot
74    and then pass the results on to the output queue.
75
76    Members:
77        builder: The builder which contains information we might need
78        thread_num: Our thread number (0-n-1), used to decide on a
79                temporary directory
80    """
81    def __init__(self, builder, thread_num):
82        """Set up a new builder thread"""
83        threading.Thread.__init__(self)
84        self.builder = builder
85        self.thread_num = thread_num
86
87    def Make(self, commit, brd, stage, cwd, *args, **kwargs):
88        """Run 'make' on a particular commit and board.
89
90        The source code will already be checked out, so the 'commit'
91        argument is only for information.
92
93        Args:
94            commit: Commit object that is being built
95            brd: Board object that is being built
96            stage: Stage of the build. Valid stages are:
97                        mrproper - can be called to clean source
98                        config - called to configure for a board
99                        build - the main make invocation - it does the build
100            args: A list of arguments to pass to 'make'
101            kwargs: A list of keyword arguments to pass to command.RunPipe()
102
103        Returns:
104            CommandResult object
105        """
106        return self.builder.do_make(commit, brd, stage, cwd, *args,
107                **kwargs)
108
109    def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
110                  force_build_failures):
111        """Build a particular commit.
112
113        If the build is already done, and we are not forcing a build, we skip
114        the build and just return the previously-saved results.
115
116        Args:
117            commit_upto: Commit number to build (0...n-1)
118            brd: Board object to build
119            work_dir: Directory to which the source will be checked out
120            do_config: True to run a make <board>_defconfig on the source
121            force_build: Force a build even if one was previously done
122            force_build_failures: Force a bulid if the previous result showed
123                failure
124
125        Returns:
126            tuple containing:
127                - CommandResult object containing the results of the build
128                - boolean indicating whether 'make config' is still needed
129        """
130        # Create a default result - it will be overwritte by the call to
131        # self.Make() below, in the event that we do a build.
132        result = command.CommandResult()
133        result.return_code = 0
134        if self.builder.in_tree:
135            out_dir = work_dir
136        else:
137            out_dir = os.path.join(work_dir, 'build')
138
139        # Check if the job was already completed last time
140        done_file = self.builder.GetDoneFile(commit_upto, brd.target)
141        result.already_done = os.path.exists(done_file)
142        will_build = (force_build or force_build_failures or
143            not result.already_done)
144        if result.already_done:
145            # Get the return code from that build and use it
146            with open(done_file, 'r') as fd:
147                result.return_code = int(fd.readline())
148            if will_build:
149                err_file = self.builder.GetErrFile(commit_upto, brd.target)
150                if os.path.exists(err_file) and os.stat(err_file).st_size:
151                    result.stderr = 'bad'
152                elif not force_build:
153                    # The build passed, so no need to build it again
154                    will_build = False
155
156        if will_build:
157            # We are going to have to build it. First, get a toolchain
158            if not self.toolchain:
159                try:
160                    self.toolchain = self.builder.toolchains.Select(brd.arch)
161                except ValueError as err:
162                    result.return_code = 10
163                    result.stdout = ''
164                    result.stderr = str(err)
165                    # TODO(sjg@chromium.org): This gets swallowed, but needs
166                    # to be reported.
167
168            if self.toolchain:
169                # Checkout the right commit
170                if self.builder.commits:
171                    commit = self.builder.commits[commit_upto]
172                    if self.builder.checkout:
173                        git_dir = os.path.join(work_dir, '.git')
174                        gitutil.Checkout(commit.hash, git_dir, work_dir,
175                                         force=True)
176                else:
177                    commit = 'current'
178
179                # Set up the environment and command line
180                env = self.toolchain.MakeEnvironment(self.builder.full_path)
181                Mkdir(out_dir)
182                args = []
183                cwd = work_dir
184                src_dir = os.path.realpath(work_dir)
185                if not self.builder.in_tree:
186                    if commit_upto is None:
187                        # In this case we are building in the original source
188                        # directory (i.e. the current directory where buildman
189                        # is invoked. The output directory is set to this
190                        # thread's selected work directory.
191                        #
192                        # Symlinks can confuse U-Boot's Makefile since
193                        # we may use '..' in our path, so remove them.
194                        work_dir = os.path.realpath(work_dir)
195                        args.append('O=%s/build' % work_dir)
196                        cwd = None
197                        src_dir = os.getcwd()
198                    else:
199                        args.append('O=build')
200                if not self.builder.verbose_build:
201                    args.append('-s')
202                if self.builder.num_jobs is not None:
203                    args.extend(['-j', str(self.builder.num_jobs)])
204                config_args = ['%s_defconfig' % brd.target]
205                config_out = ''
206                args.extend(self.builder.toolchains.GetMakeArguments(brd))
207
208                # If we need to reconfigure, do that now
209                if do_config:
210                    result = self.Make(commit, brd, 'mrproper', cwd,
211                            'mrproper', *args, env=env)
212                    result = self.Make(commit, brd, 'config', cwd,
213                            *(args + config_args), env=env)
214                    config_out = result.combined
215                    do_config = False   # No need to configure next time
216                if result.return_code == 0:
217                    result = self.Make(commit, brd, 'build', cwd, *args,
218                            env=env)
219                result.stderr = result.stderr.replace(src_dir + '/', '')
220            else:
221                result.return_code = 1
222                result.stderr = 'No tool chain for %s\n' % brd.arch
223            result.already_done = False
224
225        result.toolchain = self.toolchain
226        result.brd = brd
227        result.commit_upto = commit_upto
228        result.out_dir = out_dir
229        return result, do_config
230
231    def _WriteResult(self, result, keep_outputs):
232        """Write a built result to the output directory.
233
234        Args:
235            result: CommandResult object containing result to write
236            keep_outputs: True to store the output binaries, False
237                to delete them
238        """
239        # Fatal error
240        if result.return_code < 0:
241            return
242
243        # Aborted?
244        if result.stderr and 'No child processes' in result.stderr:
245            return
246
247        if result.already_done:
248            return
249
250        # Write the output and stderr
251        output_dir = self.builder._GetOutputDir(result.commit_upto)
252        Mkdir(output_dir)
253        build_dir = self.builder.GetBuildDir(result.commit_upto,
254                result.brd.target)
255        Mkdir(build_dir)
256
257        outfile = os.path.join(build_dir, 'log')
258        with open(outfile, 'w') as fd:
259            if result.stdout:
260                fd.write(result.stdout)
261
262        errfile = self.builder.GetErrFile(result.commit_upto,
263                result.brd.target)
264        if result.stderr:
265            with open(errfile, 'w') as fd:
266                fd.write(result.stderr)
267        elif os.path.exists(errfile):
268            os.remove(errfile)
269
270        if result.toolchain:
271            # Write the build result and toolchain information.
272            done_file = self.builder.GetDoneFile(result.commit_upto,
273                    result.brd.target)
274            with open(done_file, 'w') as fd:
275                fd.write('%s' % result.return_code)
276            with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
277                print >>fd, 'gcc', result.toolchain.gcc
278                print >>fd, 'path', result.toolchain.path
279                print >>fd, 'cross', result.toolchain.cross
280                print >>fd, 'arch', result.toolchain.arch
281                fd.write('%s' % result.return_code)
282
283            with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
284                print >>fd, 'gcc', result.toolchain.gcc
285                print >>fd, 'path', result.toolchain.path
286
287            # Write out the image and function size information and an objdump
288            env = result.toolchain.MakeEnvironment(self.builder.full_path)
289            lines = []
290            for fname in ['u-boot', 'spl/u-boot-spl']:
291                cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
292                nm_result = command.RunPipe([cmd], capture=True,
293                        capture_stderr=True, cwd=result.out_dir,
294                        raise_on_error=False, env=env)
295                if nm_result.stdout:
296                    nm = self.builder.GetFuncSizesFile(result.commit_upto,
297                                    result.brd.target, fname)
298                    with open(nm, 'w') as fd:
299                        print >>fd, nm_result.stdout,
300
301                cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
302                dump_result = command.RunPipe([cmd], capture=True,
303                        capture_stderr=True, cwd=result.out_dir,
304                        raise_on_error=False, env=env)
305                rodata_size = ''
306                if dump_result.stdout:
307                    objdump = self.builder.GetObjdumpFile(result.commit_upto,
308                                    result.brd.target, fname)
309                    with open(objdump, 'w') as fd:
310                        print >>fd, dump_result.stdout,
311                    for line in dump_result.stdout.splitlines():
312                        fields = line.split()
313                        if len(fields) > 5 and fields[1] == '.rodata':
314                            rodata_size = fields[2]
315
316                cmd = ['%ssize' % self.toolchain.cross, fname]
317                size_result = command.RunPipe([cmd], capture=True,
318                        capture_stderr=True, cwd=result.out_dir,
319                        raise_on_error=False, env=env)
320                if size_result.stdout:
321                    lines.append(size_result.stdout.splitlines()[1] + ' ' +
322                                 rodata_size)
323
324            # Write out the image sizes file. This is similar to the output
325            # of binutil's 'size' utility, but it omits the header line and
326            # adds an additional hex value at the end of each line for the
327            # rodata size
328            if len(lines):
329                sizes = self.builder.GetSizesFile(result.commit_upto,
330                                result.brd.target)
331                with open(sizes, 'w') as fd:
332                    print >>fd, '\n'.join(lines)
333
334        # Now write the actual build output
335        if keep_outputs:
336            patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', '*.img',
337                        'include/autoconf.mk', 'spl/u-boot-spl',
338                        'spl/u-boot-spl.bin']
339            for pattern in patterns:
340                file_list = glob.glob(os.path.join(result.out_dir, pattern))
341                for fname in file_list:
342                    shutil.copy(fname, build_dir)
343
344
345    def RunJob(self, job):
346        """Run a single job
347
348        A job consists of a building a list of commits for a particular board.
349
350        Args:
351            job: Job to build
352        """
353        brd = job.board
354        work_dir = self.builder.GetThreadDir(self.thread_num)
355        self.toolchain = None
356        if job.commits:
357            # Run 'make board_defconfig' on the first commit
358            do_config = True
359            commit_upto  = 0
360            force_build = False
361            for commit_upto in range(0, len(job.commits), job.step):
362                result, request_config = self.RunCommit(commit_upto, brd,
363                        work_dir, do_config,
364                        force_build or self.builder.force_build,
365                        self.builder.force_build_failures)
366                failed = result.return_code or result.stderr
367                did_config = do_config
368                if failed and not do_config:
369                    # If our incremental build failed, try building again
370                    # with a reconfig.
371                    if self.builder.force_config_on_failure:
372                        result, request_config = self.RunCommit(commit_upto,
373                            brd, work_dir, True, True, False)
374                        did_config = True
375                if not self.builder.force_reconfig:
376                    do_config = request_config
377
378                # If we built that commit, then config is done. But if we got
379                # an warning, reconfig next time to force it to build the same
380                # files that created warnings this time. Otherwise an
381                # incremental build may not build the same file, and we will
382                # think that the warning has gone away.
383                # We could avoid this by using -Werror everywhere...
384                # For errors, the problem doesn't happen, since presumably
385                # the build stopped and didn't generate output, so will retry
386                # that file next time. So we could detect warnings and deal
387                # with them specially here. For now, we just reconfigure if
388                # anything goes work.
389                # Of course this is substantially slower if there are build
390                # errors/warnings (e.g. 2-3x slower even if only 10% of builds
391                # have problems).
392                if (failed and not result.already_done and not did_config and
393                        self.builder.force_config_on_failure):
394                    # If this build failed, try the next one with a
395                    # reconfigure.
396                    # Sometimes if the board_config.h file changes it can mess
397                    # with dependencies, and we get:
398                    # make: *** No rule to make target `include/autoconf.mk',
399                    #     needed by `depend'.
400                    do_config = True
401                    force_build = True
402                else:
403                    force_build = False
404                    if self.builder.force_config_on_failure:
405                        if failed:
406                            do_config = True
407                    result.commit_upto = commit_upto
408                    if result.return_code < 0:
409                        raise ValueError('Interrupt')
410
411                # We have the build results, so output the result
412                self._WriteResult(result, job.keep_outputs)
413                self.builder.out_queue.put(result)
414        else:
415            # Just build the currently checked-out build
416            result, request_config = self.RunCommit(None, brd, work_dir, True,
417                        True, self.builder.force_build_failures)
418            result.commit_upto = 0
419            self._WriteResult(result, job.keep_outputs)
420            self.builder.out_queue.put(result)
421
422    def run(self):
423        """Our thread's run function
424
425        This thread picks a job from the queue, runs it, and then goes to the
426        next job.
427        """
428        alive = True
429        while True:
430            job = self.builder.queue.get()
431            if self.builder.active and alive:
432                self.RunJob(job)
433            '''
434            try:
435                if self.builder.active and alive:
436                    self.RunJob(job)
437            except Exception as err:
438                alive = False
439                print err
440            '''
441            self.builder.queue.task_done()
442