1# Copyright (c) 2014 Google, Inc
2#
3# SPDX-License-Identifier:      GPL-2.0+
4#
5
6import errno
7import glob
8import os
9import shutil
10import threading
11
12import command
13import gitutil
14
15def Mkdir(dirname):
16    """Make a directory if it doesn't already exist.
17
18    Args:
19        dirname: Directory to create
20    """
21    try:
22        os.mkdir(dirname)
23    except OSError as err:
24        if err.errno == errno.EEXIST:
25            pass
26        else:
27            raise
28
29class BuilderJob:
30    """Holds information about a job to be performed by a thread
31
32    Members:
33        board: Board object to build
34        commits: List of commit options to build.
35    """
36    def __init__(self):
37        self.board = None
38        self.commits = []
39
40
41class ResultThread(threading.Thread):
42    """This thread processes results from builder threads.
43
44    It simply passes the results on to the builder. There is only one
45    result thread, and this helps to serialise the build output.
46    """
47    def __init__(self, builder):
48        """Set up a new result thread
49
50        Args:
51            builder: Builder which will be sent each result
52        """
53        threading.Thread.__init__(self)
54        self.builder = builder
55
56    def run(self):
57        """Called to start up the result thread.
58
59        We collect the next result job and pass it on to the build.
60        """
61        while True:
62            result = self.builder.out_queue.get()
63            self.builder.ProcessResult(result)
64            self.builder.out_queue.task_done()
65
66
67class BuilderThread(threading.Thread):
68    """This thread builds U-Boot for a particular board.
69
70    An input queue provides each new job. We run 'make' to build U-Boot
71    and then pass the results on to the output queue.
72
73    Members:
74        builder: The builder which contains information we might need
75        thread_num: Our thread number (0-n-1), used to decide on a
76                temporary directory
77    """
78    def __init__(self, builder, thread_num):
79        """Set up a new builder thread"""
80        threading.Thread.__init__(self)
81        self.builder = builder
82        self.thread_num = thread_num
83
84    def Make(self, commit, brd, stage, cwd, *args, **kwargs):
85        """Run 'make' on a particular commit and board.
86
87        The source code will already be checked out, so the 'commit'
88        argument is only for information.
89
90        Args:
91            commit: Commit object that is being built
92            brd: Board object that is being built
93            stage: Stage of the build. Valid stages are:
94                        mrproper - can be called to clean source
95                        config - called to configure for a board
96                        build - the main make invocation - it does the build
97            args: A list of arguments to pass to 'make'
98            kwargs: A list of keyword arguments to pass to command.RunPipe()
99
100        Returns:
101            CommandResult object
102        """
103        return self.builder.do_make(commit, brd, stage, cwd, *args,
104                **kwargs)
105
106    def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
107                  force_build_failures):
108        """Build a particular commit.
109
110        If the build is already done, and we are not forcing a build, we skip
111        the build and just return the previously-saved results.
112
113        Args:
114            commit_upto: Commit number to build (0...n-1)
115            brd: Board object to build
116            work_dir: Directory to which the source will be checked out
117            do_config: True to run a make <board>_defconfig on the source
118            force_build: Force a build even if one was previously done
119            force_build_failures: Force a bulid if the previous result showed
120                failure
121
122        Returns:
123            tuple containing:
124                - CommandResult object containing the results of the build
125                - boolean indicating whether 'make config' is still needed
126        """
127        # Create a default result - it will be overwritte by the call to
128        # self.Make() below, in the event that we do a build.
129        result = command.CommandResult()
130        result.return_code = 0
131        if self.builder.in_tree:
132            out_dir = work_dir
133        else:
134            out_dir = os.path.join(work_dir, 'build')
135
136        # Check if the job was already completed last time
137        done_file = self.builder.GetDoneFile(commit_upto, brd.target)
138        result.already_done = os.path.exists(done_file)
139        will_build = (force_build or force_build_failures or
140            not result.already_done)
141        if result.already_done and will_build:
142            # Get the return code from that build and use it
143            with open(done_file, 'r') as fd:
144                result.return_code = int(fd.readline())
145            err_file = self.builder.GetErrFile(commit_upto, brd.target)
146            if os.path.exists(err_file) and os.stat(err_file).st_size:
147                result.stderr = 'bad'
148            elif not force_build:
149                # The build passed, so no need to build it again
150                will_build = False
151
152        if will_build:
153            # We are going to have to build it. First, get a toolchain
154            if not self.toolchain:
155                try:
156                    self.toolchain = self.builder.toolchains.Select(brd.arch)
157                except ValueError as err:
158                    result.return_code = 10
159                    result.stdout = ''
160                    result.stderr = str(err)
161                    # TODO(sjg@chromium.org): This gets swallowed, but needs
162                    # to be reported.
163
164            if self.toolchain:
165                # Checkout the right commit
166                if self.builder.commits:
167                    commit = self.builder.commits[commit_upto]
168                    if self.builder.checkout:
169                        git_dir = os.path.join(work_dir, '.git')
170                        gitutil.Checkout(commit.hash, git_dir, work_dir,
171                                         force=True)
172                else:
173                    commit = 'current'
174
175                # Set up the environment and command line
176                env = self.toolchain.MakeEnvironment()
177                Mkdir(out_dir)
178                args = []
179                cwd = work_dir
180                if not self.builder.in_tree:
181                    if commit_upto is None:
182                        # In this case we are building in the original source
183                        # directory (i.e. the current directory where buildman
184                        # is invoked. The output directory is set to this
185                        # thread's selected work directory.
186                        #
187                        # Symlinks can confuse U-Boot's Makefile since
188                        # we may use '..' in our path, so remove them.
189                        work_dir = os.path.realpath(work_dir)
190                        args.append('O=%s/build' % work_dir)
191                        cwd = None
192                    else:
193                        args.append('O=build')
194                args.append('-s')
195                if self.builder.num_jobs is not None:
196                    args.extend(['-j', str(self.builder.num_jobs)])
197                config_args = ['%s_defconfig' % brd.target]
198                config_out = ''
199                args.extend(self.builder.toolchains.GetMakeArguments(brd))
200
201                # If we need to reconfigure, do that now
202                if do_config:
203                    result = self.Make(commit, brd, 'mrproper', cwd,
204                            'mrproper', *args, env=env)
205                    result = self.Make(commit, brd, 'config', cwd,
206                            *(args + config_args), env=env)
207                    config_out = result.combined
208                    do_config = False   # No need to configure next time
209                if result.return_code == 0:
210                    result = self.Make(commit, brd, 'build', cwd, *args,
211                            env=env)
212                    result.stdout = config_out + result.stdout
213            else:
214                result.return_code = 1
215                result.stderr = 'No tool chain for %s\n' % brd.arch
216            result.already_done = False
217
218        result.toolchain = self.toolchain
219        result.brd = brd
220        result.commit_upto = commit_upto
221        result.out_dir = out_dir
222        return result, do_config
223
224    def _WriteResult(self, result, keep_outputs):
225        """Write a built result to the output directory.
226
227        Args:
228            result: CommandResult object containing result to write
229            keep_outputs: True to store the output binaries, False
230                to delete them
231        """
232        # Fatal error
233        if result.return_code < 0:
234            return
235
236        # Aborted?
237        if result.stderr and 'No child processes' in result.stderr:
238            return
239
240        if result.already_done:
241            return
242
243        # Write the output and stderr
244        output_dir = self.builder._GetOutputDir(result.commit_upto)
245        Mkdir(output_dir)
246        build_dir = self.builder.GetBuildDir(result.commit_upto,
247                result.brd.target)
248        Mkdir(build_dir)
249
250        outfile = os.path.join(build_dir, 'log')
251        with open(outfile, 'w') as fd:
252            if result.stdout:
253                fd.write(result.stdout)
254
255        errfile = self.builder.GetErrFile(result.commit_upto,
256                result.brd.target)
257        if result.stderr:
258            with open(errfile, 'w') as fd:
259                fd.write(result.stderr)
260        elif os.path.exists(errfile):
261            os.remove(errfile)
262
263        if result.toolchain:
264            # Write the build result and toolchain information.
265            done_file = self.builder.GetDoneFile(result.commit_upto,
266                    result.brd.target)
267            with open(done_file, 'w') as fd:
268                fd.write('%s' % result.return_code)
269            with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
270                print >>fd, 'gcc', result.toolchain.gcc
271                print >>fd, 'path', result.toolchain.path
272                print >>fd, 'cross', result.toolchain.cross
273                print >>fd, 'arch', result.toolchain.arch
274                fd.write('%s' % result.return_code)
275
276            with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
277                print >>fd, 'gcc', result.toolchain.gcc
278                print >>fd, 'path', result.toolchain.path
279
280            # Write out the image and function size information and an objdump
281            env = result.toolchain.MakeEnvironment()
282            lines = []
283            for fname in ['u-boot', 'spl/u-boot-spl']:
284                cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
285                nm_result = command.RunPipe([cmd], capture=True,
286                        capture_stderr=True, cwd=result.out_dir,
287                        raise_on_error=False, env=env)
288                if nm_result.stdout:
289                    nm = self.builder.GetFuncSizesFile(result.commit_upto,
290                                    result.brd.target, fname)
291                    with open(nm, 'w') as fd:
292                        print >>fd, nm_result.stdout,
293
294                cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
295                dump_result = command.RunPipe([cmd], capture=True,
296                        capture_stderr=True, cwd=result.out_dir,
297                        raise_on_error=False, env=env)
298                rodata_size = ''
299                if dump_result.stdout:
300                    objdump = self.builder.GetObjdumpFile(result.commit_upto,
301                                    result.brd.target, fname)
302                    with open(objdump, 'w') as fd:
303                        print >>fd, dump_result.stdout,
304                    for line in dump_result.stdout.splitlines():
305                        fields = line.split()
306                        if len(fields) > 5 and fields[1] == '.rodata':
307                            rodata_size = fields[2]
308
309                cmd = ['%ssize' % self.toolchain.cross, fname]
310                size_result = command.RunPipe([cmd], capture=True,
311                        capture_stderr=True, cwd=result.out_dir,
312                        raise_on_error=False, env=env)
313                if size_result.stdout:
314                    lines.append(size_result.stdout.splitlines()[1] + ' ' +
315                                 rodata_size)
316
317            # Write out the image sizes file. This is similar to the output
318            # of binutil's 'size' utility, but it omits the header line and
319            # adds an additional hex value at the end of each line for the
320            # rodata size
321            if len(lines):
322                sizes = self.builder.GetSizesFile(result.commit_upto,
323                                result.brd.target)
324                with open(sizes, 'w') as fd:
325                    print >>fd, '\n'.join(lines)
326
327        # Now write the actual build output
328        if keep_outputs:
329            patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map',
330                        'include/autoconf.mk', 'spl/u-boot-spl',
331                        'spl/u-boot-spl.bin']
332            for pattern in patterns:
333                file_list = glob.glob(os.path.join(result.out_dir, pattern))
334                for fname in file_list:
335                    shutil.copy(fname, build_dir)
336
337
338    def RunJob(self, job):
339        """Run a single job
340
341        A job consists of a building a list of commits for a particular board.
342
343        Args:
344            job: Job to build
345        """
346        brd = job.board
347        work_dir = self.builder.GetThreadDir(self.thread_num)
348        self.toolchain = None
349        if job.commits:
350            # Run 'make board_defconfig' on the first commit
351            do_config = True
352            commit_upto  = 0
353            force_build = False
354            for commit_upto in range(0, len(job.commits), job.step):
355                result, request_config = self.RunCommit(commit_upto, brd,
356                        work_dir, do_config,
357                        force_build or self.builder.force_build,
358                        self.builder.force_build_failures)
359                failed = result.return_code or result.stderr
360                did_config = do_config
361                if failed and not do_config:
362                    # If our incremental build failed, try building again
363                    # with a reconfig.
364                    if self.builder.force_config_on_failure:
365                        result, request_config = self.RunCommit(commit_upto,
366                            brd, work_dir, True, True, False)
367                        did_config = True
368                if not self.builder.force_reconfig:
369                    do_config = request_config
370
371                # If we built that commit, then config is done. But if we got
372                # an warning, reconfig next time to force it to build the same
373                # files that created warnings this time. Otherwise an
374                # incremental build may not build the same file, and we will
375                # think that the warning has gone away.
376                # We could avoid this by using -Werror everywhere...
377                # For errors, the problem doesn't happen, since presumably
378                # the build stopped and didn't generate output, so will retry
379                # that file next time. So we could detect warnings and deal
380                # with them specially here. For now, we just reconfigure if
381                # anything goes work.
382                # Of course this is substantially slower if there are build
383                # errors/warnings (e.g. 2-3x slower even if only 10% of builds
384                # have problems).
385                if (failed and not result.already_done and not did_config and
386                        self.builder.force_config_on_failure):
387                    # If this build failed, try the next one with a
388                    # reconfigure.
389                    # Sometimes if the board_config.h file changes it can mess
390                    # with dependencies, and we get:
391                    # make: *** No rule to make target `include/autoconf.mk',
392                    #     needed by `depend'.
393                    do_config = True
394                    force_build = True
395                else:
396                    force_build = False
397                    if self.builder.force_config_on_failure:
398                        if failed:
399                            do_config = True
400                    result.commit_upto = commit_upto
401                    if result.return_code < 0:
402                        raise ValueError('Interrupt')
403
404                # We have the build results, so output the result
405                self._WriteResult(result, job.keep_outputs)
406                self.builder.out_queue.put(result)
407        else:
408            # Just build the currently checked-out build
409            result, request_config = self.RunCommit(None, brd, work_dir, True,
410                        True, self.builder.force_build_failures)
411            result.commit_upto = 0
412            self._WriteResult(result, job.keep_outputs)
413            self.builder.out_queue.put(result)
414
415    def run(self):
416        """Our thread's run function
417
418        This thread picks a job from the queue, runs it, and then goes to the
419        next job.
420        """
421        alive = True
422        while True:
423            job = self.builder.queue.get()
424            if self.builder.active and alive:
425                self.RunJob(job)
426            '''
427            try:
428                if self.builder.active and alive:
429                    self.RunJob(job)
430            except Exception as err:
431                alive = False
432                print err
433            '''
434            self.builder.queue.task_done()
435