xref: /openbmc/u-boot/tools/buildman/func_test.py (revision f7582ce8496f19edf845d1f62c4b7f385e4cd864) 
1#
2# Copyright (c) 2014 Google, Inc
3#
4# SPDX-License-Identifier:      GPL-2.0+
5#
6
7import os
8import shutil
9import sys
10import tempfile
11import unittest
12
13import board
14import bsettings
15import cmdline
16import command
17import control
18import gitutil
19import terminal
20import toolchain
21
22settings_data = '''
23# Buildman settings file
24
25[toolchain]
26
27[toolchain-alias]
28
29[make-flags]
30src=/home/sjg/c/src
31chroot=/home/sjg/c/chroot
32vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
33chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
34chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
35chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
36'''
37
38boards = [
39    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
40    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
41    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
42    ['Active', 'powerpc', 'mpc5xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
43    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
44]
45
46commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4739403bb patman: Use --no-pager' to stop git from forking a pager
48db6e6f2 patman: Remove the -a option
49f2ccf03 patman: Correct unit tests to run correctly
501d097f9 patman: Fix indentation in terminal.py
51d073747 patman: Support the 'reverse' option for 'git log
52"""
53
54commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
55Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
56Date:   Fri Aug 22 19:12:41 2014 +0900
57
58    buildman: refactor help message
59
60    "buildman [options]" is displayed by default.
61
62    Append the rest of help messages to parser.usage
63    instead of replacing it.
64
65    Besides, "-b <branch>" is not mandatory since commit fea5858e.
66    Drop it from the usage.
67
68    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
69""",
70"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
71Author: Simon Glass <sjg@chromium.org>
72Date:   Thu Aug 14 16:48:25 2014 -0600
73
74    patman: Support the 'reverse' option for 'git log'
75
76    This option is currently not supported, but needs to be, for buildman to
77    operate as expected.
78
79    Series-changes: 7
80    - Add new patch to fix the 'reverse' bug
81
82
83    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
84    Reported-by: York Sun <yorksun@freescale.com>
85    Signed-off-by: Simon Glass <sjg@chromium.org>
86
87""",
88"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
89Author: Simon Glass <sjg@chromium.org>
90Date:   Sat Aug 9 11:44:32 2014 -0600
91
92    patman: Fix indentation in terminal.py
93
94    This code came from a different project with 2-character indentation. Fix
95    it for U-Boot.
96
97    Series-changes: 6
98    - Add new patch to fix indentation in teminal.py
99
100    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
101    Signed-off-by: Simon Glass <sjg@chromium.org>
102
103""",
104"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
105Author: Simon Glass <sjg@chromium.org>
106Date:   Sat Aug 9 11:08:24 2014 -0600
107
108    patman: Correct unit tests to run correctly
109
110    It seems that doctest behaves differently now, and some of the unit tests
111    do not run. Adjust the tests to work correctly.
112
113     ./tools/patman/patman --test
114    <unittest.result.TestResult run=10 errors=0 failures=0>
115
116    Series-changes: 6
117    - Add new patch to fix patman unit tests
118
119    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
120
121""",
122"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
123Author: Simon Glass <sjg@chromium.org>
124Date:   Sat Aug 9 12:06:02 2014 -0600
125
126    patman: Remove the -a option
127
128    It seems that this is no longer needed, since checkpatch.pl will catch
129    whitespace problems in patches. Also the option is not widely used, so
130    it seems safe to just remove it.
131
132    Series-changes: 6
133    - Add new patch to remove patman's -a option
134
135    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
136    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
137
138""",
139"""commit 39403bb4f838153028a6f21ca30bf100f3791133
140Author: Simon Glass <sjg@chromium.org>
141Date:   Thu Aug 14 21:50:52 2014 -0600
142
143    patman: Use --no-pager' to stop git from forking a pager
144
145""",
146"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
147Author: Simon Glass <sjg@chromium.org>
148Date:   Fri Aug 22 15:57:39 2014 -0600
149
150    patman: Avoid changing the order of tags
151
152    patman collects tags that it sees in the commit and places them nicely
153    sorted at the end of the patch. However, this is not really necessary and
154    in fact is apparently not desirable.
155
156    Series-changes: 9
157    - Add new patch to avoid changing the order of tags
158
159    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
160    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
161"""]
162
163TEST_BRANCH = '__testbranch'
164
165class TestFunctional(unittest.TestCase):
166    """Functional test for buildman.
167
168    This aims to test from just below the invocation of buildman (parsing
169    of arguments) to 'make' and 'git' invocation. It is not a true
170    emd-to-end test, as it mocks git, make and the tool chain. But this
171    makes it easier to detect when the builder is doing the wrong thing,
172    since in many cases this test code will fail. For example, only a
173    very limited subset of 'git' arguments is supported - anything
174    unexpected will fail.
175    """
176    def setUp(self):
177        self._base_dir = tempfile.mkdtemp()
178        self._git_dir = os.path.join(self._base_dir, 'src')
179        self._buildman_pathname = sys.argv[0]
180        self._buildman_dir = os.path.dirname(sys.argv[0])
181        command.test_result = self._HandleCommand
182        self.setupToolchains()
183        self._toolchains.Add('arm-gcc', test=False)
184        self._toolchains.Add('powerpc-gcc', test=False)
185        bsettings.Setup(None)
186        bsettings.AddFile(settings_data)
187        self._boards = board.Boards()
188        for brd in boards:
189            self._boards.AddBoard(board.Board(*brd))
190
191        # Directories where the source been cloned
192        self._clone_dirs = []
193        self._commits = len(commit_shortlog.splitlines()) + 1
194        self._total_builds = self._commits * len(boards)
195
196        # Number of calls to make
197        self._make_calls = 0
198
199        # Map of [board, commit] to error messages
200        self._error = {}
201
202        self._test_branch = TEST_BRANCH
203
204        # Avoid sending any output and clear all terminal output
205        terminal.SetPrintTestMode()
206        terminal.GetPrintTestLines()
207
208    def tearDown(self):
209        shutil.rmtree(self._base_dir)
210
211    def setupToolchains(self):
212        self._toolchains = toolchain.Toolchains()
213        self._toolchains.Add('gcc', test=False)
214
215    def _RunBuildman(self, *args):
216        return command.RunPipe([[self._buildman_pathname] + list(args)],
217                capture=True, capture_stderr=True)
218
219    def _RunControl(self, *args, **kwargs):
220        sys.argv = [sys.argv[0]] + list(args)
221        options, args = cmdline.ParseArgs()
222        result = control.DoBuildman(options, args, toolchains=self._toolchains,
223                make_func=self._HandleMake, boards=self._boards,
224                clean_dir=kwargs.get('clean_dir', True))
225        self._builder = control.builder
226        return result
227
228    def testFullHelp(self):
229        command.test_result = None
230        result = self._RunBuildman('-H')
231        help_file = os.path.join(self._buildman_dir, 'README')
232        self.assertEqual(len(result.stdout), os.path.getsize(help_file))
233        self.assertEqual(0, len(result.stderr))
234        self.assertEqual(0, result.return_code)
235
236    def testHelp(self):
237        command.test_result = None
238        result = self._RunBuildman('-h')
239        help_file = os.path.join(self._buildman_dir, 'README')
240        self.assertTrue(len(result.stdout) > 1000)
241        self.assertEqual(0, len(result.stderr))
242        self.assertEqual(0, result.return_code)
243
244    def testGitSetup(self):
245        """Test gitutils.Setup(), from outside the module itself"""
246        command.test_result = command.CommandResult(return_code=1)
247        gitutil.Setup()
248        self.assertEqual(gitutil.use_no_decorate, False)
249
250        command.test_result = command.CommandResult(return_code=0)
251        gitutil.Setup()
252        self.assertEqual(gitutil.use_no_decorate, True)
253
254    def _HandleCommandGitLog(self, args):
255        if '-n0' in args:
256            return command.CommandResult(return_code=0)
257        elif args[-1] == 'upstream/master..%s' % self._test_branch:
258            return command.CommandResult(return_code=0, stdout=commit_shortlog)
259        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
260            if args[-1] == self._test_branch:
261                count = int(args[3][2:])
262                return command.CommandResult(return_code=0,
263                                            stdout=''.join(commit_log[:count]))
264
265        # Not handled, so abort
266        print 'git log', args
267        sys.exit(1)
268
269    def _HandleCommandGitConfig(self, args):
270        config = args[0]
271        if config == 'sendemail.aliasesfile':
272            return command.CommandResult(return_code=0)
273        elif config.startswith('branch.badbranch'):
274            return command.CommandResult(return_code=1)
275        elif config == 'branch.%s.remote' % self._test_branch:
276            return command.CommandResult(return_code=0, stdout='upstream\n')
277        elif config == 'branch.%s.merge' % self._test_branch:
278            return command.CommandResult(return_code=0,
279                                         stdout='refs/heads/master\n')
280
281        # Not handled, so abort
282        print 'git config', args
283        sys.exit(1)
284
285    def _HandleCommandGit(self, in_args):
286        """Handle execution of a git command
287
288        This uses a hacked-up parser.
289
290        Args:
291            in_args: Arguments after 'git' from the command line
292        """
293        git_args = []           # Top-level arguments to git itself
294        sub_cmd = None          # Git sub-command selected
295        args = []               # Arguments to the git sub-command
296        for arg in in_args:
297            if sub_cmd:
298                args.append(arg)
299            elif arg[0] == '-':
300                git_args.append(arg)
301            else:
302                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
303                    git_args.append(arg)
304                else:
305                    sub_cmd = arg
306        if sub_cmd == 'config':
307            return self._HandleCommandGitConfig(args)
308        elif sub_cmd == 'log':
309            return self._HandleCommandGitLog(args)
310        elif sub_cmd == 'clone':
311            return command.CommandResult(return_code=0)
312        elif sub_cmd == 'checkout':
313            return command.CommandResult(return_code=0)
314
315        # Not handled, so abort
316        print 'git', git_args, sub_cmd, args
317        sys.exit(1)
318
319    def _HandleCommandNm(self, args):
320        return command.CommandResult(return_code=0)
321
322    def _HandleCommandObjdump(self, args):
323        return command.CommandResult(return_code=0)
324
325    def _HandleCommandSize(self, args):
326        return command.CommandResult(return_code=0)
327
328    def _HandleCommand(self, **kwargs):
329        """Handle a command execution.
330
331        The command is in kwargs['pipe-list'], as a list of pipes, each a
332        list of commands. The command should be emulated as required for
333        testing purposes.
334
335        Returns:
336            A CommandResult object
337        """
338        pipe_list = kwargs['pipe_list']
339        wc = False
340        if len(pipe_list) != 1:
341            if pipe_list[1] == ['wc', '-l']:
342                wc = True
343            else:
344                print 'invalid pipe', kwargs
345                sys.exit(1)
346        cmd = pipe_list[0][0]
347        args = pipe_list[0][1:]
348        result = None
349        if cmd == 'git':
350            result = self._HandleCommandGit(args)
351        elif cmd == './scripts/show-gnu-make':
352            return command.CommandResult(return_code=0, stdout='make')
353        elif cmd.endswith('nm'):
354            return self._HandleCommandNm(args)
355        elif cmd.endswith('objdump'):
356            return self._HandleCommandObjdump(args)
357        elif cmd.endswith( 'size'):
358            return self._HandleCommandSize(args)
359
360        if not result:
361            # Not handled, so abort
362            print 'unknown command', kwargs
363            sys.exit(1)
364
365        if wc:
366            result.stdout = len(result.stdout.splitlines())
367        return result
368
369    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
370        """Handle execution of 'make'
371
372        Args:
373            commit: Commit object that is being built
374            brd: Board object that is being built
375            stage: Stage that we are at (mrproper, config, build)
376            cwd: Directory where make should be run
377            args: Arguments to pass to make
378            kwargs: Arguments to pass to command.RunPipe()
379        """
380        self._make_calls += 1
381        if stage == 'mrproper':
382            return command.CommandResult(return_code=0)
383        elif stage == 'config':
384            return command.CommandResult(return_code=0,
385                    combined='Test configuration complete')
386        elif stage == 'build':
387            stderr = ''
388            if type(commit) is not str:
389                stderr = self._error.get((brd.target, commit.sequence))
390            if stderr:
391                return command.CommandResult(return_code=1, stderr=stderr)
392            return command.CommandResult(return_code=0)
393
394        # Not handled, so abort
395        print 'make', stage
396        sys.exit(1)
397
398    # Example function to print output lines
399    def print_lines(self, lines):
400        print len(lines)
401        for line in lines:
402            print line
403        #self.print_lines(terminal.GetPrintTestLines())
404
405    def testNoBoards(self):
406        """Test that buildman aborts when there are no boards"""
407        self._boards = board.Boards()
408        with self.assertRaises(SystemExit):
409            self._RunControl()
410
411    def testCurrentSource(self):
412        """Very simple test to invoke buildman on the current source"""
413        self.setupToolchains();
414        self._RunControl()
415        lines = terminal.GetPrintTestLines()
416        self.assertIn('Building current source for %d boards' % len(boards),
417                      lines[0].text)
418
419    def testBadBranch(self):
420        """Test that we can detect an invalid branch"""
421        with self.assertRaises(ValueError):
422            self._RunControl('-b', 'badbranch')
423
424    def testBadToolchain(self):
425        """Test that missing toolchains are detected"""
426        self.setupToolchains();
427        ret_code = self._RunControl('-b', TEST_BRANCH)
428        lines = terminal.GetPrintTestLines()
429
430        # Buildman always builds the upstream commit as well
431        self.assertIn('Building %d commits for %d boards' %
432                (self._commits, len(boards)), lines[0].text)
433        self.assertEqual(self._builder.count, self._total_builds)
434
435        # Only sandbox should succeed, the others don't have toolchains
436        self.assertEqual(self._builder.fail,
437                         self._total_builds - self._commits)
438        self.assertEqual(ret_code, 128)
439
440        for commit in range(self._commits):
441            for board in self._boards.GetList():
442                if board.arch != 'sandbox':
443                  errfile = self._builder.GetErrFile(commit, board.target)
444                  fd = open(errfile)
445                  self.assertEqual(fd.readlines(),
446                          ['No tool chain for %s\n' % board.arch])
447                  fd.close()
448
449    def testBranch(self):
450        """Test building a branch with all toolchains present"""
451        self._RunControl('-b', TEST_BRANCH)
452        self.assertEqual(self._builder.count, self._total_builds)
453        self.assertEqual(self._builder.fail, 0)
454
455    def testCount(self):
456        """Test building a specific number of commitst"""
457        self._RunControl('-b', TEST_BRANCH, '-c2')
458        self.assertEqual(self._builder.count, 2 * len(boards))
459        self.assertEqual(self._builder.fail, 0)
460        # Each board has a mrproper, config, and then one make per commit
461        self.assertEqual(self._make_calls, len(boards) * (2 + 2))
462
463    def testIncremental(self):
464        """Test building a branch twice - the second time should do nothing"""
465        self._RunControl('-b', TEST_BRANCH)
466
467        # Each board has a mrproper, config, and then one make per commit
468        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
469        self._make_calls = 0
470        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
471        self.assertEqual(self._make_calls, 0)
472        self.assertEqual(self._builder.count, self._total_builds)
473        self.assertEqual(self._builder.fail, 0)
474
475    def testForceBuild(self):
476        """The -f flag should force a rebuild"""
477        self._RunControl('-b', TEST_BRANCH)
478        self._make_calls = 0
479        self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
480        # Each board has a mrproper, config, and then one make per commit
481        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
482
483    def testForceReconfigure(self):
484        """The -f flag should force a rebuild"""
485        self._RunControl('-b', TEST_BRANCH, '-C')
486        # Each commit has a mrproper, config and make
487        self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
488
489    def testErrors(self):
490        """Test handling of build errors"""
491        self._error['board2', 1] = 'fred\n'
492        self._RunControl('-b', TEST_BRANCH)
493        self.assertEqual(self._builder.count, self._total_builds)
494        self.assertEqual(self._builder.fail, 1)
495
496        # Remove the error. This should have no effect since the commit will
497        # not be rebuilt
498        del self._error['board2', 1]
499        self._make_calls = 0
500        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
501        self.assertEqual(self._builder.count, self._total_builds)
502        self.assertEqual(self._make_calls, 0)
503        self.assertEqual(self._builder.fail, 1)
504
505        # Now use the -F flag to force rebuild of the bad commit
506        self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
507        self.assertEqual(self._builder.count, self._total_builds)
508        self.assertEqual(self._builder.fail, 0)
509        self.assertEqual(self._make_calls, 3)
510
511    def testBranchWithSlash(self):
512        """Test building a branch with a '/' in the name"""
513        self._test_branch = '/__dev/__testbranch'
514        self._RunControl('-b', self._test_branch, clean_dir=False)
515        self.assertEqual(self._builder.count, self._total_builds)
516        self.assertEqual(self._builder.fail, 0)
517