xref: /openbmc/u-boot/tools/buildman/func_test.py (revision dfb7e932350f3afee230d733e32335fe3c9b96b1) 
1#
2# Copyright (c) 2014 Google, Inc
3#
4# SPDX-License-Identifier:      GPL-2.0+
5#
6
7import os
8import shutil
9import sys
10import tempfile
11import unittest
12
13import board
14import bsettings
15import cmdline
16import command
17import control
18import gitutil
19import terminal
20import toolchain
21
22settings_data = '''
23# Buildman settings file
24
25[toolchain]
26
27[toolchain-alias]
28
29[make-flags]
30src=/home/sjg/c/src
31chroot=/home/sjg/c/chroot
32vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
33chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
34chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
35chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
36'''
37
38boards = [
39    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
40    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
41    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
42    ['Active', 'powerpc', 'mpc5xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
43    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
44]
45
46commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4739403bb patman: Use --no-pager' to stop git from forking a pager
48db6e6f2 patman: Remove the -a option
49f2ccf03 patman: Correct unit tests to run correctly
501d097f9 patman: Fix indentation in terminal.py
51d073747 patman: Support the 'reverse' option for 'git log
52"""
53
54commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
55Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
56Date:   Fri Aug 22 19:12:41 2014 +0900
57
58    buildman: refactor help message
59
60    "buildman [options]" is displayed by default.
61
62    Append the rest of help messages to parser.usage
63    instead of replacing it.
64
65    Besides, "-b <branch>" is not mandatory since commit fea5858e.
66    Drop it from the usage.
67
68    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
69""",
70"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
71Author: Simon Glass <sjg@chromium.org>
72Date:   Thu Aug 14 16:48:25 2014 -0600
73
74    patman: Support the 'reverse' option for 'git log'
75
76    This option is currently not supported, but needs to be, for buildman to
77    operate as expected.
78
79    Series-changes: 7
80    - Add new patch to fix the 'reverse' bug
81
82
83    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
84    Reported-by: York Sun <yorksun@freescale.com>
85    Signed-off-by: Simon Glass <sjg@chromium.org>
86
87""",
88"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
89Author: Simon Glass <sjg@chromium.org>
90Date:   Sat Aug 9 11:44:32 2014 -0600
91
92    patman: Fix indentation in terminal.py
93
94    This code came from a different project with 2-character indentation. Fix
95    it for U-Boot.
96
97    Series-changes: 6
98    - Add new patch to fix indentation in teminal.py
99
100    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
101    Signed-off-by: Simon Glass <sjg@chromium.org>
102
103""",
104"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
105Author: Simon Glass <sjg@chromium.org>
106Date:   Sat Aug 9 11:08:24 2014 -0600
107
108    patman: Correct unit tests to run correctly
109
110    It seems that doctest behaves differently now, and some of the unit tests
111    do not run. Adjust the tests to work correctly.
112
113     ./tools/patman/patman --test
114    <unittest.result.TestResult run=10 errors=0 failures=0>
115
116    Series-changes: 6
117    - Add new patch to fix patman unit tests
118
119    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
120
121""",
122"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
123Author: Simon Glass <sjg@chromium.org>
124Date:   Sat Aug 9 12:06:02 2014 -0600
125
126    patman: Remove the -a option
127
128    It seems that this is no longer needed, since checkpatch.pl will catch
129    whitespace problems in patches. Also the option is not widely used, so
130    it seems safe to just remove it.
131
132    Series-changes: 6
133    - Add new patch to remove patman's -a option
134
135    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
136    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
137
138""",
139"""commit 39403bb4f838153028a6f21ca30bf100f3791133
140Author: Simon Glass <sjg@chromium.org>
141Date:   Thu Aug 14 21:50:52 2014 -0600
142
143    patman: Use --no-pager' to stop git from forking a pager
144
145""",
146"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
147Author: Simon Glass <sjg@chromium.org>
148Date:   Fri Aug 22 15:57:39 2014 -0600
149
150    patman: Avoid changing the order of tags
151
152    patman collects tags that it sees in the commit and places them nicely
153    sorted at the end of the patch. However, this is not really necessary and
154    in fact is apparently not desirable.
155
156    Series-changes: 9
157    - Add new patch to avoid changing the order of tags
158
159    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
160    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
161"""]
162
163TEST_BRANCH = '__testbranch'
164
165class TestFunctional(unittest.TestCase):
166    """Functional test for buildman.
167
168    This aims to test from just below the invocation of buildman (parsing
169    of arguments) to 'make' and 'git' invocation. It is not a true
170    emd-to-end test, as it mocks git, make and the tool chain. But this
171    makes it easier to detect when the builder is doing the wrong thing,
172    since in many cases this test code will fail. For example, only a
173    very limited subset of 'git' arguments is supported - anything
174    unexpected will fail.
175    """
176    def setUp(self):
177        self._base_dir = tempfile.mkdtemp()
178        self._git_dir = os.path.join(self._base_dir, 'src')
179        self._buildman_pathname = sys.argv[0]
180        self._buildman_dir = os.path.dirname(sys.argv[0])
181        command.test_result = self._HandleCommand
182        self.setupToolchains()
183        self._toolchains.Add('arm-gcc', test=False)
184        self._toolchains.Add('powerpc-gcc', test=False)
185        bsettings.Setup(None)
186        bsettings.AddFile(settings_data)
187        self._boards = board.Boards()
188        for brd in boards:
189            self._boards.AddBoard(board.Board(*brd))
190
191        # Directories where the source been cloned
192        self._clone_dirs = []
193        self._commits = len(commit_shortlog.splitlines()) + 1
194        self._total_builds = self._commits * len(boards)
195
196        # Number of calls to make
197        self._make_calls = 0
198
199        # Map of [board, commit] to error messages
200        self._error = {}
201
202        # Avoid sending any output and clear all terminal output
203        terminal.SetPrintTestMode()
204        terminal.GetPrintTestLines()
205
206    def tearDown(self):
207        shutil.rmtree(self._base_dir)
208
209    def setupToolchains(self):
210        self._toolchains = toolchain.Toolchains()
211        self._toolchains.Add('gcc', test=False)
212
213    def _RunBuildman(self, *args):
214        return command.RunPipe([[self._buildman_pathname] + list(args)],
215                capture=True, capture_stderr=True)
216
217    def _RunControl(self, *args, **kwargs):
218        sys.argv = [sys.argv[0]] + list(args)
219        options, args = cmdline.ParseArgs()
220        result = control.DoBuildman(options, args, toolchains=self._toolchains,
221                make_func=self._HandleMake, boards=self._boards,
222                clean_dir=kwargs.get('clean_dir', True))
223        self._builder = control.builder
224        return result
225
226    def testFullHelp(self):
227        command.test_result = None
228        result = self._RunBuildman('-H')
229        help_file = os.path.join(self._buildman_dir, 'README')
230        self.assertEqual(len(result.stdout), os.path.getsize(help_file))
231        self.assertEqual(0, len(result.stderr))
232        self.assertEqual(0, result.return_code)
233
234    def testHelp(self):
235        command.test_result = None
236        result = self._RunBuildman('-h')
237        help_file = os.path.join(self._buildman_dir, 'README')
238        self.assertTrue(len(result.stdout) > 1000)
239        self.assertEqual(0, len(result.stderr))
240        self.assertEqual(0, result.return_code)
241
242    def testGitSetup(self):
243        """Test gitutils.Setup(), from outside the module itself"""
244        command.test_result = command.CommandResult(return_code=1)
245        gitutil.Setup()
246        self.assertEqual(gitutil.use_no_decorate, False)
247
248        command.test_result = command.CommandResult(return_code=0)
249        gitutil.Setup()
250        self.assertEqual(gitutil.use_no_decorate, True)
251
252    def _HandleCommandGitLog(self, args):
253        if '-n0' in args:
254            return command.CommandResult(return_code=0)
255        elif args[-1] == 'upstream/master..%s' % TEST_BRANCH:
256            return command.CommandResult(return_code=0, stdout=commit_shortlog)
257        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
258            if args[-1] == TEST_BRANCH:
259                count = int(args[3][2:])
260                return command.CommandResult(return_code=0,
261                                            stdout=''.join(commit_log[:count]))
262
263        # Not handled, so abort
264        print 'git log', args
265        sys.exit(1)
266
267    def _HandleCommandGitConfig(self, args):
268        config = args[0]
269        if config == 'sendemail.aliasesfile':
270            return command.CommandResult(return_code=0)
271        elif config.startswith('branch.badbranch'):
272            return command.CommandResult(return_code=1)
273        elif config == 'branch.%s.remote' % TEST_BRANCH:
274            return command.CommandResult(return_code=0, stdout='upstream\n')
275        elif config == 'branch.%s.merge' % TEST_BRANCH:
276            return command.CommandResult(return_code=0,
277                                         stdout='refs/heads/master\n')
278
279        # Not handled, so abort
280        print 'git config', args
281        sys.exit(1)
282
283    def _HandleCommandGit(self, in_args):
284        """Handle execution of a git command
285
286        This uses a hacked-up parser.
287
288        Args:
289            in_args: Arguments after 'git' from the command line
290        """
291        git_args = []           # Top-level arguments to git itself
292        sub_cmd = None          # Git sub-command selected
293        args = []               # Arguments to the git sub-command
294        for arg in in_args:
295            if sub_cmd:
296                args.append(arg)
297            elif arg[0] == '-':
298                git_args.append(arg)
299            else:
300                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
301                    git_args.append(arg)
302                else:
303                    sub_cmd = arg
304        if sub_cmd == 'config':
305            return self._HandleCommandGitConfig(args)
306        elif sub_cmd == 'log':
307            return self._HandleCommandGitLog(args)
308        elif sub_cmd == 'clone':
309            return command.CommandResult(return_code=0)
310        elif sub_cmd == 'checkout':
311            return command.CommandResult(return_code=0)
312
313        # Not handled, so abort
314        print 'git', git_args, sub_cmd, args
315        sys.exit(1)
316
317    def _HandleCommandNm(self, args):
318        return command.CommandResult(return_code=0)
319
320    def _HandleCommandObjdump(self, args):
321        return command.CommandResult(return_code=0)
322
323    def _HandleCommandSize(self, args):
324        return command.CommandResult(return_code=0)
325
326    def _HandleCommand(self, **kwargs):
327        """Handle a command execution.
328
329        The command is in kwargs['pipe-list'], as a list of pipes, each a
330        list of commands. The command should be emulated as required for
331        testing purposes.
332
333        Returns:
334            A CommandResult object
335        """
336        pipe_list = kwargs['pipe_list']
337        wc = False
338        if len(pipe_list) != 1:
339            if pipe_list[1] == ['wc', '-l']:
340                wc = True
341            else:
342                print 'invalid pipe', kwargs
343                sys.exit(1)
344        cmd = pipe_list[0][0]
345        args = pipe_list[0][1:]
346        result = None
347        if cmd == 'git':
348            result = self._HandleCommandGit(args)
349        elif cmd == './scripts/show-gnu-make':
350            return command.CommandResult(return_code=0, stdout='make')
351        elif cmd.endswith('nm'):
352            return self._HandleCommandNm(args)
353        elif cmd.endswith('objdump'):
354            return self._HandleCommandObjdump(args)
355        elif cmd.endswith( 'size'):
356            return self._HandleCommandSize(args)
357
358        if not result:
359            # Not handled, so abort
360            print 'unknown command', kwargs
361            sys.exit(1)
362
363        if wc:
364            result.stdout = len(result.stdout.splitlines())
365        return result
366
367    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
368        """Handle execution of 'make'
369
370        Args:
371            commit: Commit object that is being built
372            brd: Board object that is being built
373            stage: Stage that we are at (mrproper, config, build)
374            cwd: Directory where make should be run
375            args: Arguments to pass to make
376            kwargs: Arguments to pass to command.RunPipe()
377        """
378        self._make_calls += 1
379        if stage == 'mrproper':
380            return command.CommandResult(return_code=0)
381        elif stage == 'config':
382            return command.CommandResult(return_code=0,
383                    combined='Test configuration complete')
384        elif stage == 'build':
385            stderr = ''
386            if type(commit) is not str:
387                stderr = self._error.get((brd.target, commit.sequence))
388            if stderr:
389                return command.CommandResult(return_code=1, stderr=stderr)
390            return command.CommandResult(return_code=0)
391
392        # Not handled, so abort
393        print 'make', stage
394        sys.exit(1)
395
396    # Example function to print output lines
397    def print_lines(self, lines):
398        print len(lines)
399        for line in lines:
400            print line
401        #self.print_lines(terminal.GetPrintTestLines())
402
403    def testNoBoards(self):
404        """Test that buildman aborts when there are no boards"""
405        self._boards = board.Boards()
406        with self.assertRaises(SystemExit):
407            self._RunControl()
408
409    def testCurrentSource(self):
410        """Very simple test to invoke buildman on the current source"""
411        self.setupToolchains();
412        self._RunControl()
413        lines = terminal.GetPrintTestLines()
414        self.assertIn('Building current source for %d boards' % len(boards),
415                      lines[0].text)
416
417    def testBadBranch(self):
418        """Test that we can detect an invalid branch"""
419        with self.assertRaises(ValueError):
420            self._RunControl('-b', 'badbranch')
421
422    def testBadToolchain(self):
423        """Test that missing toolchains are detected"""
424        self.setupToolchains();
425        ret_code = self._RunControl('-b', TEST_BRANCH)
426        lines = terminal.GetPrintTestLines()
427
428        # Buildman always builds the upstream commit as well
429        self.assertIn('Building %d commits for %d boards' %
430                (self._commits, len(boards)), lines[0].text)
431        self.assertEqual(self._builder.count, self._total_builds)
432
433        # Only sandbox should succeed, the others don't have toolchains
434        self.assertEqual(self._builder.fail,
435                         self._total_builds - self._commits)
436        self.assertEqual(ret_code, 128)
437
438        for commit in range(self._commits):
439            for board in self._boards.GetList():
440                if board.arch != 'sandbox':
441                  errfile = self._builder.GetErrFile(commit, board.target)
442                  fd = open(errfile)
443                  self.assertEqual(fd.readlines(),
444                          ['No tool chain for %s\n' % board.arch])
445                  fd.close()
446
447    def testBranch(self):
448        """Test building a branch with all toolchains present"""
449        self._RunControl('-b', TEST_BRANCH)
450        self.assertEqual(self._builder.count, self._total_builds)
451        self.assertEqual(self._builder.fail, 0)
452
453    def testCount(self):
454        """Test building a specific number of commitst"""
455        self._RunControl('-b', TEST_BRANCH, '-c2')
456        self.assertEqual(self._builder.count, 2 * len(boards))
457        self.assertEqual(self._builder.fail, 0)
458        # Each board has a mrproper, config, and then one make per commit
459        self.assertEqual(self._make_calls, len(boards) * (2 + 2))
460
461    def testIncremental(self):
462        """Test building a branch twice - the second time should do nothing"""
463        self._RunControl('-b', TEST_BRANCH)
464
465        # Each board has a mrproper, config, and then one make per commit
466        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
467        self._make_calls = 0
468        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
469        self.assertEqual(self._make_calls, 0)
470        self.assertEqual(self._builder.count, self._total_builds)
471        self.assertEqual(self._builder.fail, 0)
472
473    def testForceBuild(self):
474        """The -f flag should force a rebuild"""
475        self._RunControl('-b', TEST_BRANCH)
476        self._make_calls = 0
477        self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
478        # Each board has a mrproper, config, and then one make per commit
479        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
480
481    def testForceReconfigure(self):
482        """The -f flag should force a rebuild"""
483        self._RunControl('-b', TEST_BRANCH, '-C')
484        # Each commit has a mrproper, config and make
485        self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
486
487    def testErrors(self):
488        """Test handling of build errors"""
489        self._error['board2', 1] = 'fred\n'
490        self._RunControl('-b', TEST_BRANCH)
491        self.assertEqual(self._builder.count, self._total_builds)
492        self.assertEqual(self._builder.fail, 1)
493
494        # Remove the error. This should have no effect since the commit will
495        # not be rebuilt
496        del self._error['board2', 1]
497        self._make_calls = 0
498        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
499        self.assertEqual(self._builder.count, self._total_builds)
500        self.assertEqual(self._make_calls, 0)
501        self.assertEqual(self._builder.fail, 1)
502
503        # Now use the -F flag to force rebuild of the bad commit
504        self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
505        self.assertEqual(self._builder.count, self._total_builds)
506        self.assertEqual(self._builder.fail, 0)
507        self.assertEqual(self._make_calls, 3)
508