xref: /openbmc/u-boot/tools/buildman/func_test.py (revision 79df00fd)
1#
2# Copyright (c) 2014 Google, Inc
3#
4# SPDX-License-Identifier:      GPL-2.0+
5#
6
7import os
8import shutil
9import sys
10import tempfile
11import unittest
12
13import board
14import bsettings
15import cmdline
16import command
17import control
18import gitutil
19import terminal
20import toolchain
21
22settings_data = '''
23# Buildman settings file
24
25[toolchain]
26
27[toolchain-alias]
28
29[make-flags]
30src=/home/sjg/c/src
31chroot=/home/sjg/c/chroot
32vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
33chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
34chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
35chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
36'''
37
38boards = [
39    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
40    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
41    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
42    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
43]
44
45commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4639403bb patman: Use --no-pager' to stop git from forking a pager
47db6e6f2 patman: Remove the -a option
48f2ccf03 patman: Correct unit tests to run correctly
491d097f9 patman: Fix indentation in terminal.py
50d073747 patman: Support the 'reverse' option for 'git log
51"""
52
53commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
54Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
55Date:   Fri Aug 22 19:12:41 2014 +0900
56
57    buildman: refactor help message
58
59    "buildman [options]" is displayed by default.
60
61    Append the rest of help messages to parser.usage
62    instead of replacing it.
63
64    Besides, "-b <branch>" is not mandatory since commit fea5858e.
65    Drop it from the usage.
66
67    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
68""",
69"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
70Author: Simon Glass <sjg@chromium.org>
71Date:   Thu Aug 14 16:48:25 2014 -0600
72
73    patman: Support the 'reverse' option for 'git log'
74
75    This option is currently not supported, but needs to be, for buildman to
76    operate as expected.
77
78    Series-changes: 7
79    - Add new patch to fix the 'reverse' bug
80
81    Series-version: 8
82
83    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
84    Reported-by: York Sun <yorksun@freescale.com>
85    Signed-off-by: Simon Glass <sjg@chromium.org>
86
87""",
88"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
89Author: Simon Glass <sjg@chromium.org>
90Date:   Sat Aug 9 11:44:32 2014 -0600
91
92    patman: Fix indentation in terminal.py
93
94    This code came from a different project with 2-character indentation. Fix
95    it for U-Boot.
96
97    Series-changes: 6
98    - Add new patch to fix indentation in teminal.py
99
100    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
101    Signed-off-by: Simon Glass <sjg@chromium.org>
102
103""",
104"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
105Author: Simon Glass <sjg@chromium.org>
106Date:   Sat Aug 9 11:08:24 2014 -0600
107
108    patman: Correct unit tests to run correctly
109
110    It seems that doctest behaves differently now, and some of the unit tests
111    do not run. Adjust the tests to work correctly.
112
113     ./tools/patman/patman --test
114    <unittest.result.TestResult run=10 errors=0 failures=0>
115
116    Series-changes: 6
117    - Add new patch to fix patman unit tests
118
119    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
120
121""",
122"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
123Author: Simon Glass <sjg@chromium.org>
124Date:   Sat Aug 9 12:06:02 2014 -0600
125
126    patman: Remove the -a option
127
128    It seems that this is no longer needed, since checkpatch.pl will catch
129    whitespace problems in patches. Also the option is not widely used, so
130    it seems safe to just remove it.
131
132    Series-changes: 6
133    - Add new patch to remove patman's -a option
134
135    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
136    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
137
138""",
139"""commit 39403bb4f838153028a6f21ca30bf100f3791133
140Author: Simon Glass <sjg@chromium.org>
141Date:   Thu Aug 14 21:50:52 2014 -0600
142
143    patman: Use --no-pager' to stop git from forking a pager
144
145""",
146"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
147Author: Simon Glass <sjg@chromium.org>
148Date:   Fri Aug 22 15:57:39 2014 -0600
149
150    patman: Avoid changing the order of tags
151
152    patman collects tags that it sees in the commit and places them nicely
153    sorted at the end of the patch. However, this is not really necessary and
154    in fact is apparently not desirable.
155
156    Series-changes: 9
157    - Add new patch to avoid changing the order of tags
158
159    Series-version: 9
160
161    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
162    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
163"""]
164
165TEST_BRANCH = '__testbranch'
166
167class TestFunctional(unittest.TestCase):
168    """Functional test for buildman.
169
170    This aims to test from just below the invocation of buildman (parsing
171    of arguments) to 'make' and 'git' invocation. It is not a true
172    emd-to-end test, as it mocks git, make and the tool chain. But this
173    makes it easier to detect when the builder is doing the wrong thing,
174    since in many cases this test code will fail. For example, only a
175    very limited subset of 'git' arguments is supported - anything
176    unexpected will fail.
177    """
178    def setUp(self):
179        self._base_dir = tempfile.mkdtemp()
180        self._git_dir = os.path.join(self._base_dir, 'src')
181        self._buildman_pathname = sys.argv[0]
182        self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
183        command.test_result = self._HandleCommand
184        self.setupToolchains()
185        self._toolchains.Add('arm-gcc', test=False)
186        self._toolchains.Add('powerpc-gcc', test=False)
187        bsettings.Setup(None)
188        bsettings.AddFile(settings_data)
189        self._boards = board.Boards()
190        for brd in boards:
191            self._boards.AddBoard(board.Board(*brd))
192
193        # Directories where the source been cloned
194        self._clone_dirs = []
195        self._commits = len(commit_shortlog.splitlines()) + 1
196        self._total_builds = self._commits * len(boards)
197
198        # Number of calls to make
199        self._make_calls = 0
200
201        # Map of [board, commit] to error messages
202        self._error = {}
203
204        self._test_branch = TEST_BRANCH
205
206        # Avoid sending any output and clear all terminal output
207        terminal.SetPrintTestMode()
208        terminal.GetPrintTestLines()
209
210    def tearDown(self):
211        shutil.rmtree(self._base_dir)
212
213    def setupToolchains(self):
214        self._toolchains = toolchain.Toolchains()
215        self._toolchains.Add('gcc', test=False)
216
217    def _RunBuildman(self, *args):
218        return command.RunPipe([[self._buildman_pathname] + list(args)],
219                capture=True, capture_stderr=True)
220
221    def _RunControl(self, *args, **kwargs):
222        sys.argv = [sys.argv[0]] + list(args)
223        options, args = cmdline.ParseArgs()
224        result = control.DoBuildman(options, args, toolchains=self._toolchains,
225                make_func=self._HandleMake, boards=self._boards,
226                clean_dir=kwargs.get('clean_dir', True))
227        self._builder = control.builder
228        return result
229
230    def testFullHelp(self):
231        command.test_result = None
232        result = self._RunBuildman('-H')
233        help_file = os.path.join(self._buildman_dir, 'README')
234        self.assertEqual(len(result.stdout), os.path.getsize(help_file))
235        self.assertEqual(0, len(result.stderr))
236        self.assertEqual(0, result.return_code)
237
238    def testHelp(self):
239        command.test_result = None
240        result = self._RunBuildman('-h')
241        help_file = os.path.join(self._buildman_dir, 'README')
242        self.assertTrue(len(result.stdout) > 1000)
243        self.assertEqual(0, len(result.stderr))
244        self.assertEqual(0, result.return_code)
245
246    def testGitSetup(self):
247        """Test gitutils.Setup(), from outside the module itself"""
248        command.test_result = command.CommandResult(return_code=1)
249        gitutil.Setup()
250        self.assertEqual(gitutil.use_no_decorate, False)
251
252        command.test_result = command.CommandResult(return_code=0)
253        gitutil.Setup()
254        self.assertEqual(gitutil.use_no_decorate, True)
255
256    def _HandleCommandGitLog(self, args):
257        if args[-1] == '--':
258            args = args[:-1]
259        if '-n0' in args:
260            return command.CommandResult(return_code=0)
261        elif args[-1] == 'upstream/master..%s' % self._test_branch:
262            return command.CommandResult(return_code=0, stdout=commit_shortlog)
263        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
264            if args[-1] == self._test_branch:
265                count = int(args[3][2:])
266                return command.CommandResult(return_code=0,
267                                            stdout=''.join(commit_log[:count]))
268
269        # Not handled, so abort
270        print 'git log', args
271        sys.exit(1)
272
273    def _HandleCommandGitConfig(self, args):
274        config = args[0]
275        if config == 'sendemail.aliasesfile':
276            return command.CommandResult(return_code=0)
277        elif config.startswith('branch.badbranch'):
278            return command.CommandResult(return_code=1)
279        elif config == 'branch.%s.remote' % self._test_branch:
280            return command.CommandResult(return_code=0, stdout='upstream\n')
281        elif config == 'branch.%s.merge' % self._test_branch:
282            return command.CommandResult(return_code=0,
283                                         stdout='refs/heads/master\n')
284
285        # Not handled, so abort
286        print 'git config', args
287        sys.exit(1)
288
289    def _HandleCommandGit(self, in_args):
290        """Handle execution of a git command
291
292        This uses a hacked-up parser.
293
294        Args:
295            in_args: Arguments after 'git' from the command line
296        """
297        git_args = []           # Top-level arguments to git itself
298        sub_cmd = None          # Git sub-command selected
299        args = []               # Arguments to the git sub-command
300        for arg in in_args:
301            if sub_cmd:
302                args.append(arg)
303            elif arg[0] == '-':
304                git_args.append(arg)
305            else:
306                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
307                    git_args.append(arg)
308                else:
309                    sub_cmd = arg
310        if sub_cmd == 'config':
311            return self._HandleCommandGitConfig(args)
312        elif sub_cmd == 'log':
313            return self._HandleCommandGitLog(args)
314        elif sub_cmd == 'clone':
315            return command.CommandResult(return_code=0)
316        elif sub_cmd == 'checkout':
317            return command.CommandResult(return_code=0)
318
319        # Not handled, so abort
320        print 'git', git_args, sub_cmd, args
321        sys.exit(1)
322
323    def _HandleCommandNm(self, args):
324        return command.CommandResult(return_code=0)
325
326    def _HandleCommandObjdump(self, args):
327        return command.CommandResult(return_code=0)
328
329    def _HandleCommandSize(self, args):
330        return command.CommandResult(return_code=0)
331
332    def _HandleCommand(self, **kwargs):
333        """Handle a command execution.
334
335        The command is in kwargs['pipe-list'], as a list of pipes, each a
336        list of commands. The command should be emulated as required for
337        testing purposes.
338
339        Returns:
340            A CommandResult object
341        """
342        pipe_list = kwargs['pipe_list']
343        wc = False
344        if len(pipe_list) != 1:
345            if pipe_list[1] == ['wc', '-l']:
346                wc = True
347            else:
348                print 'invalid pipe', kwargs
349                sys.exit(1)
350        cmd = pipe_list[0][0]
351        args = pipe_list[0][1:]
352        result = None
353        if cmd == 'git':
354            result = self._HandleCommandGit(args)
355        elif cmd == './scripts/show-gnu-make':
356            return command.CommandResult(return_code=0, stdout='make')
357        elif cmd.endswith('nm'):
358            return self._HandleCommandNm(args)
359        elif cmd.endswith('objdump'):
360            return self._HandleCommandObjdump(args)
361        elif cmd.endswith( 'size'):
362            return self._HandleCommandSize(args)
363
364        if not result:
365            # Not handled, so abort
366            print 'unknown command', kwargs
367            sys.exit(1)
368
369        if wc:
370            result.stdout = len(result.stdout.splitlines())
371        return result
372
373    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
374        """Handle execution of 'make'
375
376        Args:
377            commit: Commit object that is being built
378            brd: Board object that is being built
379            stage: Stage that we are at (mrproper, config, build)
380            cwd: Directory where make should be run
381            args: Arguments to pass to make
382            kwargs: Arguments to pass to command.RunPipe()
383        """
384        self._make_calls += 1
385        if stage == 'mrproper':
386            return command.CommandResult(return_code=0)
387        elif stage == 'config':
388            return command.CommandResult(return_code=0,
389                    combined='Test configuration complete')
390        elif stage == 'build':
391            stderr = ''
392            if type(commit) is not str:
393                stderr = self._error.get((brd.target, commit.sequence))
394            if stderr:
395                return command.CommandResult(return_code=1, stderr=stderr)
396            return command.CommandResult(return_code=0)
397
398        # Not handled, so abort
399        print 'make', stage
400        sys.exit(1)
401
402    # Example function to print output lines
403    def print_lines(self, lines):
404        print len(lines)
405        for line in lines:
406            print line
407        #self.print_lines(terminal.GetPrintTestLines())
408
409    def testNoBoards(self):
410        """Test that buildman aborts when there are no boards"""
411        self._boards = board.Boards()
412        with self.assertRaises(SystemExit):
413            self._RunControl()
414
415    def testCurrentSource(self):
416        """Very simple test to invoke buildman on the current source"""
417        self.setupToolchains();
418        self._RunControl()
419        lines = terminal.GetPrintTestLines()
420        self.assertIn('Building current source for %d boards' % len(boards),
421                      lines[0].text)
422
423    def testBadBranch(self):
424        """Test that we can detect an invalid branch"""
425        with self.assertRaises(ValueError):
426            self._RunControl('-b', 'badbranch')
427
428    def testBadToolchain(self):
429        """Test that missing toolchains are detected"""
430        self.setupToolchains();
431        ret_code = self._RunControl('-b', TEST_BRANCH)
432        lines = terminal.GetPrintTestLines()
433
434        # Buildman always builds the upstream commit as well
435        self.assertIn('Building %d commits for %d boards' %
436                (self._commits, len(boards)), lines[0].text)
437        self.assertEqual(self._builder.count, self._total_builds)
438
439        # Only sandbox should succeed, the others don't have toolchains
440        self.assertEqual(self._builder.fail,
441                         self._total_builds - self._commits)
442        self.assertEqual(ret_code, 128)
443
444        for commit in range(self._commits):
445            for board in self._boards.GetList():
446                if board.arch != 'sandbox':
447                  errfile = self._builder.GetErrFile(commit, board.target)
448                  fd = open(errfile)
449                  self.assertEqual(fd.readlines(),
450                          ['No tool chain for %s\n' % board.arch])
451                  fd.close()
452
453    def testBranch(self):
454        """Test building a branch with all toolchains present"""
455        self._RunControl('-b', TEST_BRANCH)
456        self.assertEqual(self._builder.count, self._total_builds)
457        self.assertEqual(self._builder.fail, 0)
458
459    def testCount(self):
460        """Test building a specific number of commitst"""
461        self._RunControl('-b', TEST_BRANCH, '-c2')
462        self.assertEqual(self._builder.count, 2 * len(boards))
463        self.assertEqual(self._builder.fail, 0)
464        # Each board has a mrproper, config, and then one make per commit
465        self.assertEqual(self._make_calls, len(boards) * (2 + 2))
466
467    def testIncremental(self):
468        """Test building a branch twice - the second time should do nothing"""
469        self._RunControl('-b', TEST_BRANCH)
470
471        # Each board has a mrproper, config, and then one make per commit
472        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
473        self._make_calls = 0
474        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
475        self.assertEqual(self._make_calls, 0)
476        self.assertEqual(self._builder.count, self._total_builds)
477        self.assertEqual(self._builder.fail, 0)
478
479    def testForceBuild(self):
480        """The -f flag should force a rebuild"""
481        self._RunControl('-b', TEST_BRANCH)
482        self._make_calls = 0
483        self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
484        # Each board has a mrproper, config, and then one make per commit
485        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
486
487    def testForceReconfigure(self):
488        """The -f flag should force a rebuild"""
489        self._RunControl('-b', TEST_BRANCH, '-C')
490        # Each commit has a mrproper, config and make
491        self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
492
493    def testErrors(self):
494        """Test handling of build errors"""
495        self._error['board2', 1] = 'fred\n'
496        self._RunControl('-b', TEST_BRANCH)
497        self.assertEqual(self._builder.count, self._total_builds)
498        self.assertEqual(self._builder.fail, 1)
499
500        # Remove the error. This should have no effect since the commit will
501        # not be rebuilt
502        del self._error['board2', 1]
503        self._make_calls = 0
504        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
505        self.assertEqual(self._builder.count, self._total_builds)
506        self.assertEqual(self._make_calls, 0)
507        self.assertEqual(self._builder.fail, 1)
508
509        # Now use the -F flag to force rebuild of the bad commit
510        self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
511        self.assertEqual(self._builder.count, self._total_builds)
512        self.assertEqual(self._builder.fail, 0)
513        self.assertEqual(self._make_calls, 3)
514
515    def testBranchWithSlash(self):
516        """Test building a branch with a '/' in the name"""
517        self._test_branch = '/__dev/__testbranch'
518        self._RunControl('-b', self._test_branch, clean_dir=False)
519        self.assertEqual(self._builder.count, self._total_builds)
520        self.assertEqual(self._builder.fail, 0)
521