xref: /openbmc/u-boot/tools/buildman/func_test.py (revision ca6c5e03) 
1#
2# Copyright (c) 2014 Google, Inc
3#
4# SPDX-License-Identifier:      GPL-2.0+
5#
6
7import os
8import shutil
9import sys
10import tempfile
11import unittest
12
13import board
14import bsettings
15import cmdline
16import command
17import control
18import gitutil
19import terminal
20import toolchain
21
22settings_data = '''
23# Buildman settings file
24
25[toolchain]
26
27[toolchain-alias]
28
29[make-flags]
30src=/home/sjg/c/src
31chroot=/home/sjg/c/chroot
32vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
33chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
34chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
35chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
36'''
37
38boards = [
39    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
40    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
41    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
42    ['Active', 'powerpc', 'mpc5xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
43    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
44]
45
46commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4739403bb patman: Use --no-pager' to stop git from forking a pager
48db6e6f2 patman: Remove the -a option
49f2ccf03 patman: Correct unit tests to run correctly
501d097f9 patman: Fix indentation in terminal.py
51d073747 patman: Support the 'reverse' option for 'git log
52"""
53
54commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
55Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
56Date:   Fri Aug 22 19:12:41 2014 +0900
57
58    buildman: refactor help message
59
60    "buildman [options]" is displayed by default.
61
62    Append the rest of help messages to parser.usage
63    instead of replacing it.
64
65    Besides, "-b <branch>" is not mandatory since commit fea5858e.
66    Drop it from the usage.
67
68    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
69""",
70"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
71Author: Simon Glass <sjg@chromium.org>
72Date:   Thu Aug 14 16:48:25 2014 -0600
73
74    patman: Support the 'reverse' option for 'git log'
75
76    This option is currently not supported, but needs to be, for buildman to
77    operate as expected.
78
79    Series-changes: 7
80    - Add new patch to fix the 'reverse' bug
81
82    Series-version: 8
83
84    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
85    Reported-by: York Sun <yorksun@freescale.com>
86    Signed-off-by: Simon Glass <sjg@chromium.org>
87
88""",
89"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
90Author: Simon Glass <sjg@chromium.org>
91Date:   Sat Aug 9 11:44:32 2014 -0600
92
93    patman: Fix indentation in terminal.py
94
95    This code came from a different project with 2-character indentation. Fix
96    it for U-Boot.
97
98    Series-changes: 6
99    - Add new patch to fix indentation in teminal.py
100
101    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
102    Signed-off-by: Simon Glass <sjg@chromium.org>
103
104""",
105"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
106Author: Simon Glass <sjg@chromium.org>
107Date:   Sat Aug 9 11:08:24 2014 -0600
108
109    patman: Correct unit tests to run correctly
110
111    It seems that doctest behaves differently now, and some of the unit tests
112    do not run. Adjust the tests to work correctly.
113
114     ./tools/patman/patman --test
115    <unittest.result.TestResult run=10 errors=0 failures=0>
116
117    Series-changes: 6
118    - Add new patch to fix patman unit tests
119
120    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
121
122""",
123"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
124Author: Simon Glass <sjg@chromium.org>
125Date:   Sat Aug 9 12:06:02 2014 -0600
126
127    patman: Remove the -a option
128
129    It seems that this is no longer needed, since checkpatch.pl will catch
130    whitespace problems in patches. Also the option is not widely used, so
131    it seems safe to just remove it.
132
133    Series-changes: 6
134    - Add new patch to remove patman's -a option
135
136    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
137    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
138
139""",
140"""commit 39403bb4f838153028a6f21ca30bf100f3791133
141Author: Simon Glass <sjg@chromium.org>
142Date:   Thu Aug 14 21:50:52 2014 -0600
143
144    patman: Use --no-pager' to stop git from forking a pager
145
146""",
147"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
148Author: Simon Glass <sjg@chromium.org>
149Date:   Fri Aug 22 15:57:39 2014 -0600
150
151    patman: Avoid changing the order of tags
152
153    patman collects tags that it sees in the commit and places them nicely
154    sorted at the end of the patch. However, this is not really necessary and
155    in fact is apparently not desirable.
156
157    Series-changes: 9
158    - Add new patch to avoid changing the order of tags
159
160    Series-version: 9
161
162    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
163    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
164"""]
165
166TEST_BRANCH = '__testbranch'
167
168class TestFunctional(unittest.TestCase):
169    """Functional test for buildman.
170
171    This aims to test from just below the invocation of buildman (parsing
172    of arguments) to 'make' and 'git' invocation. It is not a true
173    emd-to-end test, as it mocks git, make and the tool chain. But this
174    makes it easier to detect when the builder is doing the wrong thing,
175    since in many cases this test code will fail. For example, only a
176    very limited subset of 'git' arguments is supported - anything
177    unexpected will fail.
178    """
179    def setUp(self):
180        self._base_dir = tempfile.mkdtemp()
181        self._git_dir = os.path.join(self._base_dir, 'src')
182        self._buildman_pathname = sys.argv[0]
183        self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
184        command.test_result = self._HandleCommand
185        self.setupToolchains()
186        self._toolchains.Add('arm-gcc', test=False)
187        self._toolchains.Add('powerpc-gcc', test=False)
188        bsettings.Setup(None)
189        bsettings.AddFile(settings_data)
190        self._boards = board.Boards()
191        for brd in boards:
192            self._boards.AddBoard(board.Board(*brd))
193
194        # Directories where the source been cloned
195        self._clone_dirs = []
196        self._commits = len(commit_shortlog.splitlines()) + 1
197        self._total_builds = self._commits * len(boards)
198
199        # Number of calls to make
200        self._make_calls = 0
201
202        # Map of [board, commit] to error messages
203        self._error = {}
204
205        self._test_branch = TEST_BRANCH
206
207        # Avoid sending any output and clear all terminal output
208        terminal.SetPrintTestMode()
209        terminal.GetPrintTestLines()
210
211    def tearDown(self):
212        shutil.rmtree(self._base_dir)
213
214    def setupToolchains(self):
215        self._toolchains = toolchain.Toolchains()
216        self._toolchains.Add('gcc', test=False)
217
218    def _RunBuildman(self, *args):
219        return command.RunPipe([[self._buildman_pathname] + list(args)],
220                capture=True, capture_stderr=True)
221
222    def _RunControl(self, *args, **kwargs):
223        sys.argv = [sys.argv[0]] + list(args)
224        options, args = cmdline.ParseArgs()
225        result = control.DoBuildman(options, args, toolchains=self._toolchains,
226                make_func=self._HandleMake, boards=self._boards,
227                clean_dir=kwargs.get('clean_dir', True))
228        self._builder = control.builder
229        return result
230
231    def testFullHelp(self):
232        command.test_result = None
233        result = self._RunBuildman('-H')
234        help_file = os.path.join(self._buildman_dir, 'README')
235        self.assertEqual(len(result.stdout), os.path.getsize(help_file))
236        self.assertEqual(0, len(result.stderr))
237        self.assertEqual(0, result.return_code)
238
239    def testHelp(self):
240        command.test_result = None
241        result = self._RunBuildman('-h')
242        help_file = os.path.join(self._buildman_dir, 'README')
243        self.assertTrue(len(result.stdout) > 1000)
244        self.assertEqual(0, len(result.stderr))
245        self.assertEqual(0, result.return_code)
246
247    def testGitSetup(self):
248        """Test gitutils.Setup(), from outside the module itself"""
249        command.test_result = command.CommandResult(return_code=1)
250        gitutil.Setup()
251        self.assertEqual(gitutil.use_no_decorate, False)
252
253        command.test_result = command.CommandResult(return_code=0)
254        gitutil.Setup()
255        self.assertEqual(gitutil.use_no_decorate, True)
256
257    def _HandleCommandGitLog(self, args):
258        if args[-1] == '--':
259            args = args[:-1]
260        if '-n0' in args:
261            return command.CommandResult(return_code=0)
262        elif args[-1] == 'upstream/master..%s' % self._test_branch:
263            return command.CommandResult(return_code=0, stdout=commit_shortlog)
264        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
265            if args[-1] == self._test_branch:
266                count = int(args[3][2:])
267                return command.CommandResult(return_code=0,
268                                            stdout=''.join(commit_log[:count]))
269
270        # Not handled, so abort
271        print 'git log', args
272        sys.exit(1)
273
274    def _HandleCommandGitConfig(self, args):
275        config = args[0]
276        if config == 'sendemail.aliasesfile':
277            return command.CommandResult(return_code=0)
278        elif config.startswith('branch.badbranch'):
279            return command.CommandResult(return_code=1)
280        elif config == 'branch.%s.remote' % self._test_branch:
281            return command.CommandResult(return_code=0, stdout='upstream\n')
282        elif config == 'branch.%s.merge' % self._test_branch:
283            return command.CommandResult(return_code=0,
284                                         stdout='refs/heads/master\n')
285
286        # Not handled, so abort
287        print 'git config', args
288        sys.exit(1)
289
290    def _HandleCommandGit(self, in_args):
291        """Handle execution of a git command
292
293        This uses a hacked-up parser.
294
295        Args:
296            in_args: Arguments after 'git' from the command line
297        """
298        git_args = []           # Top-level arguments to git itself
299        sub_cmd = None          # Git sub-command selected
300        args = []               # Arguments to the git sub-command
301        for arg in in_args:
302            if sub_cmd:
303                args.append(arg)
304            elif arg[0] == '-':
305                git_args.append(arg)
306            else:
307                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
308                    git_args.append(arg)
309                else:
310                    sub_cmd = arg
311        if sub_cmd == 'config':
312            return self._HandleCommandGitConfig(args)
313        elif sub_cmd == 'log':
314            return self._HandleCommandGitLog(args)
315        elif sub_cmd == 'clone':
316            return command.CommandResult(return_code=0)
317        elif sub_cmd == 'checkout':
318            return command.CommandResult(return_code=0)
319
320        # Not handled, so abort
321        print 'git', git_args, sub_cmd, args
322        sys.exit(1)
323
324    def _HandleCommandNm(self, args):
325        return command.CommandResult(return_code=0)
326
327    def _HandleCommandObjdump(self, args):
328        return command.CommandResult(return_code=0)
329
330    def _HandleCommandSize(self, args):
331        return command.CommandResult(return_code=0)
332
333    def _HandleCommand(self, **kwargs):
334        """Handle a command execution.
335
336        The command is in kwargs['pipe-list'], as a list of pipes, each a
337        list of commands. The command should be emulated as required for
338        testing purposes.
339
340        Returns:
341            A CommandResult object
342        """
343        pipe_list = kwargs['pipe_list']
344        wc = False
345        if len(pipe_list) != 1:
346            if pipe_list[1] == ['wc', '-l']:
347                wc = True
348            else:
349                print 'invalid pipe', kwargs
350                sys.exit(1)
351        cmd = pipe_list[0][0]
352        args = pipe_list[0][1:]
353        result = None
354        if cmd == 'git':
355            result = self._HandleCommandGit(args)
356        elif cmd == './scripts/show-gnu-make':
357            return command.CommandResult(return_code=0, stdout='make')
358        elif cmd.endswith('nm'):
359            return self._HandleCommandNm(args)
360        elif cmd.endswith('objdump'):
361            return self._HandleCommandObjdump(args)
362        elif cmd.endswith( 'size'):
363            return self._HandleCommandSize(args)
364
365        if not result:
366            # Not handled, so abort
367            print 'unknown command', kwargs
368            sys.exit(1)
369
370        if wc:
371            result.stdout = len(result.stdout.splitlines())
372        return result
373
374    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
375        """Handle execution of 'make'
376
377        Args:
378            commit: Commit object that is being built
379            brd: Board object that is being built
380            stage: Stage that we are at (mrproper, config, build)
381            cwd: Directory where make should be run
382            args: Arguments to pass to make
383            kwargs: Arguments to pass to command.RunPipe()
384        """
385        self._make_calls += 1
386        if stage == 'mrproper':
387            return command.CommandResult(return_code=0)
388        elif stage == 'config':
389            return command.CommandResult(return_code=0,
390                    combined='Test configuration complete')
391        elif stage == 'build':
392            stderr = ''
393            if type(commit) is not str:
394                stderr = self._error.get((brd.target, commit.sequence))
395            if stderr:
396                return command.CommandResult(return_code=1, stderr=stderr)
397            return command.CommandResult(return_code=0)
398
399        # Not handled, so abort
400        print 'make', stage
401        sys.exit(1)
402
403    # Example function to print output lines
404    def print_lines(self, lines):
405        print len(lines)
406        for line in lines:
407            print line
408        #self.print_lines(terminal.GetPrintTestLines())
409
410    def testNoBoards(self):
411        """Test that buildman aborts when there are no boards"""
412        self._boards = board.Boards()
413        with self.assertRaises(SystemExit):
414            self._RunControl()
415
416    def testCurrentSource(self):
417        """Very simple test to invoke buildman on the current source"""
418        self.setupToolchains();
419        self._RunControl()
420        lines = terminal.GetPrintTestLines()
421        self.assertIn('Building current source for %d boards' % len(boards),
422                      lines[0].text)
423
424    def testBadBranch(self):
425        """Test that we can detect an invalid branch"""
426        with self.assertRaises(ValueError):
427            self._RunControl('-b', 'badbranch')
428
429    def testBadToolchain(self):
430        """Test that missing toolchains are detected"""
431        self.setupToolchains();
432        ret_code = self._RunControl('-b', TEST_BRANCH)
433        lines = terminal.GetPrintTestLines()
434
435        # Buildman always builds the upstream commit as well
436        self.assertIn('Building %d commits for %d boards' %
437                (self._commits, len(boards)), lines[0].text)
438        self.assertEqual(self._builder.count, self._total_builds)
439
440        # Only sandbox should succeed, the others don't have toolchains
441        self.assertEqual(self._builder.fail,
442                         self._total_builds - self._commits)
443        self.assertEqual(ret_code, 128)
444
445        for commit in range(self._commits):
446            for board in self._boards.GetList():
447                if board.arch != 'sandbox':
448                  errfile = self._builder.GetErrFile(commit, board.target)
449                  fd = open(errfile)
450                  self.assertEqual(fd.readlines(),
451                          ['No tool chain for %s\n' % board.arch])
452                  fd.close()
453
454    def testBranch(self):
455        """Test building a branch with all toolchains present"""
456        self._RunControl('-b', TEST_BRANCH)
457        self.assertEqual(self._builder.count, self._total_builds)
458        self.assertEqual(self._builder.fail, 0)
459
460    def testCount(self):
461        """Test building a specific number of commitst"""
462        self._RunControl('-b', TEST_BRANCH, '-c2')
463        self.assertEqual(self._builder.count, 2 * len(boards))
464        self.assertEqual(self._builder.fail, 0)
465        # Each board has a mrproper, config, and then one make per commit
466        self.assertEqual(self._make_calls, len(boards) * (2 + 2))
467
468    def testIncremental(self):
469        """Test building a branch twice - the second time should do nothing"""
470        self._RunControl('-b', TEST_BRANCH)
471
472        # Each board has a mrproper, config, and then one make per commit
473        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
474        self._make_calls = 0
475        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
476        self.assertEqual(self._make_calls, 0)
477        self.assertEqual(self._builder.count, self._total_builds)
478        self.assertEqual(self._builder.fail, 0)
479
480    def testForceBuild(self):
481        """The -f flag should force a rebuild"""
482        self._RunControl('-b', TEST_BRANCH)
483        self._make_calls = 0
484        self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
485        # Each board has a mrproper, config, and then one make per commit
486        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
487
488    def testForceReconfigure(self):
489        """The -f flag should force a rebuild"""
490        self._RunControl('-b', TEST_BRANCH, '-C')
491        # Each commit has a mrproper, config and make
492        self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
493
494    def testErrors(self):
495        """Test handling of build errors"""
496        self._error['board2', 1] = 'fred\n'
497        self._RunControl('-b', TEST_BRANCH)
498        self.assertEqual(self._builder.count, self._total_builds)
499        self.assertEqual(self._builder.fail, 1)
500
501        # Remove the error. This should have no effect since the commit will
502        # not be rebuilt
503        del self._error['board2', 1]
504        self._make_calls = 0
505        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
506        self.assertEqual(self._builder.count, self._total_builds)
507        self.assertEqual(self._make_calls, 0)
508        self.assertEqual(self._builder.fail, 1)
509
510        # Now use the -F flag to force rebuild of the bad commit
511        self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
512        self.assertEqual(self._builder.count, self._total_builds)
513        self.assertEqual(self._builder.fail, 0)
514        self.assertEqual(self._make_calls, 3)
515
516    def testBranchWithSlash(self):
517        """Test building a branch with a '/' in the name"""
518        self._test_branch = '/__dev/__testbranch'
519        self._RunControl('-b', self._test_branch, clean_dir=False)
520        self.assertEqual(self._builder.count, self._total_builds)
521        self.assertEqual(self._builder.fail, 0)
522