xref: /openbmc/u-boot/tools/buildman/func_test.py (revision 29b103c7) 
1#
2# Copyright (c) 2014 Google, Inc
3#
4# SPDX-License-Identifier:      GPL-2.0+
5#
6
7import os
8import shutil
9import sys
10import tempfile
11import unittest
12
13import board
14import bsettings
15import cmdline
16import command
17import control
18import gitutil
19import terminal
20import toolchain
21
22settings_data = '''
23# Buildman settings file
24
25[toolchain]
26
27[toolchain-alias]
28
29[make-flags]
30src=/home/sjg/c/src
31chroot=/home/sjg/c/chroot
32vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
33chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
34chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
35chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
36'''
37
38boards = [
39    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
40    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
41    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
42    ['Active', 'powerpc', 'mpc5xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
43    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
44]
45
46commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4739403bb patman: Use --no-pager' to stop git from forking a pager
48db6e6f2 patman: Remove the -a option
49f2ccf03 patman: Correct unit tests to run correctly
501d097f9 patman: Fix indentation in terminal.py
51d073747 patman: Support the 'reverse' option for 'git log
52"""
53
54commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
55Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
56Date:   Fri Aug 22 19:12:41 2014 +0900
57
58    buildman: refactor help message
59
60    "buildman [options]" is displayed by default.
61
62    Append the rest of help messages to parser.usage
63    instead of replacing it.
64
65    Besides, "-b <branch>" is not mandatory since commit fea5858e.
66    Drop it from the usage.
67
68    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
69""",
70"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
71Author: Simon Glass <sjg@chromium.org>
72Date:   Thu Aug 14 16:48:25 2014 -0600
73
74    patman: Support the 'reverse' option for 'git log'
75
76    This option is currently not supported, but needs to be, for buildman to
77    operate as expected.
78
79    Series-changes: 7
80    - Add new patch to fix the 'reverse' bug
81
82    Series-version: 8
83
84    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
85    Reported-by: York Sun <yorksun@freescale.com>
86    Signed-off-by: Simon Glass <sjg@chromium.org>
87
88""",
89"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
90Author: Simon Glass <sjg@chromium.org>
91Date:   Sat Aug 9 11:44:32 2014 -0600
92
93    patman: Fix indentation in terminal.py
94
95    This code came from a different project with 2-character indentation. Fix
96    it for U-Boot.
97
98    Series-changes: 6
99    - Add new patch to fix indentation in teminal.py
100
101    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
102    Signed-off-by: Simon Glass <sjg@chromium.org>
103
104""",
105"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
106Author: Simon Glass <sjg@chromium.org>
107Date:   Sat Aug 9 11:08:24 2014 -0600
108
109    patman: Correct unit tests to run correctly
110
111    It seems that doctest behaves differently now, and some of the unit tests
112    do not run. Adjust the tests to work correctly.
113
114     ./tools/patman/patman --test
115    <unittest.result.TestResult run=10 errors=0 failures=0>
116
117    Series-changes: 6
118    - Add new patch to fix patman unit tests
119
120    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
121
122""",
123"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
124Author: Simon Glass <sjg@chromium.org>
125Date:   Sat Aug 9 12:06:02 2014 -0600
126
127    patman: Remove the -a option
128
129    It seems that this is no longer needed, since checkpatch.pl will catch
130    whitespace problems in patches. Also the option is not widely used, so
131    it seems safe to just remove it.
132
133    Series-changes: 6
134    - Add new patch to remove patman's -a option
135
136    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
137    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
138
139""",
140"""commit 39403bb4f838153028a6f21ca30bf100f3791133
141Author: Simon Glass <sjg@chromium.org>
142Date:   Thu Aug 14 21:50:52 2014 -0600
143
144    patman: Use --no-pager' to stop git from forking a pager
145
146""",
147"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
148Author: Simon Glass <sjg@chromium.org>
149Date:   Fri Aug 22 15:57:39 2014 -0600
150
151    patman: Avoid changing the order of tags
152
153    patman collects tags that it sees in the commit and places them nicely
154    sorted at the end of the patch. However, this is not really necessary and
155    in fact is apparently not desirable.
156
157    Series-changes: 9
158    - Add new patch to avoid changing the order of tags
159
160    Series-version: 9
161
162    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
163    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
164"""]
165
166TEST_BRANCH = '__testbranch'
167
168class TestFunctional(unittest.TestCase):
169    """Functional test for buildman.
170
171    This aims to test from just below the invocation of buildman (parsing
172    of arguments) to 'make' and 'git' invocation. It is not a true
173    emd-to-end test, as it mocks git, make and the tool chain. But this
174    makes it easier to detect when the builder is doing the wrong thing,
175    since in many cases this test code will fail. For example, only a
176    very limited subset of 'git' arguments is supported - anything
177    unexpected will fail.
178    """
179    def setUp(self):
180        self._base_dir = tempfile.mkdtemp()
181        self._git_dir = os.path.join(self._base_dir, 'src')
182        self._buildman_pathname = sys.argv[0]
183        self._buildman_dir = os.path.dirname(sys.argv[0])
184        command.test_result = self._HandleCommand
185        self.setupToolchains()
186        self._toolchains.Add('arm-gcc', test=False)
187        self._toolchains.Add('powerpc-gcc', test=False)
188        bsettings.Setup(None)
189        bsettings.AddFile(settings_data)
190        self._boards = board.Boards()
191        for brd in boards:
192            self._boards.AddBoard(board.Board(*brd))
193
194        # Directories where the source been cloned
195        self._clone_dirs = []
196        self._commits = len(commit_shortlog.splitlines()) + 1
197        self._total_builds = self._commits * len(boards)
198
199        # Number of calls to make
200        self._make_calls = 0
201
202        # Map of [board, commit] to error messages
203        self._error = {}
204
205        self._test_branch = TEST_BRANCH
206
207        # Avoid sending any output and clear all terminal output
208        terminal.SetPrintTestMode()
209        terminal.GetPrintTestLines()
210
211    def tearDown(self):
212        shutil.rmtree(self._base_dir)
213
214    def setupToolchains(self):
215        self._toolchains = toolchain.Toolchains()
216        self._toolchains.Add('gcc', test=False)
217
218    def _RunBuildman(self, *args):
219        return command.RunPipe([[self._buildman_pathname] + list(args)],
220                capture=True, capture_stderr=True)
221
222    def _RunControl(self, *args, **kwargs):
223        sys.argv = [sys.argv[0]] + list(args)
224        options, args = cmdline.ParseArgs()
225        result = control.DoBuildman(options, args, toolchains=self._toolchains,
226                make_func=self._HandleMake, boards=self._boards,
227                clean_dir=kwargs.get('clean_dir', True))
228        self._builder = control.builder
229        return result
230
231    def testFullHelp(self):
232        command.test_result = None
233        result = self._RunBuildman('-H')
234        help_file = os.path.join(self._buildman_dir, 'README')
235        self.assertEqual(len(result.stdout), os.path.getsize(help_file))
236        self.assertEqual(0, len(result.stderr))
237        self.assertEqual(0, result.return_code)
238
239    def testHelp(self):
240        command.test_result = None
241        result = self._RunBuildman('-h')
242        help_file = os.path.join(self._buildman_dir, 'README')
243        self.assertTrue(len(result.stdout) > 1000)
244        self.assertEqual(0, len(result.stderr))
245        self.assertEqual(0, result.return_code)
246
247    def testGitSetup(self):
248        """Test gitutils.Setup(), from outside the module itself"""
249        command.test_result = command.CommandResult(return_code=1)
250        gitutil.Setup()
251        self.assertEqual(gitutil.use_no_decorate, False)
252
253        command.test_result = command.CommandResult(return_code=0)
254        gitutil.Setup()
255        self.assertEqual(gitutil.use_no_decorate, True)
256
257    def _HandleCommandGitLog(self, args):
258        if '-n0' in args:
259            return command.CommandResult(return_code=0)
260        elif args[-1] == 'upstream/master..%s' % self._test_branch:
261            return command.CommandResult(return_code=0, stdout=commit_shortlog)
262        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
263            if args[-1] == self._test_branch:
264                count = int(args[3][2:])
265                return command.CommandResult(return_code=0,
266                                            stdout=''.join(commit_log[:count]))
267
268        # Not handled, so abort
269        print 'git log', args
270        sys.exit(1)
271
272    def _HandleCommandGitConfig(self, args):
273        config = args[0]
274        if config == 'sendemail.aliasesfile':
275            return command.CommandResult(return_code=0)
276        elif config.startswith('branch.badbranch'):
277            return command.CommandResult(return_code=1)
278        elif config == 'branch.%s.remote' % self._test_branch:
279            return command.CommandResult(return_code=0, stdout='upstream\n')
280        elif config == 'branch.%s.merge' % self._test_branch:
281            return command.CommandResult(return_code=0,
282                                         stdout='refs/heads/master\n')
283
284        # Not handled, so abort
285        print 'git config', args
286        sys.exit(1)
287
288    def _HandleCommandGit(self, in_args):
289        """Handle execution of a git command
290
291        This uses a hacked-up parser.
292
293        Args:
294            in_args: Arguments after 'git' from the command line
295        """
296        git_args = []           # Top-level arguments to git itself
297        sub_cmd = None          # Git sub-command selected
298        args = []               # Arguments to the git sub-command
299        for arg in in_args:
300            if sub_cmd:
301                args.append(arg)
302            elif arg[0] == '-':
303                git_args.append(arg)
304            else:
305                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
306                    git_args.append(arg)
307                else:
308                    sub_cmd = arg
309        if sub_cmd == 'config':
310            return self._HandleCommandGitConfig(args)
311        elif sub_cmd == 'log':
312            return self._HandleCommandGitLog(args)
313        elif sub_cmd == 'clone':
314            return command.CommandResult(return_code=0)
315        elif sub_cmd == 'checkout':
316            return command.CommandResult(return_code=0)
317
318        # Not handled, so abort
319        print 'git', git_args, sub_cmd, args
320        sys.exit(1)
321
322    def _HandleCommandNm(self, args):
323        return command.CommandResult(return_code=0)
324
325    def _HandleCommandObjdump(self, args):
326        return command.CommandResult(return_code=0)
327
328    def _HandleCommandSize(self, args):
329        return command.CommandResult(return_code=0)
330
331    def _HandleCommand(self, **kwargs):
332        """Handle a command execution.
333
334        The command is in kwargs['pipe-list'], as a list of pipes, each a
335        list of commands. The command should be emulated as required for
336        testing purposes.
337
338        Returns:
339            A CommandResult object
340        """
341        pipe_list = kwargs['pipe_list']
342        wc = False
343        if len(pipe_list) != 1:
344            if pipe_list[1] == ['wc', '-l']:
345                wc = True
346            else:
347                print 'invalid pipe', kwargs
348                sys.exit(1)
349        cmd = pipe_list[0][0]
350        args = pipe_list[0][1:]
351        result = None
352        if cmd == 'git':
353            result = self._HandleCommandGit(args)
354        elif cmd == './scripts/show-gnu-make':
355            return command.CommandResult(return_code=0, stdout='make')
356        elif cmd.endswith('nm'):
357            return self._HandleCommandNm(args)
358        elif cmd.endswith('objdump'):
359            return self._HandleCommandObjdump(args)
360        elif cmd.endswith( 'size'):
361            return self._HandleCommandSize(args)
362
363        if not result:
364            # Not handled, so abort
365            print 'unknown command', kwargs
366            sys.exit(1)
367
368        if wc:
369            result.stdout = len(result.stdout.splitlines())
370        return result
371
372    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
373        """Handle execution of 'make'
374
375        Args:
376            commit: Commit object that is being built
377            brd: Board object that is being built
378            stage: Stage that we are at (mrproper, config, build)
379            cwd: Directory where make should be run
380            args: Arguments to pass to make
381            kwargs: Arguments to pass to command.RunPipe()
382        """
383        self._make_calls += 1
384        if stage == 'mrproper':
385            return command.CommandResult(return_code=0)
386        elif stage == 'config':
387            return command.CommandResult(return_code=0,
388                    combined='Test configuration complete')
389        elif stage == 'build':
390            stderr = ''
391            if type(commit) is not str:
392                stderr = self._error.get((brd.target, commit.sequence))
393            if stderr:
394                return command.CommandResult(return_code=1, stderr=stderr)
395            return command.CommandResult(return_code=0)
396
397        # Not handled, so abort
398        print 'make', stage
399        sys.exit(1)
400
401    # Example function to print output lines
402    def print_lines(self, lines):
403        print len(lines)
404        for line in lines:
405            print line
406        #self.print_lines(terminal.GetPrintTestLines())
407
408    def testNoBoards(self):
409        """Test that buildman aborts when there are no boards"""
410        self._boards = board.Boards()
411        with self.assertRaises(SystemExit):
412            self._RunControl()
413
414    def testCurrentSource(self):
415        """Very simple test to invoke buildman on the current source"""
416        self.setupToolchains();
417        self._RunControl()
418        lines = terminal.GetPrintTestLines()
419        self.assertIn('Building current source for %d boards' % len(boards),
420                      lines[0].text)
421
422    def testBadBranch(self):
423        """Test that we can detect an invalid branch"""
424        with self.assertRaises(ValueError):
425            self._RunControl('-b', 'badbranch')
426
427    def testBadToolchain(self):
428        """Test that missing toolchains are detected"""
429        self.setupToolchains();
430        ret_code = self._RunControl('-b', TEST_BRANCH)
431        lines = terminal.GetPrintTestLines()
432
433        # Buildman always builds the upstream commit as well
434        self.assertIn('Building %d commits for %d boards' %
435                (self._commits, len(boards)), lines[0].text)
436        self.assertEqual(self._builder.count, self._total_builds)
437
438        # Only sandbox should succeed, the others don't have toolchains
439        self.assertEqual(self._builder.fail,
440                         self._total_builds - self._commits)
441        self.assertEqual(ret_code, 128)
442
443        for commit in range(self._commits):
444            for board in self._boards.GetList():
445                if board.arch != 'sandbox':
446                  errfile = self._builder.GetErrFile(commit, board.target)
447                  fd = open(errfile)
448                  self.assertEqual(fd.readlines(),
449                          ['No tool chain for %s\n' % board.arch])
450                  fd.close()
451
452    def testBranch(self):
453        """Test building a branch with all toolchains present"""
454        self._RunControl('-b', TEST_BRANCH)
455        self.assertEqual(self._builder.count, self._total_builds)
456        self.assertEqual(self._builder.fail, 0)
457
458    def testCount(self):
459        """Test building a specific number of commitst"""
460        self._RunControl('-b', TEST_BRANCH, '-c2')
461        self.assertEqual(self._builder.count, 2 * len(boards))
462        self.assertEqual(self._builder.fail, 0)
463        # Each board has a mrproper, config, and then one make per commit
464        self.assertEqual(self._make_calls, len(boards) * (2 + 2))
465
466    def testIncremental(self):
467        """Test building a branch twice - the second time should do nothing"""
468        self._RunControl('-b', TEST_BRANCH)
469
470        # Each board has a mrproper, config, and then one make per commit
471        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
472        self._make_calls = 0
473        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
474        self.assertEqual(self._make_calls, 0)
475        self.assertEqual(self._builder.count, self._total_builds)
476        self.assertEqual(self._builder.fail, 0)
477
478    def testForceBuild(self):
479        """The -f flag should force a rebuild"""
480        self._RunControl('-b', TEST_BRANCH)
481        self._make_calls = 0
482        self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
483        # Each board has a mrproper, config, and then one make per commit
484        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
485
486    def testForceReconfigure(self):
487        """The -f flag should force a rebuild"""
488        self._RunControl('-b', TEST_BRANCH, '-C')
489        # Each commit has a mrproper, config and make
490        self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
491
492    def testErrors(self):
493        """Test handling of build errors"""
494        self._error['board2', 1] = 'fred\n'
495        self._RunControl('-b', TEST_BRANCH)
496        self.assertEqual(self._builder.count, self._total_builds)
497        self.assertEqual(self._builder.fail, 1)
498
499        # Remove the error. This should have no effect since the commit will
500        # not be rebuilt
501        del self._error['board2', 1]
502        self._make_calls = 0
503        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
504        self.assertEqual(self._builder.count, self._total_builds)
505        self.assertEqual(self._make_calls, 0)
506        self.assertEqual(self._builder.fail, 1)
507
508        # Now use the -F flag to force rebuild of the bad commit
509        self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
510        self.assertEqual(self._builder.count, self._total_builds)
511        self.assertEqual(self._builder.fail, 0)
512        self.assertEqual(self._make_calls, 3)
513
514    def testBranchWithSlash(self):
515        """Test building a branch with a '/' in the name"""
516        self._test_branch = '/__dev/__testbranch'
517        self._RunControl('-b', self._test_branch, clean_dir=False)
518        self.assertEqual(self._builder.count, self._total_builds)
519        self.assertEqual(self._builder.fail, 0)
520