xref: /openbmc/u-boot/tools/buildman/func_test.py (revision 8ee59472) 
1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2014 Google, Inc
3#
4
5import os
6import shutil
7import sys
8import tempfile
9import unittest
10
11import board
12import bsettings
13import cmdline
14import command
15import control
16import gitutil
17import terminal
18import toolchain
19
20settings_data = '''
21# Buildman settings file
22
23[toolchain]
24
25[toolchain-alias]
26
27[make-flags]
28src=/home/sjg/c/src
29chroot=/home/sjg/c/chroot
30vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
31chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
32chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
33chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
34'''
35
36boards = [
37    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
38    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
39    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
40    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
41]
42
43commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4439403bb patman: Use --no-pager' to stop git from forking a pager
45db6e6f2 patman: Remove the -a option
46f2ccf03 patman: Correct unit tests to run correctly
471d097f9 patman: Fix indentation in terminal.py
48d073747 patman: Support the 'reverse' option for 'git log
49"""
50
51commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
52Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
53Date:   Fri Aug 22 19:12:41 2014 +0900
54
55    buildman: refactor help message
56
57    "buildman [options]" is displayed by default.
58
59    Append the rest of help messages to parser.usage
60    instead of replacing it.
61
62    Besides, "-b <branch>" is not mandatory since commit fea5858e.
63    Drop it from the usage.
64
65    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
66""",
67"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
68Author: Simon Glass <sjg@chromium.org>
69Date:   Thu Aug 14 16:48:25 2014 -0600
70
71    patman: Support the 'reverse' option for 'git log'
72
73    This option is currently not supported, but needs to be, for buildman to
74    operate as expected.
75
76    Series-changes: 7
77    - Add new patch to fix the 'reverse' bug
78
79    Series-version: 8
80
81    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
82    Reported-by: York Sun <yorksun@freescale.com>
83    Signed-off-by: Simon Glass <sjg@chromium.org>
84
85""",
86"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
87Author: Simon Glass <sjg@chromium.org>
88Date:   Sat Aug 9 11:44:32 2014 -0600
89
90    patman: Fix indentation in terminal.py
91
92    This code came from a different project with 2-character indentation. Fix
93    it for U-Boot.
94
95    Series-changes: 6
96    - Add new patch to fix indentation in teminal.py
97
98    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
99    Signed-off-by: Simon Glass <sjg@chromium.org>
100
101""",
102"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
103Author: Simon Glass <sjg@chromium.org>
104Date:   Sat Aug 9 11:08:24 2014 -0600
105
106    patman: Correct unit tests to run correctly
107
108    It seems that doctest behaves differently now, and some of the unit tests
109    do not run. Adjust the tests to work correctly.
110
111     ./tools/patman/patman --test
112    <unittest.result.TestResult run=10 errors=0 failures=0>
113
114    Series-changes: 6
115    - Add new patch to fix patman unit tests
116
117    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
118
119""",
120"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
121Author: Simon Glass <sjg@chromium.org>
122Date:   Sat Aug 9 12:06:02 2014 -0600
123
124    patman: Remove the -a option
125
126    It seems that this is no longer needed, since checkpatch.pl will catch
127    whitespace problems in patches. Also the option is not widely used, so
128    it seems safe to just remove it.
129
130    Series-changes: 6
131    - Add new patch to remove patman's -a option
132
133    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
134    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
135
136""",
137"""commit 39403bb4f838153028a6f21ca30bf100f3791133
138Author: Simon Glass <sjg@chromium.org>
139Date:   Thu Aug 14 21:50:52 2014 -0600
140
141    patman: Use --no-pager' to stop git from forking a pager
142
143""",
144"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
145Author: Simon Glass <sjg@chromium.org>
146Date:   Fri Aug 22 15:57:39 2014 -0600
147
148    patman: Avoid changing the order of tags
149
150    patman collects tags that it sees in the commit and places them nicely
151    sorted at the end of the patch. However, this is not really necessary and
152    in fact is apparently not desirable.
153
154    Series-changes: 9
155    - Add new patch to avoid changing the order of tags
156
157    Series-version: 9
158
159    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
160    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
161"""]
162
163TEST_BRANCH = '__testbranch'
164
165class TestFunctional(unittest.TestCase):
166    """Functional test for buildman.
167
168    This aims to test from just below the invocation of buildman (parsing
169    of arguments) to 'make' and 'git' invocation. It is not a true
170    emd-to-end test, as it mocks git, make and the tool chain. But this
171    makes it easier to detect when the builder is doing the wrong thing,
172    since in many cases this test code will fail. For example, only a
173    very limited subset of 'git' arguments is supported - anything
174    unexpected will fail.
175    """
176    def setUp(self):
177        self._base_dir = tempfile.mkdtemp()
178        self._git_dir = os.path.join(self._base_dir, 'src')
179        self._buildman_pathname = sys.argv[0]
180        self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
181        command.test_result = self._HandleCommand
182        self.setupToolchains()
183        self._toolchains.Add('arm-gcc', test=False)
184        self._toolchains.Add('powerpc-gcc', test=False)
185        bsettings.Setup(None)
186        bsettings.AddFile(settings_data)
187        self._boards = board.Boards()
188        for brd in boards:
189            self._boards.AddBoard(board.Board(*brd))
190
191        # Directories where the source been cloned
192        self._clone_dirs = []
193        self._commits = len(commit_shortlog.splitlines()) + 1
194        self._total_builds = self._commits * len(boards)
195
196        # Number of calls to make
197        self._make_calls = 0
198
199        # Map of [board, commit] to error messages
200        self._error = {}
201
202        self._test_branch = TEST_BRANCH
203
204        # Avoid sending any output and clear all terminal output
205        terminal.SetPrintTestMode()
206        terminal.GetPrintTestLines()
207
208    def tearDown(self):
209        shutil.rmtree(self._base_dir)
210
211    def setupToolchains(self):
212        self._toolchains = toolchain.Toolchains()
213        self._toolchains.Add('gcc', test=False)
214
215    def _RunBuildman(self, *args):
216        return command.RunPipe([[self._buildman_pathname] + list(args)],
217                capture=True, capture_stderr=True)
218
219    def _RunControl(self, *args, **kwargs):
220        sys.argv = [sys.argv[0]] + list(args)
221        options, args = cmdline.ParseArgs()
222        result = control.DoBuildman(options, args, toolchains=self._toolchains,
223                make_func=self._HandleMake, boards=self._boards,
224                clean_dir=kwargs.get('clean_dir', True))
225        self._builder = control.builder
226        return result
227
228    def testFullHelp(self):
229        command.test_result = None
230        result = self._RunBuildman('-H')
231        help_file = os.path.join(self._buildman_dir, 'README')
232        # Remove possible extraneous strings
233        extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
234        gothelp = result.stdout.replace(extra, '')
235        self.assertEqual(len(gothelp), os.path.getsize(help_file))
236        self.assertEqual(0, len(result.stderr))
237        self.assertEqual(0, result.return_code)
238
239    def testHelp(self):
240        command.test_result = None
241        result = self._RunBuildman('-h')
242        help_file = os.path.join(self._buildman_dir, 'README')
243        self.assertTrue(len(result.stdout) > 1000)
244        self.assertEqual(0, len(result.stderr))
245        self.assertEqual(0, result.return_code)
246
247    def testGitSetup(self):
248        """Test gitutils.Setup(), from outside the module itself"""
249        command.test_result = command.CommandResult(return_code=1)
250        gitutil.Setup()
251        self.assertEqual(gitutil.use_no_decorate, False)
252
253        command.test_result = command.CommandResult(return_code=0)
254        gitutil.Setup()
255        self.assertEqual(gitutil.use_no_decorate, True)
256
257    def _HandleCommandGitLog(self, args):
258        if args[-1] == '--':
259            args = args[:-1]
260        if '-n0' in args:
261            return command.CommandResult(return_code=0)
262        elif args[-1] == 'upstream/master..%s' % self._test_branch:
263            return command.CommandResult(return_code=0, stdout=commit_shortlog)
264        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
265            if args[-1] == self._test_branch:
266                count = int(args[3][2:])
267                return command.CommandResult(return_code=0,
268                                            stdout=''.join(commit_log[:count]))
269
270        # Not handled, so abort
271        print 'git log', args
272        sys.exit(1)
273
274    def _HandleCommandGitConfig(self, args):
275        config = args[0]
276        if config == 'sendemail.aliasesfile':
277            return command.CommandResult(return_code=0)
278        elif config.startswith('branch.badbranch'):
279            return command.CommandResult(return_code=1)
280        elif config == 'branch.%s.remote' % self._test_branch:
281            return command.CommandResult(return_code=0, stdout='upstream\n')
282        elif config == 'branch.%s.merge' % self._test_branch:
283            return command.CommandResult(return_code=0,
284                                         stdout='refs/heads/master\n')
285
286        # Not handled, so abort
287        print 'git config', args
288        sys.exit(1)
289
290    def _HandleCommandGit(self, in_args):
291        """Handle execution of a git command
292
293        This uses a hacked-up parser.
294
295        Args:
296            in_args: Arguments after 'git' from the command line
297        """
298        git_args = []           # Top-level arguments to git itself
299        sub_cmd = None          # Git sub-command selected
300        args = []               # Arguments to the git sub-command
301        for arg in in_args:
302            if sub_cmd:
303                args.append(arg)
304            elif arg[0] == '-':
305                git_args.append(arg)
306            else:
307                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
308                    git_args.append(arg)
309                else:
310                    sub_cmd = arg
311        if sub_cmd == 'config':
312            return self._HandleCommandGitConfig(args)
313        elif sub_cmd == 'log':
314            return self._HandleCommandGitLog(args)
315        elif sub_cmd == 'clone':
316            return command.CommandResult(return_code=0)
317        elif sub_cmd == 'checkout':
318            return command.CommandResult(return_code=0)
319
320        # Not handled, so abort
321        print 'git', git_args, sub_cmd, args
322        sys.exit(1)
323
324    def _HandleCommandNm(self, args):
325        return command.CommandResult(return_code=0)
326
327    def _HandleCommandObjdump(self, args):
328        return command.CommandResult(return_code=0)
329
330    def _HandleCommandSize(self, args):
331        return command.CommandResult(return_code=0)
332
333    def _HandleCommand(self, **kwargs):
334        """Handle a command execution.
335
336        The command is in kwargs['pipe-list'], as a list of pipes, each a
337        list of commands. The command should be emulated as required for
338        testing purposes.
339
340        Returns:
341            A CommandResult object
342        """
343        pipe_list = kwargs['pipe_list']
344        wc = False
345        if len(pipe_list) != 1:
346            if pipe_list[1] == ['wc', '-l']:
347                wc = True
348            else:
349                print 'invalid pipe', kwargs
350                sys.exit(1)
351        cmd = pipe_list[0][0]
352        args = pipe_list[0][1:]
353        result = None
354        if cmd == 'git':
355            result = self._HandleCommandGit(args)
356        elif cmd == './scripts/show-gnu-make':
357            return command.CommandResult(return_code=0, stdout='make')
358        elif cmd.endswith('nm'):
359            return self._HandleCommandNm(args)
360        elif cmd.endswith('objdump'):
361            return self._HandleCommandObjdump(args)
362        elif cmd.endswith( 'size'):
363            return self._HandleCommandSize(args)
364
365        if not result:
366            # Not handled, so abort
367            print 'unknown command', kwargs
368            sys.exit(1)
369
370        if wc:
371            result.stdout = len(result.stdout.splitlines())
372        return result
373
374    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
375        """Handle execution of 'make'
376
377        Args:
378            commit: Commit object that is being built
379            brd: Board object that is being built
380            stage: Stage that we are at (mrproper, config, build)
381            cwd: Directory where make should be run
382            args: Arguments to pass to make
383            kwargs: Arguments to pass to command.RunPipe()
384        """
385        self._make_calls += 1
386        if stage == 'mrproper':
387            return command.CommandResult(return_code=0)
388        elif stage == 'config':
389            return command.CommandResult(return_code=0,
390                    combined='Test configuration complete')
391        elif stage == 'build':
392            stderr = ''
393            if type(commit) is not str:
394                stderr = self._error.get((brd.target, commit.sequence))
395            if stderr:
396                return command.CommandResult(return_code=1, stderr=stderr)
397            return command.CommandResult(return_code=0)
398
399        # Not handled, so abort
400        print 'make', stage
401        sys.exit(1)
402
403    # Example function to print output lines
404    def print_lines(self, lines):
405        print len(lines)
406        for line in lines:
407            print line
408        #self.print_lines(terminal.GetPrintTestLines())
409
410    def testNoBoards(self):
411        """Test that buildman aborts when there are no boards"""
412        self._boards = board.Boards()
413        with self.assertRaises(SystemExit):
414            self._RunControl()
415
416    def testCurrentSource(self):
417        """Very simple test to invoke buildman on the current source"""
418        self.setupToolchains();
419        self._RunControl()
420        lines = terminal.GetPrintTestLines()
421        self.assertIn('Building current source for %d boards' % len(boards),
422                      lines[0].text)
423
424    def testBadBranch(self):
425        """Test that we can detect an invalid branch"""
426        with self.assertRaises(ValueError):
427            self._RunControl('-b', 'badbranch')
428
429    def testBadToolchain(self):
430        """Test that missing toolchains are detected"""
431        self.setupToolchains();
432        ret_code = self._RunControl('-b', TEST_BRANCH)
433        lines = terminal.GetPrintTestLines()
434
435        # Buildman always builds the upstream commit as well
436        self.assertIn('Building %d commits for %d boards' %
437                (self._commits, len(boards)), lines[0].text)
438        self.assertEqual(self._builder.count, self._total_builds)
439
440        # Only sandbox should succeed, the others don't have toolchains
441        self.assertEqual(self._builder.fail,
442                         self._total_builds - self._commits)
443        self.assertEqual(ret_code, 128)
444
445        for commit in range(self._commits):
446            for board in self._boards.GetList():
447                if board.arch != 'sandbox':
448                  errfile = self._builder.GetErrFile(commit, board.target)
449                  fd = open(errfile)
450                  self.assertEqual(fd.readlines(),
451                          ['No tool chain for %s\n' % board.arch])
452                  fd.close()
453
454    def testBranch(self):
455        """Test building a branch with all toolchains present"""
456        self._RunControl('-b', TEST_BRANCH)
457        self.assertEqual(self._builder.count, self._total_builds)
458        self.assertEqual(self._builder.fail, 0)
459
460    def testCount(self):
461        """Test building a specific number of commitst"""
462        self._RunControl('-b', TEST_BRANCH, '-c2')
463        self.assertEqual(self._builder.count, 2 * len(boards))
464        self.assertEqual(self._builder.fail, 0)
465        # Each board has a mrproper, config, and then one make per commit
466        self.assertEqual(self._make_calls, len(boards) * (2 + 2))
467
468    def testIncremental(self):
469        """Test building a branch twice - the second time should do nothing"""
470        self._RunControl('-b', TEST_BRANCH)
471
472        # Each board has a mrproper, config, and then one make per commit
473        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
474        self._make_calls = 0
475        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
476        self.assertEqual(self._make_calls, 0)
477        self.assertEqual(self._builder.count, self._total_builds)
478        self.assertEqual(self._builder.fail, 0)
479
480    def testForceBuild(self):
481        """The -f flag should force a rebuild"""
482        self._RunControl('-b', TEST_BRANCH)
483        self._make_calls = 0
484        self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
485        # Each board has a mrproper, config, and then one make per commit
486        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
487
488    def testForceReconfigure(self):
489        """The -f flag should force a rebuild"""
490        self._RunControl('-b', TEST_BRANCH, '-C')
491        # Each commit has a mrproper, config and make
492        self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
493
494    def testErrors(self):
495        """Test handling of build errors"""
496        self._error['board2', 1] = 'fred\n'
497        self._RunControl('-b', TEST_BRANCH)
498        self.assertEqual(self._builder.count, self._total_builds)
499        self.assertEqual(self._builder.fail, 1)
500
501        # Remove the error. This should have no effect since the commit will
502        # not be rebuilt
503        del self._error['board2', 1]
504        self._make_calls = 0
505        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
506        self.assertEqual(self._builder.count, self._total_builds)
507        self.assertEqual(self._make_calls, 0)
508        self.assertEqual(self._builder.fail, 1)
509
510        # Now use the -F flag to force rebuild of the bad commit
511        self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
512        self.assertEqual(self._builder.count, self._total_builds)
513        self.assertEqual(self._builder.fail, 0)
514        self.assertEqual(self._make_calls, 3)
515
516    def testBranchWithSlash(self):
517        """Test building a branch with a '/' in the name"""
518        self._test_branch = '/__dev/__testbranch'
519        self._RunControl('-b', self._test_branch, clean_dir=False)
520        self.assertEqual(self._builder.count, self._total_builds)
521        self.assertEqual(self._builder.fail, 0)
522
523    def testBadOutputDir(self):
524        """Test building with an output dir the same as out current dir"""
525        self._test_branch = '/__dev/__testbranch'
526        with self.assertRaises(SystemExit):
527            self._RunControl('-b', self._test_branch, '-o', os.getcwd())
528        with self.assertRaises(SystemExit):
529            self._RunControl('-b', self._test_branch, '-o',
530                             os.path.join(os.getcwd(), 'test'))
531