xref: /openbmc/u-boot/tools/buildman/func_test.py (revision ae485b54) 
1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2014 Google, Inc
3#
4
5import os
6import shutil
7import sys
8import tempfile
9import unittest
10
11import board
12import bsettings
13import cmdline
14import command
15import control
16import gitutil
17import terminal
18import toolchain
19
20settings_data = '''
21# Buildman settings file
22
23[toolchain]
24
25[toolchain-alias]
26
27[make-flags]
28src=/home/sjg/c/src
29chroot=/home/sjg/c/chroot
30vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
31chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
32chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
33chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
34'''
35
36boards = [
37    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
38    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
39    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
40    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
41]
42
43commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4439403bb patman: Use --no-pager' to stop git from forking a pager
45db6e6f2 patman: Remove the -a option
46f2ccf03 patman: Correct unit tests to run correctly
471d097f9 patman: Fix indentation in terminal.py
48d073747 patman: Support the 'reverse' option for 'git log
49"""
50
51commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
52Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
53Date:   Fri Aug 22 19:12:41 2014 +0900
54
55    buildman: refactor help message
56
57    "buildman [options]" is displayed by default.
58
59    Append the rest of help messages to parser.usage
60    instead of replacing it.
61
62    Besides, "-b <branch>" is not mandatory since commit fea5858e.
63    Drop it from the usage.
64
65    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
66""",
67"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
68Author: Simon Glass <sjg@chromium.org>
69Date:   Thu Aug 14 16:48:25 2014 -0600
70
71    patman: Support the 'reverse' option for 'git log'
72
73    This option is currently not supported, but needs to be, for buildman to
74    operate as expected.
75
76    Series-changes: 7
77    - Add new patch to fix the 'reverse' bug
78
79    Series-version: 8
80
81    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
82    Reported-by: York Sun <yorksun@freescale.com>
83    Signed-off-by: Simon Glass <sjg@chromium.org>
84
85""",
86"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
87Author: Simon Glass <sjg@chromium.org>
88Date:   Sat Aug 9 11:44:32 2014 -0600
89
90    patman: Fix indentation in terminal.py
91
92    This code came from a different project with 2-character indentation. Fix
93    it for U-Boot.
94
95    Series-changes: 6
96    - Add new patch to fix indentation in teminal.py
97
98    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
99    Signed-off-by: Simon Glass <sjg@chromium.org>
100
101""",
102"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
103Author: Simon Glass <sjg@chromium.org>
104Date:   Sat Aug 9 11:08:24 2014 -0600
105
106    patman: Correct unit tests to run correctly
107
108    It seems that doctest behaves differently now, and some of the unit tests
109    do not run. Adjust the tests to work correctly.
110
111     ./tools/patman/patman --test
112    <unittest.result.TestResult run=10 errors=0 failures=0>
113
114    Series-changes: 6
115    - Add new patch to fix patman unit tests
116
117    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
118
119""",
120"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
121Author: Simon Glass <sjg@chromium.org>
122Date:   Sat Aug 9 12:06:02 2014 -0600
123
124    patman: Remove the -a option
125
126    It seems that this is no longer needed, since checkpatch.pl will catch
127    whitespace problems in patches. Also the option is not widely used, so
128    it seems safe to just remove it.
129
130    Series-changes: 6
131    - Add new patch to remove patman's -a option
132
133    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
134    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
135
136""",
137"""commit 39403bb4f838153028a6f21ca30bf100f3791133
138Author: Simon Glass <sjg@chromium.org>
139Date:   Thu Aug 14 21:50:52 2014 -0600
140
141    patman: Use --no-pager' to stop git from forking a pager
142
143""",
144"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
145Author: Simon Glass <sjg@chromium.org>
146Date:   Fri Aug 22 15:57:39 2014 -0600
147
148    patman: Avoid changing the order of tags
149
150    patman collects tags that it sees in the commit and places them nicely
151    sorted at the end of the patch. However, this is not really necessary and
152    in fact is apparently not desirable.
153
154    Series-changes: 9
155    - Add new patch to avoid changing the order of tags
156
157    Series-version: 9
158
159    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
160    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
161"""]
162
163TEST_BRANCH = '__testbranch'
164
165class TestFunctional(unittest.TestCase):
166    """Functional test for buildman.
167
168    This aims to test from just below the invocation of buildman (parsing
169    of arguments) to 'make' and 'git' invocation. It is not a true
170    emd-to-end test, as it mocks git, make and the tool chain. But this
171    makes it easier to detect when the builder is doing the wrong thing,
172    since in many cases this test code will fail. For example, only a
173    very limited subset of 'git' arguments is supported - anything
174    unexpected will fail.
175    """
176    def setUp(self):
177        self._base_dir = tempfile.mkdtemp()
178        self._git_dir = os.path.join(self._base_dir, 'src')
179        self._buildman_pathname = sys.argv[0]
180        self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
181        command.test_result = self._HandleCommand
182        self.setupToolchains()
183        self._toolchains.Add('arm-gcc', test=False)
184        self._toolchains.Add('powerpc-gcc', test=False)
185        bsettings.Setup(None)
186        bsettings.AddFile(settings_data)
187        self._boards = board.Boards()
188        for brd in boards:
189            self._boards.AddBoard(board.Board(*brd))
190
191        # Directories where the source been cloned
192        self._clone_dirs = []
193        self._commits = len(commit_shortlog.splitlines()) + 1
194        self._total_builds = self._commits * len(boards)
195
196        # Number of calls to make
197        self._make_calls = 0
198
199        # Map of [board, commit] to error messages
200        self._error = {}
201
202        self._test_branch = TEST_BRANCH
203
204        # Avoid sending any output and clear all terminal output
205        terminal.SetPrintTestMode()
206        terminal.GetPrintTestLines()
207
208    def tearDown(self):
209        shutil.rmtree(self._base_dir)
210
211    def setupToolchains(self):
212        self._toolchains = toolchain.Toolchains()
213        self._toolchains.Add('gcc', test=False)
214
215    def _RunBuildman(self, *args):
216        return command.RunPipe([[self._buildman_pathname] + list(args)],
217                capture=True, capture_stderr=True)
218
219    def _RunControl(self, *args, **kwargs):
220        sys.argv = [sys.argv[0]] + list(args)
221        options, args = cmdline.ParseArgs()
222        result = control.DoBuildman(options, args, toolchains=self._toolchains,
223                make_func=self._HandleMake, boards=self._boards,
224                clean_dir=kwargs.get('clean_dir', True))
225        self._builder = control.builder
226        return result
227
228    def testFullHelp(self):
229        command.test_result = None
230        result = self._RunBuildman('-H')
231        help_file = os.path.join(self._buildman_dir, 'README')
232        # Remove possible extraneous strings
233        extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
234        gothelp = result.stdout.replace(extra, '')
235        self.assertEqual(len(gothelp), os.path.getsize(help_file))
236        self.assertEqual(0, len(result.stderr))
237        self.assertEqual(0, result.return_code)
238
239    def testHelp(self):
240        command.test_result = None
241        result = self._RunBuildman('-h')
242        help_file = os.path.join(self._buildman_dir, 'README')
243        self.assertTrue(len(result.stdout) > 1000)
244        self.assertEqual(0, len(result.stderr))
245        self.assertEqual(0, result.return_code)
246
247    def testGitSetup(self):
248        """Test gitutils.Setup(), from outside the module itself"""
249        command.test_result = command.CommandResult(return_code=1)
250        gitutil.Setup()
251        self.assertEqual(gitutil.use_no_decorate, False)
252
253        command.test_result = command.CommandResult(return_code=0)
254        gitutil.Setup()
255        self.assertEqual(gitutil.use_no_decorate, True)
256
257    def _HandleCommandGitLog(self, args):
258        if args[-1] == '--':
259            args = args[:-1]
260        if '-n0' in args:
261            return command.CommandResult(return_code=0)
262        elif args[-1] == 'upstream/master..%s' % self._test_branch:
263            return command.CommandResult(return_code=0, stdout=commit_shortlog)
264        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
265            if args[-1] == self._test_branch:
266                count = int(args[3][2:])
267                return command.CommandResult(return_code=0,
268                                            stdout=''.join(commit_log[:count]))
269
270        # Not handled, so abort
271        print 'git log', args
272        sys.exit(1)
273
274    def _HandleCommandGitConfig(self, args):
275        config = args[0]
276        if config == 'sendemail.aliasesfile':
277            return command.CommandResult(return_code=0)
278        elif config.startswith('branch.badbranch'):
279            return command.CommandResult(return_code=1)
280        elif config == 'branch.%s.remote' % self._test_branch:
281            return command.CommandResult(return_code=0, stdout='upstream\n')
282        elif config == 'branch.%s.merge' % self._test_branch:
283            return command.CommandResult(return_code=0,
284                                         stdout='refs/heads/master\n')
285
286        # Not handled, so abort
287        print 'git config', args
288        sys.exit(1)
289
290    def _HandleCommandGit(self, in_args):
291        """Handle execution of a git command
292
293        This uses a hacked-up parser.
294
295        Args:
296            in_args: Arguments after 'git' from the command line
297        """
298        git_args = []           # Top-level arguments to git itself
299        sub_cmd = None          # Git sub-command selected
300        args = []               # Arguments to the git sub-command
301        for arg in in_args:
302            if sub_cmd:
303                args.append(arg)
304            elif arg[0] == '-':
305                git_args.append(arg)
306            else:
307                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
308                    git_args.append(arg)
309                else:
310                    sub_cmd = arg
311        if sub_cmd == 'config':
312            return self._HandleCommandGitConfig(args)
313        elif sub_cmd == 'log':
314            return self._HandleCommandGitLog(args)
315        elif sub_cmd == 'clone':
316            return command.CommandResult(return_code=0)
317        elif sub_cmd == 'checkout':
318            return command.CommandResult(return_code=0)
319
320        # Not handled, so abort
321        print 'git', git_args, sub_cmd, args
322        sys.exit(1)
323
324    def _HandleCommandNm(self, args):
325        return command.CommandResult(return_code=0)
326
327    def _HandleCommandObjdump(self, args):
328        return command.CommandResult(return_code=0)
329
330    def _HandleCommandObjcopy(self, args):
331        return command.CommandResult(return_code=0)
332
333    def _HandleCommandSize(self, args):
334        return command.CommandResult(return_code=0)
335
336    def _HandleCommand(self, **kwargs):
337        """Handle a command execution.
338
339        The command is in kwargs['pipe-list'], as a list of pipes, each a
340        list of commands. The command should be emulated as required for
341        testing purposes.
342
343        Returns:
344            A CommandResult object
345        """
346        pipe_list = kwargs['pipe_list']
347        wc = False
348        if len(pipe_list) != 1:
349            if pipe_list[1] == ['wc', '-l']:
350                wc = True
351            else:
352                print 'invalid pipe', kwargs
353                sys.exit(1)
354        cmd = pipe_list[0][0]
355        args = pipe_list[0][1:]
356        result = None
357        if cmd == 'git':
358            result = self._HandleCommandGit(args)
359        elif cmd == './scripts/show-gnu-make':
360            return command.CommandResult(return_code=0, stdout='make')
361        elif cmd.endswith('nm'):
362            return self._HandleCommandNm(args)
363        elif cmd.endswith('objdump'):
364            return self._HandleCommandObjdump(args)
365        elif cmd.endswith('objcopy'):
366            return self._HandleCommandObjcopy(args)
367        elif cmd.endswith( 'size'):
368            return self._HandleCommandSize(args)
369
370        if not result:
371            # Not handled, so abort
372            print 'unknown command', kwargs
373            sys.exit(1)
374
375        if wc:
376            result.stdout = len(result.stdout.splitlines())
377        return result
378
379    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
380        """Handle execution of 'make'
381
382        Args:
383            commit: Commit object that is being built
384            brd: Board object that is being built
385            stage: Stage that we are at (mrproper, config, build)
386            cwd: Directory where make should be run
387            args: Arguments to pass to make
388            kwargs: Arguments to pass to command.RunPipe()
389        """
390        self._make_calls += 1
391        if stage == 'mrproper':
392            return command.CommandResult(return_code=0)
393        elif stage == 'config':
394            return command.CommandResult(return_code=0,
395                    combined='Test configuration complete')
396        elif stage == 'build':
397            stderr = ''
398            if type(commit) is not str:
399                stderr = self._error.get((brd.target, commit.sequence))
400            if stderr:
401                return command.CommandResult(return_code=1, stderr=stderr)
402            return command.CommandResult(return_code=0)
403
404        # Not handled, so abort
405        print 'make', stage
406        sys.exit(1)
407
408    # Example function to print output lines
409    def print_lines(self, lines):
410        print len(lines)
411        for line in lines:
412            print line
413        #self.print_lines(terminal.GetPrintTestLines())
414
415    def testNoBoards(self):
416        """Test that buildman aborts when there are no boards"""
417        self._boards = board.Boards()
418        with self.assertRaises(SystemExit):
419            self._RunControl()
420
421    def testCurrentSource(self):
422        """Very simple test to invoke buildman on the current source"""
423        self.setupToolchains();
424        self._RunControl()
425        lines = terminal.GetPrintTestLines()
426        self.assertIn('Building current source for %d boards' % len(boards),
427                      lines[0].text)
428
429    def testBadBranch(self):
430        """Test that we can detect an invalid branch"""
431        with self.assertRaises(ValueError):
432            self._RunControl('-b', 'badbranch')
433
434    def testBadToolchain(self):
435        """Test that missing toolchains are detected"""
436        self.setupToolchains();
437        ret_code = self._RunControl('-b', TEST_BRANCH)
438        lines = terminal.GetPrintTestLines()
439
440        # Buildman always builds the upstream commit as well
441        self.assertIn('Building %d commits for %d boards' %
442                (self._commits, len(boards)), lines[0].text)
443        self.assertEqual(self._builder.count, self._total_builds)
444
445        # Only sandbox should succeed, the others don't have toolchains
446        self.assertEqual(self._builder.fail,
447                         self._total_builds - self._commits)
448        self.assertEqual(ret_code, 128)
449
450        for commit in range(self._commits):
451            for board in self._boards.GetList():
452                if board.arch != 'sandbox':
453                  errfile = self._builder.GetErrFile(commit, board.target)
454                  fd = open(errfile)
455                  self.assertEqual(fd.readlines(),
456                          ['No tool chain for %s\n' % board.arch])
457                  fd.close()
458
459    def testBranch(self):
460        """Test building a branch with all toolchains present"""
461        self._RunControl('-b', TEST_BRANCH)
462        self.assertEqual(self._builder.count, self._total_builds)
463        self.assertEqual(self._builder.fail, 0)
464
465    def testCount(self):
466        """Test building a specific number of commitst"""
467        self._RunControl('-b', TEST_BRANCH, '-c2')
468        self.assertEqual(self._builder.count, 2 * len(boards))
469        self.assertEqual(self._builder.fail, 0)
470        # Each board has a mrproper, config, and then one make per commit
471        self.assertEqual(self._make_calls, len(boards) * (2 + 2))
472
473    def testIncremental(self):
474        """Test building a branch twice - the second time should do nothing"""
475        self._RunControl('-b', TEST_BRANCH)
476
477        # Each board has a mrproper, config, and then one make per commit
478        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
479        self._make_calls = 0
480        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
481        self.assertEqual(self._make_calls, 0)
482        self.assertEqual(self._builder.count, self._total_builds)
483        self.assertEqual(self._builder.fail, 0)
484
485    def testForceBuild(self):
486        """The -f flag should force a rebuild"""
487        self._RunControl('-b', TEST_BRANCH)
488        self._make_calls = 0
489        self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
490        # Each board has a mrproper, config, and then one make per commit
491        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
492
493    def testForceReconfigure(self):
494        """The -f flag should force a rebuild"""
495        self._RunControl('-b', TEST_BRANCH, '-C')
496        # Each commit has a mrproper, config and make
497        self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
498
499    def testErrors(self):
500        """Test handling of build errors"""
501        self._error['board2', 1] = 'fred\n'
502        self._RunControl('-b', TEST_BRANCH)
503        self.assertEqual(self._builder.count, self._total_builds)
504        self.assertEqual(self._builder.fail, 1)
505
506        # Remove the error. This should have no effect since the commit will
507        # not be rebuilt
508        del self._error['board2', 1]
509        self._make_calls = 0
510        self._RunControl('-b', TEST_BRANCH, clean_dir=False)
511        self.assertEqual(self._builder.count, self._total_builds)
512        self.assertEqual(self._make_calls, 0)
513        self.assertEqual(self._builder.fail, 1)
514
515        # Now use the -F flag to force rebuild of the bad commit
516        self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
517        self.assertEqual(self._builder.count, self._total_builds)
518        self.assertEqual(self._builder.fail, 0)
519        self.assertEqual(self._make_calls, 3)
520
521    def testBranchWithSlash(self):
522        """Test building a branch with a '/' in the name"""
523        self._test_branch = '/__dev/__testbranch'
524        self._RunControl('-b', self._test_branch, clean_dir=False)
525        self.assertEqual(self._builder.count, self._total_builds)
526        self.assertEqual(self._builder.fail, 0)
527
528    def testBadOutputDir(self):
529        """Test building with an output dir the same as out current dir"""
530        self._test_branch = '/__dev/__testbranch'
531        with self.assertRaises(SystemExit):
532            self._RunControl('-b', self._test_branch, '-o', os.getcwd())
533        with self.assertRaises(SystemExit):
534            self._RunControl('-b', self._test_branch, '-o',
535                             os.path.join(os.getcwd(), 'test'))
536