1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7import os
8import re
9
10import oeqa.utils.ftools as ftools
11from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
12
13from oeqa.selftest.case import OESelftestTestCase
14
15class BitbakeTests(OESelftestTestCase):
16
17    def getline(self, res, line):
18        for l in res.output.split('\n'):
19            if line in l:
20                return l
21
22    # Test bitbake can run from the <builddir>/conf directory
23    def test_run_bitbake_from_dir_1(self):
24        os.chdir(os.path.join(self.builddir, 'conf'))
25        self.assertEqual(bitbake('-e').status, 0, msg = "bitbake couldn't run from \"conf\" dir")
26
27    # Test bitbake can run from the <builddir>'s parent directory
28    def test_run_bitbake_from_dir_2(self):
29        my_env = os.environ.copy()
30        my_env['BBPATH'] = my_env['BUILDDIR']
31        os.chdir(os.path.dirname(os.environ['BUILDDIR']))
32        self.assertEqual(bitbake('-e', env=my_env).status, 0, msg = "bitbake couldn't run from builddir's parent directory")
33
34    # Test bitbake can run from some other random system location (we use /tmp/)
35    def test_run_bitbake_from_dir_3(self):
36        my_env = os.environ.copy()
37        my_env['BBPATH'] = my_env['BUILDDIR']
38        os.chdir("/tmp/")
39        self.assertEqual(bitbake('-e', env=my_env).status, 0, msg = "bitbake couldn't run from /tmp/")
40
41
42    def test_event_handler(self):
43        self.write_config("INHERIT += \"test_events\"")
44        result = bitbake('selftest-hello-native')
45        find_build_started = re.search(r"NOTE: Test for bb\.event\.BuildStarted(\n.*)*NOTE: Executing.*Tasks", result.output)
46        find_build_completed = re.search(r"Tasks Summary:.*(\n.*)*NOTE: Test for bb\.event\.BuildCompleted", result.output)
47        self.assertTrue(find_build_started, msg = "Match failed in:\n%s"  % result.output)
48        self.assertTrue(find_build_completed, msg = "Match failed in:\n%s" % result.output)
49        self.assertNotIn('Test for bb.event.InvalidEvent', result.output)
50
51    def test_local_sstate(self):
52        bitbake('selftest-hello-native')
53        bitbake('selftest-hello-native -cclean')
54        result = bitbake('selftest-hello-native')
55        find_setscene = re.search("selftest-hello-native.*do_.*_setscene", result.output)
56        self.assertTrue(find_setscene, msg = "No \"selftest-hello-native.*do_.*_setscene\" message found during bitbake selftest-hello-native. bitbake output: %s" % result.output )
57
58    def test_bitbake_invalid_recipe(self):
59        result = bitbake('-b asdf', ignore_status=True)
60        self.assertTrue("ERROR: Unable to find any recipe file matching 'asdf'" in result.output, msg = "Though asdf recipe doesn't exist, bitbake didn't output any err. message. bitbake output: %s" % result.output)
61
62    def test_bitbake_invalid_target(self):
63        result = bitbake('asdf', ignore_status=True)
64        self.assertIn("ERROR: Nothing PROVIDES 'asdf'", result.output)
65
66    def test_warnings_errors(self):
67        result = bitbake('-b asdf', ignore_status=True)
68        find_warnings = re.search("Summary: There w.{2,3}? [1-9][0-9]* WARNING messages*", result.output)
69        find_errors = re.search("Summary: There w.{2,3}? [1-9][0-9]* ERROR messages*", result.output)
70        self.assertTrue(find_warnings, msg="Did not find the mumber of warnings at the end of the build:\n" + result.output)
71        self.assertTrue(find_errors, msg="Did not find the mumber of errors at the end of the build:\n" + result.output)
72
73    def test_invalid_patch(self):
74        # This patch should fail to apply.
75        self.write_recipeinc('man-db', 'FILESEXTRAPATHS:prepend := "${THISDIR}/files:"\nSRC_URI += "file://0001-Test-patch-here.patch"')
76        self.write_config("INHERIT:remove = \"report-error\"")
77        result = bitbake('man-db -c patch', ignore_status=True)
78        self.delete_recipeinc('man-db')
79        bitbake('-cclean man-db')
80        found = False
81        for l in result.output.split('\n'):
82            if l.startswith("ERROR:") and "failed" in l and "do_patch" in l:
83                found = l
84        self.assertTrue(found and found.startswith("ERROR:"), msg = "Incorrectly formed patch application didn't fail. bitbake output: %s" % result.output)
85
86    def test_force_task_1(self):
87        # test 1 from bug 5875
88        import uuid
89        test_recipe = 'zlib'
90        # Need to use uuid otherwise hash equivlance would change the workflow
91        test_data = "Microsoft Made No Profit From Anyone's Zunes Yo %s" % uuid.uuid1()
92        bb_vars = get_bb_vars(['D', 'PKGDEST', 'mandir'], test_recipe)
93        image_dir = bb_vars['D']
94        pkgsplit_dir = bb_vars['PKGDEST']
95        man_dir = bb_vars['mandir']
96        self.write_config("PACKAGE_CLASSES = \"package_rpm\"")
97
98        bitbake('-c clean %s' % test_recipe)
99        bitbake('-c package -f %s' % test_recipe)
100        self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe)
101
102        man_file = os.path.join(image_dir + man_dir, 'man3/zlib.3')
103        ftools.append_file(man_file, test_data)
104        bitbake('-c package -f %s' % test_recipe)
105
106        man_split_file = os.path.join(pkgsplit_dir, 'zlib-doc' + man_dir, 'man3/zlib.3')
107        man_split_content = ftools.read_file(man_split_file)
108        self.assertIn(test_data, man_split_content, 'The man file has not changed in packages-split.')
109
110        ret = bitbake(test_recipe)
111        self.assertIn('task do_package_write_rpm:', ret.output, 'Task do_package_write_rpm did not re-executed.')
112
113    def test_force_task_2(self):
114        # test 2 from bug 5875
115        test_recipe = 'zlib'
116
117        bitbake(test_recipe)
118        self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe)
119
120        result = bitbake('-C compile %s' % test_recipe)
121        look_for_tasks = ['do_compile:', 'do_install:', 'do_populate_sysroot:', 'do_package:']
122        for task in look_for_tasks:
123            self.assertIn(task, result.output, msg="Couldn't find %s task.")
124
125    def test_bitbake_g(self):
126        recipe = 'base-files'
127        result = bitbake('-g %s' % recipe)
128        for f in ['pn-buildlist', 'task-depends.dot']:
129            self.addCleanup(os.remove, f)
130        self.assertTrue('Task dependencies saved to \'task-depends.dot\'' in result.output, msg = "No task dependency \"task-depends.dot\" file was generated for the given task target. bitbake output: %s" % result.output)
131        self.assertIn(recipe, ftools.read_file(os.path.join(self.builddir, 'task-depends.dot')))
132
133    def test_image_manifest(self):
134        bitbake('core-image-minimal')
135        bb_vars = get_bb_vars(["DEPLOY_DIR_IMAGE", "IMAGE_LINK_NAME"], "core-image-minimal")
136        deploydir = bb_vars["DEPLOY_DIR_IMAGE"]
137        imagename = bb_vars["IMAGE_LINK_NAME"]
138        manifest = os.path.join(deploydir, imagename + ".manifest")
139        self.assertTrue(os.path.islink(manifest), msg="No manifest file created for image. It should have been created in %s" % manifest)
140
141    def test_invalid_recipe_src_uri(self):
142        data = 'SRC_URI = "file://invalid"'
143        self.write_recipeinc('man-db', data)
144        self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\"
145SSTATE_DIR = \"${TOPDIR}/download-selftest\"
146INHERIT:remove = \"report-error\"
147""")
148        self.track_for_cleanup(os.path.join(self.builddir, "download-selftest"))
149
150        result = bitbake('-c fetch man-db', ignore_status=True)
151        self.delete_recipeinc('man-db')
152        self.assertEqual(result.status, 1, msg="Command succeded when it should have failed. bitbake output: %s" % result.output)
153        self.assertIn('Unable to get checksum for man-db SRC_URI entry invalid: file could not be found', result.output)
154
155    def test_rename_downloaded_file(self):
156        # TODO unique dldir instead of using cleanall
157        # TODO: need to set sstatedir?
158        self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\"
159SSTATE_DIR = \"${TOPDIR}/download-selftest\"
160""")
161        self.track_for_cleanup(os.path.join(self.builddir, "download-selftest"))
162
163        data = 'SRC_URI = "https://downloads.yoctoproject.org/mirror/sources/aspell-${PV}.tar.gz;downloadfilename=test-aspell.tar.gz"'
164        self.write_recipeinc('aspell', data)
165        result = bitbake('-f -c fetch aspell', ignore_status=True)
166        self.delete_recipeinc('aspell')
167        self.assertEqual(result.status, 0, msg = "Couldn't fetch aspell. %s" % result.output)
168        dl_dir = get_bb_var("DL_DIR")
169        self.assertTrue(os.path.isfile(os.path.join(dl_dir, 'test-aspell.tar.gz')), msg = "File rename failed. No corresponding test-aspell.tar.gz file found under %s" % dl_dir)
170        self.assertTrue(os.path.isfile(os.path.join(dl_dir, 'test-aspell.tar.gz.done')), "File rename failed. No corresponding test-aspell.tar.gz.done file found under %s" % dl_dir)
171
172    def test_environment(self):
173        self.write_config("TEST_ENV=\"localconf\"")
174        result = runCmd('bitbake -e | grep TEST_ENV=')
175        self.assertIn('localconf', result.output)
176
177    def test_dry_run(self):
178        result = runCmd('bitbake -n selftest-hello-native')
179        self.assertEqual(0, result.status, "bitbake dry run didn't run as expected. %s" % result.output)
180
181    def test_just_parse(self):
182        result = runCmd('bitbake -p')
183        self.assertEqual(0, result.status, "errors encountered when parsing recipes. %s" % result.output)
184
185    def test_version(self):
186        result = runCmd('bitbake -s | grep wget')
187        find = re.search(r"wget *:([0-9a-zA-Z\.\-]+)", result.output)
188        self.assertTrue(find, "No version returned for searched recipe. bitbake output: %s" % result.output)
189
190    def test_prefile(self):
191        # Test when the prefile does not exist
192        result = runCmd('bitbake -r conf/prefile.conf', ignore_status=True)
193        self.assertEqual(1, result.status, "bitbake didn't error and should have when a specified prefile didn't exist: %s" % result.output)
194        # Test when the prefile exists
195        preconf = os.path.join(self.builddir, 'conf/prefile.conf')
196        self.track_for_cleanup(preconf)
197        ftools.write_file(preconf ,"TEST_PREFILE=\"prefile\"")
198        result = runCmd('bitbake -r conf/prefile.conf -e | grep TEST_PREFILE=')
199        self.assertIn('prefile', result.output)
200        self.write_config("TEST_PREFILE=\"localconf\"")
201        result = runCmd('bitbake -r conf/prefile.conf -e | grep TEST_PREFILE=')
202        self.assertIn('localconf', result.output)
203
204    def test_postfile(self):
205        # Test when the postfile does not exist
206        result = runCmd('bitbake -R conf/postfile.conf', ignore_status=True)
207        self.assertEqual(1, result.status, "bitbake didn't error and should have when a specified postfile didn't exist: %s" % result.output)
208        # Test when the postfile exists
209        postconf = os.path.join(self.builddir, 'conf/postfile.conf')
210        self.track_for_cleanup(postconf)
211        ftools.write_file(postconf , "TEST_POSTFILE=\"postfile\"")
212        self.write_config("TEST_POSTFILE=\"localconf\"")
213        result = runCmd('bitbake -R conf/postfile.conf -e | grep TEST_POSTFILE=')
214        self.assertIn('postfile', result.output)
215
216    def test_checkuri(self):
217        result = runCmd('bitbake -c checkuri m4')
218        self.assertEqual(0, result.status, msg = "\"checkuri\" task was not executed. bitbake output: %s" % result.output)
219
220    def test_continue(self):
221        self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\"
222SSTATE_DIR = \"${TOPDIR}/download-selftest\"
223INHERIT:remove = \"report-error\"
224""")
225        self.track_for_cleanup(os.path.join(self.builddir, "download-selftest"))
226        self.write_recipeinc('man-db',"\ndo_fail_task () {\nexit 1 \n}\n\naddtask do_fail_task before do_fetch\n" )
227        runCmd('bitbake -c cleanall man-db xcursor-transparent-theme')
228        result = runCmd('bitbake -c unpack -k man-db xcursor-transparent-theme', ignore_status=True)
229        errorpos = result.output.find('ERROR: Function failed: do_fail_task')
230        manver = re.search("NOTE: recipe xcursor-transparent-theme-(.*?): task do_unpack: Started", result.output)
231        continuepos = result.output.find('NOTE: recipe xcursor-transparent-theme-%s: task do_unpack: Started' % manver.group(1))
232        self.assertLess(errorpos,continuepos, msg = "bitbake didn't pass do_fail_task. bitbake output: %s" % result.output)
233
234    def test_non_gplv3(self):
235        self.write_config('INCOMPATIBLE_LICENSE = "GPL-3.0-or-later"')
236        result = bitbake('selftest-ed', ignore_status=True)
237        self.assertEqual(result.status, 0, "Bitbake failed, exit code %s, output %s" % (result.status, result.output))
238        lic_dir = get_bb_var('LICENSE_DIRECTORY')
239        arch = get_bb_var('SSTATE_PKGARCH')
240        filename = os.path.join(lic_dir, arch, 'selftest-ed', 'generic_GPL-3.0-or-later')
241        self.assertFalse(os.path.isfile(filename), msg="License file %s exists and shouldn't" % filename)
242        filename = os.path.join(lic_dir, arch, 'selftest-ed', 'generic_GPL-2.0-or-later')
243        self.assertTrue(os.path.isfile(filename), msg="License file %s doesn't exist" % filename)
244
245    def test_setscene_only(self):
246        """ Bitbake option to restore from sstate only within a build (i.e. execute no real tasks, only setscene)"""
247        test_recipe = 'selftest-hello-native'
248
249        bitbake(test_recipe)
250        bitbake('-c clean %s' % test_recipe)
251        ret = bitbake('--setscene-only %s' % test_recipe)
252
253        tasks = re.findall(r'task\s+(do_\S+):', ret.output)
254
255        for task in tasks:
256            self.assertIn('_setscene', task, 'A task different from _setscene ran: %s.\n'
257                                             'Executed tasks were: %s' % (task, str(tasks)))
258
259    def test_skip_setscene(self):
260        test_recipe = 'selftest-hello-native'
261
262        bitbake(test_recipe)
263        bitbake('-c clean %s' % test_recipe)
264
265        ret = bitbake('--setscene-only %s' % test_recipe)
266        tasks = re.findall(r'task\s+(do_\S+):', ret.output)
267
268        for task in tasks:
269            self.assertIn('_setscene', task, 'A task different from _setscene ran: %s.\n'
270                                             'Executed tasks were: %s' % (task, str(tasks)))
271
272        # Run without setscene. Should do nothing
273        ret = bitbake('--skip-setscene %s' % test_recipe)
274        tasks = re.findall(r'task\s+(do_\S+):', ret.output)
275
276        self.assertFalse(tasks, 'Tasks %s ran when they should not have' % (str(tasks)))
277
278        # Clean (leave sstate cache) and run with --skip-setscene. No setscene
279        # tasks should run
280        bitbake('-c clean %s' % test_recipe)
281
282        ret = bitbake('--skip-setscene %s' % test_recipe)
283        tasks = re.findall(r'task\s+(do_\S+):', ret.output)
284
285        for task in tasks:
286            self.assertNotIn('_setscene', task, 'A _setscene task ran: %s.\n'
287                                                'Executed tasks were: %s' % (task, str(tasks)))
288
289    def test_bbappend_order(self):
290        """ Bitbake should bbappend to recipe in a predictable order """
291        test_recipe = 'ed'
292        bb_vars = get_bb_vars(['SUMMARY', 'PV'], test_recipe)
293        test_recipe_summary_before = bb_vars['SUMMARY']
294        test_recipe_pv = bb_vars['PV']
295        recipe_append_file = test_recipe + '_' + test_recipe_pv + '.bbappend'
296        expected_recipe_summary = test_recipe_summary_before
297
298        for i in range(5):
299            recipe_append_dir = test_recipe + '_test_' + str(i)
300            recipe_append_path = os.path.join(self.testlayer_path, 'recipes-test', recipe_append_dir, recipe_append_file)
301            os.mkdir(os.path.join(self.testlayer_path, 'recipes-test', recipe_append_dir))
302            feature = 'SUMMARY += "%s"\n' % i
303            ftools.write_file(recipe_append_path, feature)
304            expected_recipe_summary += ' %s' % i
305
306        self.add_command_to_tearDown('rm -rf %s' % os.path.join(self.testlayer_path, 'recipes-test',
307                                                               test_recipe + '_test_*'))
308
309        test_recipe_summary_after = get_bb_var('SUMMARY', test_recipe)
310        self.assertEqual(expected_recipe_summary, test_recipe_summary_after)
311
312    def test_git_patchtool(self):
313        """ PATCHTOOL=git should work with non-git sources like tarballs
314            test recipe for the test must NOT containt git:// repository in SRC_URI
315        """
316        test_recipe = "man-db"
317        self.write_recipeinc(test_recipe, 'PATCHTOOL=\"git\"')
318        src = get_bb_var("SRC_URI",test_recipe)
319        gitscm = re.search("git://", src)
320        self.assertFalse(gitscm, "test_git_patchtool pre-condition failed: {} test recipe contains git repo!".format(test_recipe))
321        result = bitbake('{} -c patch'.format(test_recipe), ignore_status=False)
322        fatal = re.search("fatal: not a git repository (or any of the parent directories)", result.output)
323        self.assertFalse(fatal, "Failed to patch using PATCHTOOL=\"git\"")
324        self.delete_recipeinc(test_recipe)
325        bitbake('-cclean {}'.format(test_recipe))
326
327    def test_git_patchtool2(self):
328        """ Test if PATCHTOOL=git works with git repo and doesn't reinitialize it
329        """
330        test_recipe = "gitrepotest"
331        src = get_bb_var("SRC_URI",test_recipe)
332        gitscm = re.search("git://", src)
333        self.assertTrue(gitscm, "test_git_patchtool pre-condition failed: {} test recipe doesn't contains git repo!".format(test_recipe))
334        result = bitbake('{} -c patch'.format(test_recipe), ignore_status=False)
335        srcdir = get_bb_var('S', test_recipe)
336        result = runCmd("git log", cwd = srcdir)
337        self.assertFalse("bitbake_patching_started" in result.output, msg = "Repository has been reinitialized. {}".format(srcdir))
338        self.delete_recipeinc(test_recipe)
339        bitbake('-cclean {}'.format(test_recipe))
340
341
342    def test_git_unpack_nonetwork(self):
343        """
344        Test that a recipe with a floating tag that needs to be resolved upstream doesn't
345        access the network in a patch task run in a separate builld invocation
346        """
347
348        # Enable the recipe to float using a distro override
349        self.write_config("DISTROOVERRIDES .= \":gitunpack-enable-recipe\"")
350
351        bitbake('gitunpackoffline -c fetch')
352        bitbake('gitunpackoffline -c patch')
353
354    def test_git_unpack_nonetwork_fail(self):
355        """
356        Test that a recipe with a floating tag which doesn't call get_srcrev() in the fetcher
357        raises an error when the fetcher is called.
358        """
359
360        # Enable the recipe to float using a distro override
361        self.write_config("DISTROOVERRIDES .= \":gitunpack-enable-recipe\"")
362
363        result = bitbake('gitunpackoffline-fail -c fetch', ignore_status=True)
364        self.assertTrue(re.search("Recipe uses a floating tag/branch .* for repo .* without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev()", result.output), msg = "Recipe without PV set to SRCPV should have failed: %s" % result.output)
365