1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: MIT 5# 6 7import os 8import shutil 9import tempfile 10import urllib.parse 11 12from oeqa.utils.commands import runCmd, bitbake, get_bb_var 13from oeqa.utils.commands import get_bb_vars, create_temp_layer 14from oeqa.selftest.cases import devtool 15 16templayerdir = None 17 18def setUpModule(): 19 global templayerdir 20 templayerdir = tempfile.mkdtemp(prefix='recipetoolqa') 21 create_temp_layer(templayerdir, 'selftestrecipetool') 22 runCmd('bitbake-layers add-layer %s' % templayerdir) 23 24 25def tearDownModule(): 26 runCmd('bitbake-layers remove-layer %s' % templayerdir, ignore_status=True) 27 runCmd('rm -rf %s' % templayerdir) 28 29 30class RecipetoolBase(devtool.DevtoolTestCase): 31 32 def setUpLocal(self): 33 super(RecipetoolBase, self).setUpLocal() 34 self.templayerdir = templayerdir 35 self.tempdir = tempfile.mkdtemp(prefix='recipetoolqa') 36 self.track_for_cleanup(self.tempdir) 37 self.testfile = os.path.join(self.tempdir, 'testfile') 38 with open(self.testfile, 'w') as f: 39 f.write('Test file\n') 40 41 def tearDownLocal(self): 42 runCmd('rm -rf %s/recipes-*' % self.templayerdir) 43 super(RecipetoolBase, self).tearDownLocal() 44 45 def _try_recipetool_appendcmd(self, cmd, testrecipe, expectedfiles, expectedlines=None): 46 result = runCmd(cmd) 47 self.assertNotIn('Traceback', result.output) 48 49 # Check the bbappend was created and applies properly 50 recipefile = get_bb_var('FILE', testrecipe) 51 bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) 52 53 # Check the bbappend contents 54 if expectedlines is not None: 55 with open(bbappendfile, 'r') as f: 56 self.assertEqual(expectedlines, f.readlines(), "Expected lines are not present in %s" % bbappendfile) 57 58 # Check file was copied 59 filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe) 60 for expectedfile in expectedfiles: 61 self.assertTrue(os.path.isfile(os.path.join(filesdir, expectedfile)), 'Expected file %s to be copied next to bbappend, but it wasn\'t' % expectedfile) 62 63 # Check no other files created 64 createdfiles = [] 65 for root, _, files in os.walk(filesdir): 66 for f in files: 67 createdfiles.append(os.path.relpath(os.path.join(root, f), filesdir)) 68 self.assertTrue(sorted(createdfiles), sorted(expectedfiles)) 69 70 return bbappendfile, result.output 71 72 73class RecipetoolAppendTests(RecipetoolBase): 74 75 @classmethod 76 def setUpClass(cls): 77 super(RecipetoolAppendTests, cls).setUpClass() 78 # Ensure we have the right data in shlibs/pkgdata 79 cls.logger.info('Running bitbake to generate pkgdata') 80 bitbake('-c packagedata base-files coreutils busybox selftest-recipetool-appendfile') 81 bb_vars = get_bb_vars(['COREBASE']) 82 cls.corebase = bb_vars['COREBASE'] 83 84 def _try_recipetool_appendfile(self, testrecipe, destfile, newfile, options, expectedlines, expectedfiles): 85 cmd = 'recipetool appendfile %s %s %s %s' % (self.templayerdir, destfile, newfile, options) 86 return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) 87 88 def _try_recipetool_appendfile_fail(self, destfile, newfile, checkerror): 89 cmd = 'recipetool appendfile %s %s %s' % (self.templayerdir, destfile, newfile) 90 result = runCmd(cmd, ignore_status=True) 91 self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd) 92 self.assertNotIn('Traceback', result.output) 93 for errorstr in checkerror: 94 self.assertIn(errorstr, result.output) 95 96 def test_recipetool_appendfile_basic(self): 97 # Basic test 98 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 99 '\n'] 100 _, output = self._try_recipetool_appendfile('base-files', '/etc/motd', self.testfile, '', expectedlines, ['motd']) 101 self.assertNotIn('WARNING: ', output) 102 103 def test_recipetool_appendfile_invalid(self): 104 # Test some commands that should error 105 self._try_recipetool_appendfile_fail('/etc/passwd', self.testfile, ['ERROR: /etc/passwd cannot be handled by this tool', 'useradd', 'extrausers']) 106 self._try_recipetool_appendfile_fail('/etc/timestamp', self.testfile, ['ERROR: /etc/timestamp cannot be handled by this tool']) 107 self._try_recipetool_appendfile_fail('/dev/console', self.testfile, ['ERROR: /dev/console cannot be handled by this tool']) 108 109 def test_recipetool_appendfile_alternatives(self): 110 # Now try with a file we know should be an alternative 111 # (this is very much a fake example, but one we know is reliably an alternative) 112 self._try_recipetool_appendfile_fail('/bin/ls', self.testfile, ['ERROR: File /bin/ls is an alternative possibly provided by the following recipes:', 'coreutils', 'busybox']) 113 # Need a test file - should be executable 114 testfile2 = os.path.join(self.corebase, 'oe-init-build-env') 115 testfile2name = os.path.basename(testfile2) 116 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 117 '\n', 118 'SRC_URI += "file://%s"\n' % testfile2name, 119 '\n', 120 'do_install:append() {\n', 121 ' install -d ${D}${base_bindir}\n', 122 ' install -m 0755 ${WORKDIR}/%s ${D}${base_bindir}/ls\n' % testfile2name, 123 '}\n'] 124 self._try_recipetool_appendfile('coreutils', '/bin/ls', testfile2, '-r coreutils', expectedlines, [testfile2name]) 125 # Now try bbappending the same file again, contents should not change 126 bbappendfile, _ = self._try_recipetool_appendfile('coreutils', '/bin/ls', self.testfile, '-r coreutils', expectedlines, [testfile2name]) 127 # But file should have 128 copiedfile = os.path.join(os.path.dirname(bbappendfile), 'coreutils', testfile2name) 129 result = runCmd('diff -q %s %s' % (testfile2, copiedfile), ignore_status=True) 130 self.assertNotEqual(result.status, 0, 'New file should have been copied but was not %s' % result.output) 131 132 def test_recipetool_appendfile_binary(self): 133 # Try appending a binary file 134 # /bin/ls can be a symlink to /usr/bin/ls 135 ls = os.path.realpath("/bin/ls") 136 result = runCmd('recipetool appendfile %s /bin/ls %s -r coreutils' % (self.templayerdir, ls)) 137 self.assertIn('WARNING: ', result.output) 138 self.assertIn('is a binary', result.output) 139 140 def test_recipetool_appendfile_add(self): 141 # Try arbitrary file add to a recipe 142 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 143 '\n', 144 'SRC_URI += "file://testfile"\n', 145 '\n', 146 'do_install:append() {\n', 147 ' install -d ${D}${datadir}\n', 148 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n', 149 '}\n'] 150 self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase', expectedlines, ['testfile']) 151 # Try adding another file, this time where the source file is executable 152 # (so we're testing that, plus modifying an existing bbappend) 153 testfile2 = os.path.join(self.corebase, 'oe-init-build-env') 154 testfile2name = os.path.basename(testfile2) 155 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 156 '\n', 157 'SRC_URI += "file://testfile \\\n', 158 ' file://%s \\\n' % testfile2name, 159 ' "\n', 160 '\n', 161 'do_install:append() {\n', 162 ' install -d ${D}${datadir}\n', 163 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n', 164 ' install -m 0755 ${WORKDIR}/%s ${D}${datadir}/scriptname\n' % testfile2name, 165 '}\n'] 166 self._try_recipetool_appendfile('netbase', '/usr/share/scriptname', testfile2, '-r netbase', expectedlines, ['testfile', testfile2name]) 167 168 def test_recipetool_appendfile_add_bindir(self): 169 # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable 170 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 171 '\n', 172 'SRC_URI += "file://testfile"\n', 173 '\n', 174 'do_install:append() {\n', 175 ' install -d ${D}${bindir}\n', 176 ' install -m 0755 ${WORKDIR}/testfile ${D}${bindir}/selftest-recipetool-testbin\n', 177 '}\n'] 178 _, output = self._try_recipetool_appendfile('netbase', '/usr/bin/selftest-recipetool-testbin', self.testfile, '-r netbase', expectedlines, ['testfile']) 179 self.assertNotIn('WARNING: ', output) 180 181 def test_recipetool_appendfile_add_machine(self): 182 # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable 183 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 184 '\n', 185 'PACKAGE_ARCH = "${MACHINE_ARCH}"\n', 186 '\n', 187 'SRC_URI:append:mymachine = " file://testfile"\n', 188 '\n', 189 'do_install:append:mymachine() {\n', 190 ' install -d ${D}${datadir}\n', 191 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n', 192 '}\n'] 193 _, output = self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase -m mymachine', expectedlines, ['mymachine/testfile']) 194 self.assertNotIn('WARNING: ', output) 195 196 def test_recipetool_appendfile_orig(self): 197 # A file that's in SRC_URI and in do_install with the same name 198 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 199 '\n'] 200 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-orig', self.testfile, '', expectedlines, ['selftest-replaceme-orig']) 201 self.assertNotIn('WARNING: ', output) 202 203 def test_recipetool_appendfile_todir(self): 204 # A file that's in SRC_URI and in do_install with destination directory rather than file 205 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 206 '\n'] 207 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-todir', self.testfile, '', expectedlines, ['selftest-replaceme-todir']) 208 self.assertNotIn('WARNING: ', output) 209 210 def test_recipetool_appendfile_renamed(self): 211 # A file that's in SRC_URI with a different name to the destination file 212 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 213 '\n'] 214 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-renamed', self.testfile, '', expectedlines, ['file1']) 215 self.assertNotIn('WARNING: ', output) 216 217 def test_recipetool_appendfile_subdir(self): 218 # A file that's in SRC_URI in a subdir 219 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 220 '\n', 221 'SRC_URI += "file://testfile"\n', 222 '\n', 223 'do_install:append() {\n', 224 ' install -d ${D}${datadir}\n', 225 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-subdir\n', 226 '}\n'] 227 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-subdir', self.testfile, '', expectedlines, ['testfile']) 228 self.assertNotIn('WARNING: ', output) 229 230 def test_recipetool_appendfile_inst_glob(self): 231 # A file that's in do_install as a glob 232 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 233 '\n'] 234 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-globfile']) 235 self.assertNotIn('WARNING: ', output) 236 237 def test_recipetool_appendfile_inst_todir_glob(self): 238 # A file that's in do_install as a glob with destination as a directory 239 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 240 '\n'] 241 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-todir-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-todir-globfile']) 242 self.assertNotIn('WARNING: ', output) 243 244 def test_recipetool_appendfile_patch(self): 245 # A file that's added by a patch in SRC_URI 246 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 247 '\n', 248 'SRC_URI += "file://testfile"\n', 249 '\n', 250 'do_install:append() {\n', 251 ' install -d ${D}${sysconfdir}\n', 252 ' install -m 0644 ${WORKDIR}/testfile ${D}${sysconfdir}/selftest-replaceme-patched\n', 253 '}\n'] 254 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/etc/selftest-replaceme-patched', self.testfile, '', expectedlines, ['testfile']) 255 for line in output.splitlines(): 256 if 'WARNING: ' in line: 257 self.assertIn('add-file.patch', line, 'Unexpected warning found in output:\n%s' % line) 258 break 259 else: 260 self.fail('Patch warning not found in output:\n%s' % output) 261 262 def test_recipetool_appendfile_script(self): 263 # Now, a file that's in SRC_URI but installed by a script (so no mention in do_install) 264 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 265 '\n', 266 'SRC_URI += "file://testfile"\n', 267 '\n', 268 'do_install:append() {\n', 269 ' install -d ${D}${datadir}\n', 270 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-scripted\n', 271 '}\n'] 272 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-scripted', self.testfile, '', expectedlines, ['testfile']) 273 self.assertNotIn('WARNING: ', output) 274 275 def test_recipetool_appendfile_inst_func(self): 276 # A file that's installed from a function called by do_install 277 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 278 '\n'] 279 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-func', self.testfile, '', expectedlines, ['selftest-replaceme-inst-func']) 280 self.assertNotIn('WARNING: ', output) 281 282 def test_recipetool_appendfile_postinstall(self): 283 # A file that's created by a postinstall script (and explicitly mentioned in it) 284 # First try without specifying recipe 285 self._try_recipetool_appendfile_fail('/usr/share/selftest-replaceme-postinst', self.testfile, ['File /usr/share/selftest-replaceme-postinst may be written out in a pre/postinstall script of the following recipes:', 'selftest-recipetool-appendfile']) 286 # Now specify recipe 287 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 288 '\n', 289 'SRC_URI += "file://testfile"\n', 290 '\n', 291 'do_install:append() {\n', 292 ' install -d ${D}${datadir}\n', 293 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-postinst\n', 294 '}\n'] 295 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-postinst', self.testfile, '-r selftest-recipetool-appendfile', expectedlines, ['testfile']) 296 297 def test_recipetool_appendfile_extlayer(self): 298 # Try creating a bbappend in a layer that's not in bblayers.conf and has a different structure 299 exttemplayerdir = os.path.join(self.tempdir, 'extlayer') 300 self._create_temp_layer(exttemplayerdir, False, 'oeselftestextlayer', recipepathspec='metadata/recipes/recipes-*/*') 301 result = runCmd('recipetool appendfile %s /usr/share/selftest-replaceme-orig %s' % (exttemplayerdir, self.testfile)) 302 self.assertNotIn('Traceback', result.output) 303 createdfiles = [] 304 for root, _, files in os.walk(exttemplayerdir): 305 for f in files: 306 createdfiles.append(os.path.relpath(os.path.join(root, f), exttemplayerdir)) 307 createdfiles.remove('conf/layer.conf') 308 expectedfiles = ['metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile.bbappend', 309 'metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile/selftest-replaceme-orig'] 310 self.assertEqual(sorted(createdfiles), sorted(expectedfiles)) 311 312 def test_recipetool_appendfile_wildcard(self): 313 314 def try_appendfile_wc(options): 315 result = runCmd('recipetool appendfile %s /etc/profile %s %s' % (self.templayerdir, self.testfile, options)) 316 self.assertNotIn('Traceback', result.output) 317 bbappendfile = None 318 for root, _, files in os.walk(self.templayerdir): 319 for f in files: 320 if f.endswith('.bbappend'): 321 bbappendfile = f 322 break 323 if not bbappendfile: 324 self.fail('No bbappend file created') 325 runCmd('rm -rf %s/recipes-*' % self.templayerdir) 326 return bbappendfile 327 328 # Check without wildcard option 329 recipefn = os.path.basename(get_bb_var('FILE', 'base-files')) 330 filename = try_appendfile_wc('') 331 self.assertEqual(filename, recipefn.replace('.bb', '.bbappend')) 332 # Now check with wildcard option 333 filename = try_appendfile_wc('-w') 334 self.assertEqual(filename, recipefn.split('_')[0] + '_%.bbappend') 335 336 337class RecipetoolCreateTests(RecipetoolBase): 338 339 def test_recipetool_create(self): 340 # Try adding a recipe 341 tempsrc = os.path.join(self.tempdir, 'srctree') 342 os.makedirs(tempsrc) 343 recipefile = os.path.join(self.tempdir, 'logrotate_3.12.3.bb') 344 srcuri = 'https://github.com/logrotate/logrotate/releases/download/3.12.3/logrotate-3.12.3.tar.xz' 345 result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc)) 346 self.assertTrue(os.path.isfile(recipefile)) 347 checkvars = {} 348 checkvars['LICENSE'] = 'GPL-2.0-only' 349 checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263' 350 checkvars['SRC_URI'] = 'https://github.com/logrotate/logrotate/releases/download/${PV}/logrotate-${PV}.tar.xz' 351 checkvars['SRC_URI[md5sum]'] = 'a560c57fac87c45b2fc17406cdf79288' 352 checkvars['SRC_URI[sha256sum]'] = '2e6a401cac9024db2288297e3be1a8ab60e7401ba8e91225218aaf4a27e82a07' 353 self._test_recipe_contents(recipefile, checkvars, []) 354 355 def test_recipetool_create_autotools(self): 356 if 'x11' not in get_bb_var('DISTRO_FEATURES'): 357 self.skipTest('Test requires x11 as distro feature') 358 # Ensure we have the right data in shlibs/pkgdata 359 bitbake('libpng pango libx11 libxext jpeg libcheck') 360 # Try adding a recipe 361 tempsrc = os.path.join(self.tempdir, 'srctree') 362 os.makedirs(tempsrc) 363 recipefile = os.path.join(self.tempdir, 'libmatchbox.bb') 364 srcuri = 'git://git.yoctoproject.org/libmatchbox;protocol=https' 365 result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri + ";rev=9f7cf8895ae2d39c465c04cc78e918c157420269", '-x', tempsrc]) 366 self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) 367 checkvars = {} 368 checkvars['LICENSE'] = 'LGPL-2.1-only' 369 checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7fbc338309ac38fefcd64b04bb903e34' 370 checkvars['S'] = '${WORKDIR}/git' 371 checkvars['PV'] = '1.11+git${SRCPV}' 372 checkvars['SRC_URI'] = srcuri + ';branch=master' 373 checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxext', 'pango']) 374 inherits = ['autotools', 'pkgconfig'] 375 self._test_recipe_contents(recipefile, checkvars, inherits) 376 377 def test_recipetool_create_simple(self): 378 # Try adding a recipe 379 temprecipe = os.path.join(self.tempdir, 'recipe') 380 os.makedirs(temprecipe) 381 pv = '1.7.4.1' 382 srcuri = 'http://www.dest-unreach.org/socat/download/Archive/socat-%s.tar.bz2' % pv 383 result = runCmd('recipetool create %s -o %s' % (srcuri, temprecipe)) 384 dirlist = os.listdir(temprecipe) 385 if len(dirlist) > 1: 386 self.fail('recipetool created more than just one file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) 387 if len(dirlist) < 1 or not os.path.isfile(os.path.join(temprecipe, dirlist[0])): 388 self.fail('recipetool did not create recipe file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) 389 self.assertEqual(dirlist[0], 'socat_%s.bb' % pv, 'Recipe file incorrectly named') 390 checkvars = {} 391 checkvars['LICENSE'] = set(['Unknown', 'GPL-2.0-only']) 392 checkvars['LIC_FILES_CHKSUM'] = set(['file://COPYING.OpenSSL;md5=5c9bccc77f67a8328ef4ebaf468116f4', 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263']) 393 # We don't check DEPENDS since they are variable for this recipe depending on what's in the sysroot 394 checkvars['S'] = None 395 checkvars['SRC_URI'] = srcuri.replace(pv, '${PV}') 396 inherits = ['autotools'] 397 self._test_recipe_contents(os.path.join(temprecipe, dirlist[0]), checkvars, inherits) 398 399 def test_recipetool_create_cmake(self): 400 temprecipe = os.path.join(self.tempdir, 'recipe') 401 os.makedirs(temprecipe) 402 recipefile = os.path.join(temprecipe, 'taglib_1.11.1.bb') 403 srcuri = 'http://taglib.github.io/releases/taglib-1.11.1.tar.gz' 404 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 405 self.assertTrue(os.path.isfile(recipefile)) 406 checkvars = {} 407 checkvars['LICENSE'] = set(['LGPL-2.1-only', 'MPL-1.1-only']) 408 checkvars['SRC_URI'] = 'http://taglib.github.io/releases/taglib-${PV}.tar.gz' 409 checkvars['SRC_URI[md5sum]'] = 'cee7be0ccfc892fa433d6c837df9522a' 410 checkvars['SRC_URI[sha256sum]'] = 'b6d1a5a610aae6ff39d93de5efd0fdc787aa9e9dc1e7026fa4c961b26563526b' 411 checkvars['DEPENDS'] = set(['boost', 'zlib']) 412 inherits = ['cmake'] 413 self._test_recipe_contents(recipefile, checkvars, inherits) 414 415 def test_recipetool_create_npm(self): 416 collections = get_bb_var('BBFILE_COLLECTIONS').split() 417 if "openembedded-layer" not in collections: 418 self.skipTest("Test needs meta-oe for nodejs") 419 420 temprecipe = os.path.join(self.tempdir, 'recipe') 421 os.makedirs(temprecipe) 422 recipefile = os.path.join(temprecipe, 'savoirfairelinux-node-server-example_1.0.0.bb') 423 shrinkwrap = os.path.join(temprecipe, 'savoirfairelinux-node-server-example', 'npm-shrinkwrap.json') 424 srcuri = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 425 result = runCmd('recipetool create -o %s \'%s\'' % (temprecipe, srcuri)) 426 self.assertTrue(os.path.isfile(recipefile)) 427 self.assertTrue(os.path.isfile(shrinkwrap)) 428 checkvars = {} 429 checkvars['SUMMARY'] = 'Node Server Example' 430 checkvars['HOMEPAGE'] = 'https://github.com/savoirfairelinux/node-server-example#readme' 431 checkvars['LICENSE'] = 'BSD-3-Clause & ISC & MIT & Unknown' 432 urls = [] 433 urls.append('npm://registry.npmjs.org/;package=@savoirfairelinux/node-server-example;version=${PV}') 434 urls.append('npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json') 435 checkvars['SRC_URI'] = set(urls) 436 checkvars['S'] = '${WORKDIR}/npm' 437 checkvars['LICENSE:${PN}'] = 'MIT' 438 checkvars['LICENSE:${PN}-base64'] = 'Unknown' 439 checkvars['LICENSE:${PN}-accepts'] = 'MIT' 440 checkvars['LICENSE:${PN}-inherits'] = 'ISC' 441 inherits = ['npm'] 442 self._test_recipe_contents(recipefile, checkvars, inherits) 443 444 def test_recipetool_create_github(self): 445 # Basic test to see if github URL mangling works 446 temprecipe = os.path.join(self.tempdir, 'recipe') 447 os.makedirs(temprecipe) 448 recipefile = os.path.join(temprecipe, 'meson_git.bb') 449 srcuri = 'https://github.com/mesonbuild/meson;rev=0.32.0' 450 result = runCmd(['recipetool', 'create', '-o', temprecipe, srcuri]) 451 self.assertTrue(os.path.isfile(recipefile)) 452 checkvars = {} 453 checkvars['LICENSE'] = set(['Apache-2.0']) 454 checkvars['SRC_URI'] = 'git://github.com/mesonbuild/meson;protocol=https;branch=master' 455 inherits = ['setuptools3'] 456 self._test_recipe_contents(recipefile, checkvars, inherits) 457 458 def test_recipetool_create_python3_setuptools(self): 459 # Test creating python3 package from tarball (using setuptools3 class) 460 temprecipe = os.path.join(self.tempdir, 'recipe') 461 os.makedirs(temprecipe) 462 pn = 'python-magic' 463 pv = '0.4.15' 464 recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, pv)) 465 srcuri = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-%s.tar.gz' % pv 466 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 467 self.assertTrue(os.path.isfile(recipefile)) 468 checkvars = {} 469 checkvars['LICENSE'] = set(['MIT']) 470 checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88' 471 checkvars['SRC_URI'] = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-${PV}.tar.gz' 472 checkvars['SRC_URI[md5sum]'] = 'e384c95a47218f66c6501cd6dd45ff59' 473 checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5' 474 inherits = ['setuptools3'] 475 self._test_recipe_contents(recipefile, checkvars, inherits) 476 477 def test_recipetool_create_github_tarball(self): 478 # Basic test to ensure github URL mangling doesn't apply to release tarballs 479 temprecipe = os.path.join(self.tempdir, 'recipe') 480 os.makedirs(temprecipe) 481 pv = '0.32.0' 482 recipefile = os.path.join(temprecipe, 'meson_%s.bb' % pv) 483 srcuri = 'https://github.com/mesonbuild/meson/releases/download/%s/meson-%s.tar.gz' % (pv, pv) 484 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 485 self.assertTrue(os.path.isfile(recipefile)) 486 checkvars = {} 487 checkvars['LICENSE'] = set(['Apache-2.0']) 488 checkvars['SRC_URI'] = 'https://github.com/mesonbuild/meson/releases/download/${PV}/meson-${PV}.tar.gz' 489 inherits = ['setuptools3'] 490 self._test_recipe_contents(recipefile, checkvars, inherits) 491 492 def _test_recipetool_create_git(self, srcuri, branch=None): 493 # Basic test to check http git URL mangling works 494 temprecipe = os.path.join(self.tempdir, 'recipe') 495 os.makedirs(temprecipe) 496 name = srcuri.split(';')[0].split('/')[-1] 497 recipefile = os.path.join(temprecipe, name + '_git.bb') 498 options = ' -B %s' % branch if branch else '' 499 result = runCmd('recipetool create -o %s%s "%s"' % (temprecipe, options, srcuri)) 500 self.assertTrue(os.path.isfile(recipefile)) 501 checkvars = {} 502 checkvars['SRC_URI'] = srcuri 503 for scheme in ['http', 'https']: 504 if srcuri.startswith(scheme + ":"): 505 checkvars['SRC_URI'] = 'git%s;protocol=%s' % (srcuri[len(scheme):], scheme) 506 if ';branch=' not in srcuri: 507 checkvars['SRC_URI'] += ';branch=' + (branch or 'master') 508 self._test_recipe_contents(recipefile, checkvars, []) 509 510 def test_recipetool_create_git_http(self): 511 self._test_recipetool_create_git('http://git.yoctoproject.org/git/matchbox-keyboard') 512 513 def test_recipetool_create_git_srcuri_master(self): 514 self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;branch=master;protocol=https') 515 516 def test_recipetool_create_git_srcuri_branch(self): 517 self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;branch=matchbox-keyboard-0-1;protocol=https') 518 519 def test_recipetool_create_git_srcbranch(self): 520 self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;protocol=https', 'matchbox-keyboard-0-1') 521 522 523class RecipetoolTests(RecipetoolBase): 524 525 @classmethod 526 def setUpClass(cls): 527 import sys 528 529 super(RecipetoolTests, cls).setUpClass() 530 bb_vars = get_bb_vars(['BBPATH']) 531 cls.bbpath = bb_vars['BBPATH'] 532 libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib', 'recipetool') 533 sys.path.insert(0, libpath) 534 535 def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths): 536 dstdir = basedstdir 537 self.assertTrue(os.path.exists(dstdir)) 538 for p in paths: 539 dstdir = os.path.join(dstdir, p) 540 if not os.path.exists(dstdir): 541 os.makedirs(dstdir) 542 if p == "lib": 543 # Can race with other tests 544 self.add_command_to_tearDown('rmdir --ignore-fail-on-non-empty %s' % dstdir) 545 else: 546 self.track_for_cleanup(dstdir) 547 dstfile = os.path.join(dstdir, os.path.basename(srcfile)) 548 if srcfile != dstfile: 549 shutil.copy(srcfile, dstfile) 550 self.track_for_cleanup(dstfile) 551 552 def test_recipetool_load_plugin(self): 553 """Test that recipetool loads only the first found plugin in BBPATH.""" 554 555 recipetool = runCmd("which recipetool") 556 fromname = runCmd("recipetool --quiet pluginfile") 557 srcfile = fromname.output 558 searchpath = self.bbpath.split(':') + [os.path.dirname(recipetool.output)] 559 plugincontent = [] 560 with open(srcfile) as fh: 561 plugincontent = fh.readlines() 562 try: 563 self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found') 564 for path in searchpath: 565 self._copy_file_with_cleanup(srcfile, path, 'lib', 'recipetool') 566 result = runCmd("recipetool --quiet count") 567 self.assertEqual(result.output, '1') 568 result = runCmd("recipetool --quiet multiloaded") 569 self.assertEqual(result.output, "no") 570 for path in searchpath: 571 result = runCmd("recipetool --quiet bbdir") 572 self.assertEqual(result.output, path) 573 os.unlink(os.path.join(result.output, 'lib', 'recipetool', 'bbpath.py')) 574 finally: 575 with open(srcfile, 'w') as fh: 576 fh.writelines(plugincontent) 577 578 def test_recipetool_handle_license_vars(self): 579 from create import handle_license_vars 580 from unittest.mock import Mock 581 582 commonlicdir = get_bb_var('COMMON_LICENSE_DIR') 583 584 class DataConnectorCopy(bb.tinfoil.TinfoilDataStoreConnector): 585 pass 586 587 d = DataConnectorCopy 588 d.getVar = Mock(return_value=commonlicdir) 589 590 srctree = tempfile.mkdtemp(prefix='recipetoolqa') 591 self.track_for_cleanup(srctree) 592 593 # Multiple licenses 594 licenses = ['MIT', 'ISC', 'BSD-3-Clause', 'Apache-2.0'] 595 for licence in licenses: 596 shutil.copy(os.path.join(commonlicdir, licence), os.path.join(srctree, 'LICENSE.' + licence)) 597 # Duplicate license 598 shutil.copy(os.path.join(commonlicdir, 'MIT'), os.path.join(srctree, 'LICENSE')) 599 600 extravalues = { 601 # Duplicate and missing licenses 602 'LICENSE': 'Zlib & BSD-2-Clause & Zlib', 603 'LIC_FILES_CHKSUM': [ 604 'file://README.md;md5=0123456789abcdef0123456789abcd' 605 ] 606 } 607 lines_before = [] 608 handled = [] 609 licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d) 610 expected_lines_before = [ 611 '# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is', 612 '# your responsibility to verify that the values are complete and correct.', 613 '# NOTE: Original package / source metadata indicates license is: BSD-2-Clause & Zlib', 614 '#', 615 '# NOTE: multiple licenses have been detected; they have been separated with &', 616 '# in the LICENSE value for now since it is a reasonable assumption that all', 617 '# of the licenses apply. If instead there is a choice between the multiple', 618 '# licenses then you should change the value to separate the licenses with |', 619 '# instead of &. If there is any doubt, check the accompanying documentation', 620 '# to determine which situation is applicable.', 621 'LICENSE = "Apache-2.0 & BSD-2-Clause & BSD-3-Clause & ISC & MIT & Zlib"', 622 'LIC_FILES_CHKSUM = "file://LICENSE;md5=0835ade698e0bcf8506ecda2f7b4f302 \\\n' 623 ' file://LICENSE.Apache-2.0;md5=89aea4e17d99a7cacdbeed46a0096b10 \\\n' 624 ' file://LICENSE.BSD-3-Clause;md5=550794465ba0ec5312d6919e203a55f9 \\\n' 625 ' file://LICENSE.ISC;md5=f3b90e78ea0cffb20bf5cca7947a896d \\\n' 626 ' file://LICENSE.MIT;md5=0835ade698e0bcf8506ecda2f7b4f302 \\\n' 627 ' file://README.md;md5=0123456789abcdef0123456789abcd"', 628 '' 629 ] 630 self.assertEqual(lines_before, expected_lines_before) 631 expected_licvalues = [ 632 ('MIT', 'LICENSE', '0835ade698e0bcf8506ecda2f7b4f302'), 633 ('Apache-2.0', 'LICENSE.Apache-2.0', '89aea4e17d99a7cacdbeed46a0096b10'), 634 ('BSD-3-Clause', 'LICENSE.BSD-3-Clause', '550794465ba0ec5312d6919e203a55f9'), 635 ('ISC', 'LICENSE.ISC', 'f3b90e78ea0cffb20bf5cca7947a896d'), 636 ('MIT', 'LICENSE.MIT', '0835ade698e0bcf8506ecda2f7b4f302') 637 ] 638 self.assertEqual(handled, [('license', expected_licvalues)]) 639 self.assertEqual(extravalues, {}) 640 self.assertEqual(licvalues, expected_licvalues) 641 642 643 def test_recipetool_split_pkg_licenses(self): 644 from create import split_pkg_licenses 645 licvalues = [ 646 # Duplicate licenses 647 ('BSD-2-Clause', 'x/COPYING', None), 648 ('BSD-2-Clause', 'x/LICENSE', None), 649 # Multiple licenses 650 ('MIT', 'x/a/LICENSE.MIT', None), 651 ('ISC', 'x/a/LICENSE.ISC', None), 652 # Alternative licenses 653 ('(MIT | ISC)', 'x/b/LICENSE', None), 654 # Alternative licenses without brackets 655 ('MIT | BSD-2-Clause', 'x/c/LICENSE', None), 656 # Multi licenses with alternatives 657 ('MIT', 'x/d/COPYING', None), 658 ('MIT | BSD-2-Clause', 'x/d/LICENSE', None), 659 # Multi licenses with alternatives and brackets 660 ('Apache-2.0 & ((MIT | ISC) & BSD-3-Clause)', 'x/e/LICENSE', None) 661 ] 662 packages = { 663 '${PN}': '', 664 'a': 'x/a', 665 'b': 'x/b', 666 'c': 'x/c', 667 'd': 'x/d', 668 'e': 'x/e', 669 'f': 'x/f', 670 'g': 'x/g', 671 } 672 fallback_licenses = { 673 # Ignored 674 'a': 'BSD-3-Clause', 675 # Used 676 'f': 'BSD-3-Clause' 677 } 678 outlines = [] 679 outlicenses = split_pkg_licenses(licvalues, packages, outlines, fallback_licenses) 680 expected_outlicenses = { 681 '${PN}': ['BSD-2-Clause'], 682 'a': ['ISC', 'MIT'], 683 'b': ['(ISC | MIT)'], 684 'c': ['(BSD-2-Clause | MIT)'], 685 'd': ['(BSD-2-Clause | MIT)', 'MIT'], 686 'e': ['(ISC | MIT)', 'Apache-2.0', 'BSD-3-Clause'], 687 'f': ['BSD-3-Clause'], 688 'g': ['Unknown'] 689 } 690 self.assertEqual(outlicenses, expected_outlicenses) 691 expected_outlines = [ 692 'LICENSE:${PN} = "BSD-2-Clause"', 693 'LICENSE:a = "ISC & MIT"', 694 'LICENSE:b = "(ISC | MIT)"', 695 'LICENSE:c = "(BSD-2-Clause | MIT)"', 696 'LICENSE:d = "(BSD-2-Clause | MIT) & MIT"', 697 'LICENSE:e = "(ISC | MIT) & Apache-2.0 & BSD-3-Clause"', 698 'LICENSE:f = "BSD-3-Clause"', 699 'LICENSE:g = "Unknown"' 700 ] 701 self.assertEqual(outlines, expected_outlines) 702 703 704class RecipetoolAppendsrcBase(RecipetoolBase): 705 def _try_recipetool_appendsrcfile(self, testrecipe, newfile, destfile, options, expectedlines, expectedfiles): 706 cmd = 'recipetool appendsrcfile %s %s %s %s %s' % (options, self.templayerdir, testrecipe, newfile, destfile) 707 return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) 708 709 def _try_recipetool_appendsrcfiles(self, testrecipe, newfiles, expectedlines=None, expectedfiles=None, destdir=None, options=''): 710 711 if destdir: 712 options += ' -D %s' % destdir 713 714 if expectedfiles is None: 715 expectedfiles = [os.path.basename(f) for f in newfiles] 716 717 cmd = 'recipetool appendsrcfiles %s %s %s %s' % (options, self.templayerdir, testrecipe, ' '.join(newfiles)) 718 return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) 719 720 def _try_recipetool_appendsrcfile_fail(self, testrecipe, newfile, destfile, checkerror): 721 cmd = 'recipetool appendsrcfile %s %s %s %s' % (self.templayerdir, testrecipe, newfile, destfile or '') 722 result = runCmd(cmd, ignore_status=True) 723 self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd) 724 self.assertNotIn('Traceback', result.output) 725 for errorstr in checkerror: 726 self.assertIn(errorstr, result.output) 727 728 @staticmethod 729 def _get_first_file_uri(recipe): 730 '''Return the first file:// in SRC_URI for the specified recipe.''' 731 src_uri = get_bb_var('SRC_URI', recipe).split() 732 for uri in src_uri: 733 p = urllib.parse.urlparse(uri) 734 if p.scheme == 'file': 735 return p.netloc + p.path 736 737 def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, options=''): 738 if newfile is None: 739 newfile = self.testfile 740 741 if srcdir: 742 if destdir: 743 expected_subdir = os.path.join(srcdir, destdir) 744 else: 745 expected_subdir = srcdir 746 else: 747 options += " -W" 748 expected_subdir = destdir 749 750 if filename: 751 if destdir: 752 destpath = os.path.join(destdir, filename) 753 else: 754 destpath = filename 755 else: 756 filename = os.path.basename(newfile) 757 if destdir: 758 destpath = destdir + os.sep 759 else: 760 destpath = '.' + os.sep 761 762 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 763 '\n'] 764 if has_src_uri: 765 uri = 'file://%s' % filename 766 if expected_subdir: 767 uri += ';subdir=%s' % expected_subdir 768 expectedlines[0:0] = ['SRC_URI += "%s"\n' % uri, 769 '\n'] 770 771 return self._try_recipetool_appendsrcfile(testrecipe, newfile, destpath, options, expectedlines, [filename]) 772 773 def _test_appendsrcfiles(self, testrecipe, newfiles, expectedfiles=None, destdir=None, options=''): 774 if expectedfiles is None: 775 expectedfiles = [os.path.basename(n) for n in newfiles] 776 777 self._try_recipetool_appendsrcfiles(testrecipe, newfiles, expectedfiles=expectedfiles, destdir=destdir, options=options) 778 779 bb_vars = get_bb_vars(['SRC_URI', 'FILE', 'FILESEXTRAPATHS'], testrecipe) 780 src_uri = bb_vars['SRC_URI'].split() 781 for f in expectedfiles: 782 if destdir: 783 self.assertIn('file://%s;subdir=%s' % (f, destdir), src_uri) 784 else: 785 self.assertIn('file://%s' % f, src_uri) 786 787 recipefile = bb_vars['FILE'] 788 bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) 789 filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe) 790 filesextrapaths = bb_vars['FILESEXTRAPATHS'].split(':') 791 self.assertIn(filesdir, filesextrapaths) 792 793 794 795 796class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): 797 798 def test_recipetool_appendsrcfile_basic(self): 799 self._test_appendsrcfile('base-files', 'a-file') 800 801 def test_recipetool_appendsrcfile_basic_wildcard(self): 802 testrecipe = 'base-files' 803 self._test_appendsrcfile(testrecipe, 'a-file', options='-w') 804 recipefile = get_bb_var('FILE', testrecipe) 805 bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) 806 self.assertEqual(os.path.basename(bbappendfile), '%s_%%.bbappend' % testrecipe) 807 808 def test_recipetool_appendsrcfile_subdir_basic(self): 809 self._test_appendsrcfile('base-files', 'a-file', 'tmp') 810 811 def test_recipetool_appendsrcfile_subdir_basic_dirdest(self): 812 self._test_appendsrcfile('base-files', destdir='tmp') 813 814 def test_recipetool_appendsrcfile_srcdir_basic(self): 815 testrecipe = 'bash' 816 bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe) 817 srcdir = bb_vars['S'] 818 workdir = bb_vars['WORKDIR'] 819 subdir = os.path.relpath(srcdir, workdir) 820 self._test_appendsrcfile(testrecipe, 'a-file', srcdir=subdir) 821 822 def test_recipetool_appendsrcfile_existing_in_src_uri(self): 823 testrecipe = 'base-files' 824 filepath = self._get_first_file_uri(testrecipe) 825 self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) 826 self._test_appendsrcfile(testrecipe, filepath, has_src_uri=False) 827 828 def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self): 829 testrecipe = 'base-files' 830 subdir = 'tmp' 831 filepath = self._get_first_file_uri(testrecipe) 832 self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) 833 834 output = self._test_appendsrcfile(testrecipe, filepath, subdir, has_src_uri=False) 835 self.assertTrue(any('with different parameters' in l for l in output)) 836 837 def test_recipetool_appendsrcfile_replace_file_srcdir(self): 838 testrecipe = 'bash' 839 filepath = 'Makefile.in' 840 bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe) 841 srcdir = bb_vars['S'] 842 workdir = bb_vars['WORKDIR'] 843 subdir = os.path.relpath(srcdir, workdir) 844 845 self._test_appendsrcfile(testrecipe, filepath, srcdir=subdir) 846 bitbake('%s:do_unpack' % testrecipe) 847 with open(self.testfile, 'r') as testfile: 848 with open(os.path.join(srcdir, filepath), 'r') as makefilein: 849 self.assertEqual(testfile.read(), makefilein.read()) 850 851 def test_recipetool_appendsrcfiles_basic(self, destdir=None): 852 newfiles = [self.testfile] 853 for i in range(1, 5): 854 testfile = os.path.join(self.tempdir, 'testfile%d' % i) 855 with open(testfile, 'w') as f: 856 f.write('Test file %d\n' % i) 857 newfiles.append(testfile) 858 self._test_appendsrcfiles('gcc', newfiles, destdir=destdir, options='-W') 859 860 def test_recipetool_appendsrcfiles_basic_subdir(self): 861 self.test_recipetool_appendsrcfiles_basic(destdir='testdir') 862