1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: MIT 5# 6 7import errno 8import os 9import shutil 10import tempfile 11import urllib.parse 12 13from oeqa.utils.commands import runCmd, bitbake, get_bb_var 14from oeqa.utils.commands import get_bb_vars, create_temp_layer 15from oeqa.selftest.cases import devtool 16 17templayerdir = None 18 19def setUpModule(): 20 global templayerdir 21 templayerdir = tempfile.mkdtemp(prefix='recipetoolqa') 22 create_temp_layer(templayerdir, 'selftestrecipetool') 23 runCmd('bitbake-layers add-layer %s' % templayerdir) 24 25 26def tearDownModule(): 27 runCmd('bitbake-layers remove-layer %s' % templayerdir, ignore_status=True) 28 runCmd('rm -rf %s' % templayerdir) 29 30 31def needTomllib(test): 32 # This test require python 3.11 or above for the tomllib module or tomli module to be installed 33 try: 34 import tomllib 35 except ImportError: 36 try: 37 import tomli 38 except ImportError: 39 test.skipTest('Test requires python 3.11 or above for tomllib module or tomli module') 40 41class RecipetoolBase(devtool.DevtoolTestCase): 42 43 def setUpLocal(self): 44 super(RecipetoolBase, self).setUpLocal() 45 self.templayerdir = templayerdir 46 self.tempdir = tempfile.mkdtemp(prefix='recipetoolqa') 47 self.track_for_cleanup(self.tempdir) 48 self.testfile = os.path.join(self.tempdir, 'testfile') 49 with open(self.testfile, 'w') as f: 50 f.write('Test file\n') 51 config = 'BBMASK += "meta-poky/recipes-core/base-files/base-files_%.bbappend"\n' 52 self.append_config(config) 53 54 def tearDownLocal(self): 55 runCmd('rm -rf %s/recipes-*' % self.templayerdir) 56 super(RecipetoolBase, self).tearDownLocal() 57 58 def _try_recipetool_appendcmd(self, cmd, testrecipe, expectedfiles, expectedlines=None): 59 result = runCmd(cmd) 60 self.assertNotIn('Traceback', result.output) 61 62 # Check the bbappend was created and applies properly 63 recipefile = get_bb_var('FILE', testrecipe) 64 bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) 65 66 # Check the bbappend contents 67 if expectedlines is not None: 68 with open(bbappendfile, 'r') as f: 69 self.assertEqual(expectedlines, f.readlines(), "Expected lines are not present in %s" % bbappendfile) 70 71 # Check file was copied 72 filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe) 73 for expectedfile in expectedfiles: 74 self.assertTrue(os.path.isfile(os.path.join(filesdir, expectedfile)), 'Expected file %s to be copied next to bbappend, but it wasn\'t' % expectedfile) 75 76 # Check no other files created 77 createdfiles = [] 78 for root, _, files in os.walk(filesdir): 79 for f in files: 80 createdfiles.append(os.path.relpath(os.path.join(root, f), filesdir)) 81 self.assertTrue(sorted(createdfiles), sorted(expectedfiles)) 82 83 return bbappendfile, result.output 84 85 86class RecipetoolAppendTests(RecipetoolBase): 87 88 @classmethod 89 def setUpClass(cls): 90 super(RecipetoolAppendTests, cls).setUpClass() 91 # Ensure we have the right data in shlibs/pkgdata 92 cls.logger.info('Running bitbake to generate pkgdata') 93 bitbake('-c packagedata base-files coreutils busybox selftest-recipetool-appendfile') 94 bb_vars = get_bb_vars(['COREBASE']) 95 cls.corebase = bb_vars['COREBASE'] 96 97 def _try_recipetool_appendfile(self, testrecipe, destfile, newfile, options, expectedlines, expectedfiles): 98 cmd = 'recipetool appendfile %s %s %s %s' % (self.templayerdir, destfile, newfile, options) 99 return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) 100 101 def _try_recipetool_appendfile_fail(self, destfile, newfile, checkerror): 102 cmd = 'recipetool appendfile %s %s %s' % (self.templayerdir, destfile, newfile) 103 result = runCmd(cmd, ignore_status=True) 104 self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd) 105 self.assertNotIn('Traceback', result.output) 106 for errorstr in checkerror: 107 self.assertIn(errorstr, result.output) 108 109 def test_recipetool_appendfile_basic(self): 110 # Basic test 111 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 112 '\n'] 113 _, output = self._try_recipetool_appendfile('base-files', '/etc/motd', self.testfile, '', expectedlines, ['motd']) 114 self.assertNotIn('WARNING: ', output) 115 116 def test_recipetool_appendfile_invalid(self): 117 # Test some commands that should error 118 self._try_recipetool_appendfile_fail('/etc/passwd', self.testfile, ['ERROR: /etc/passwd cannot be handled by this tool', 'useradd', 'extrausers']) 119 self._try_recipetool_appendfile_fail('/etc/timestamp', self.testfile, ['ERROR: /etc/timestamp cannot be handled by this tool']) 120 self._try_recipetool_appendfile_fail('/dev/console', self.testfile, ['ERROR: /dev/console cannot be handled by this tool']) 121 122 def test_recipetool_appendfile_alternatives(self): 123 lspath = '/bin/ls' 124 dirname = "base_bindir" 125 if "usrmerge" in get_bb_var('DISTRO_FEATURES'): 126 lspath = '/usr/bin/ls' 127 dirname = "bindir" 128 129 # Now try with a file we know should be an alternative 130 # (this is very much a fake example, but one we know is reliably an alternative) 131 self._try_recipetool_appendfile_fail(lspath, self.testfile, ['ERROR: File %s is an alternative possibly provided by the following recipes:' % lspath, 'coreutils', 'busybox']) 132 # Need a test file - should be executable 133 testfile2 = os.path.join(self.corebase, 'oe-init-build-env') 134 testfile2name = os.path.basename(testfile2) 135 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 136 '\n', 137 'SRC_URI += "file://%s"\n' % testfile2name, 138 '\n', 139 'do_install:append() {\n', 140 ' install -d ${D}${%s}\n' % dirname, 141 ' install -m 0755 ${UNPACKDIR}/%s ${D}${%s}/ls\n' % (testfile2name, dirname), 142 '}\n'] 143 self._try_recipetool_appendfile('coreutils', lspath, testfile2, '-r coreutils', expectedlines, [testfile2name]) 144 # Now try bbappending the same file again, contents should not change 145 bbappendfile, _ = self._try_recipetool_appendfile('coreutils', lspath, self.testfile, '-r coreutils', expectedlines, [testfile2name]) 146 # But file should have 147 copiedfile = os.path.join(os.path.dirname(bbappendfile), 'coreutils', testfile2name) 148 result = runCmd('diff -q %s %s' % (testfile2, copiedfile), ignore_status=True) 149 self.assertNotEqual(result.status, 0, 'New file should have been copied but was not %s' % result.output) 150 151 def test_recipetool_appendfile_binary(self): 152 # Try appending a binary file 153 # /bin/ls can be a symlink to /usr/bin/ls 154 ls = os.path.realpath("/bin/ls") 155 result = runCmd('recipetool appendfile %s /bin/ls %s -r coreutils' % (self.templayerdir, ls)) 156 self.assertIn('WARNING: ', result.output) 157 self.assertIn('is a binary', result.output) 158 159 def test_recipetool_appendfile_add(self): 160 # Try arbitrary file add to a recipe 161 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 162 '\n', 163 'SRC_URI += "file://testfile"\n', 164 '\n', 165 'do_install:append() {\n', 166 ' install -d ${D}${datadir}\n', 167 ' install -m 0644 ${UNPACKDIR}/testfile ${D}${datadir}/something\n', 168 '}\n'] 169 self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase', expectedlines, ['testfile']) 170 # Try adding another file, this time where the source file is executable 171 # (so we're testing that, plus modifying an existing bbappend) 172 testfile2 = os.path.join(self.corebase, 'oe-init-build-env') 173 testfile2name = os.path.basename(testfile2) 174 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 175 '\n', 176 'SRC_URI += "file://testfile \\\n', 177 ' file://%s \\\n' % testfile2name, 178 ' "\n', 179 '\n', 180 'do_install:append() {\n', 181 ' install -d ${D}${datadir}\n', 182 ' install -m 0644 ${UNPACKDIR}/testfile ${D}${datadir}/something\n', 183 ' install -m 0755 ${UNPACKDIR}/%s ${D}${datadir}/scriptname\n' % testfile2name, 184 '}\n'] 185 self._try_recipetool_appendfile('netbase', '/usr/share/scriptname', testfile2, '-r netbase', expectedlines, ['testfile', testfile2name]) 186 187 def test_recipetool_appendfile_add_bindir(self): 188 # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable 189 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 190 '\n', 191 'SRC_URI += "file://testfile"\n', 192 '\n', 193 'do_install:append() {\n', 194 ' install -d ${D}${bindir}\n', 195 ' install -m 0755 ${UNPACKDIR}/testfile ${D}${bindir}/selftest-recipetool-testbin\n', 196 '}\n'] 197 _, output = self._try_recipetool_appendfile('netbase', '/usr/bin/selftest-recipetool-testbin', self.testfile, '-r netbase', expectedlines, ['testfile']) 198 self.assertNotIn('WARNING: ', output) 199 200 def test_recipetool_appendfile_add_machine(self): 201 # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable 202 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 203 '\n', 204 'PACKAGE_ARCH = "${MACHINE_ARCH}"\n', 205 '\n', 206 'SRC_URI:append:mymachine = " file://testfile"\n', 207 '\n', 208 'do_install:append:mymachine() {\n', 209 ' install -d ${D}${datadir}\n', 210 ' install -m 0644 ${UNPACKDIR}/testfile ${D}${datadir}/something\n', 211 '}\n'] 212 _, output = self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase -m mymachine', expectedlines, ['mymachine/testfile']) 213 self.assertNotIn('WARNING: ', output) 214 215 def test_recipetool_appendfile_orig(self): 216 # A file that's in SRC_URI and in do_install with the same name 217 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 218 '\n'] 219 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-orig', self.testfile, '', expectedlines, ['selftest-replaceme-orig']) 220 self.assertNotIn('WARNING: ', output) 221 222 def test_recipetool_appendfile_todir(self): 223 # A file that's in SRC_URI and in do_install with destination directory rather than file 224 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 225 '\n'] 226 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-todir', self.testfile, '', expectedlines, ['selftest-replaceme-todir']) 227 self.assertNotIn('WARNING: ', output) 228 229 def test_recipetool_appendfile_renamed(self): 230 # A file that's in SRC_URI with a different name to the destination file 231 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 232 '\n'] 233 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-renamed', self.testfile, '', expectedlines, ['file1']) 234 self.assertNotIn('WARNING: ', output) 235 236 def test_recipetool_appendfile_subdir(self): 237 # A file that's in SRC_URI in a subdir 238 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 239 '\n', 240 'SRC_URI += "file://testfile"\n', 241 '\n', 242 'do_install:append() {\n', 243 ' install -d ${D}${datadir}\n', 244 ' install -m 0644 ${UNPACKDIR}/testfile ${D}${datadir}/selftest-replaceme-subdir\n', 245 '}\n'] 246 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-subdir', self.testfile, '', expectedlines, ['testfile']) 247 self.assertNotIn('WARNING: ', output) 248 249 def test_recipetool_appendfile_inst_glob(self): 250 # A file that's in do_install as a glob 251 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 252 '\n'] 253 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-globfile']) 254 self.assertNotIn('WARNING: ', output) 255 256 def test_recipetool_appendfile_inst_todir_glob(self): 257 # A file that's in do_install as a glob with destination as a directory 258 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 259 '\n'] 260 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-todir-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-todir-globfile']) 261 self.assertNotIn('WARNING: ', output) 262 263 def test_recipetool_appendfile_patch(self): 264 # A file that's added by a patch in SRC_URI 265 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 266 '\n', 267 'SRC_URI += "file://testfile"\n', 268 '\n', 269 'do_install:append() {\n', 270 ' install -d ${D}${sysconfdir}\n', 271 ' install -m 0644 ${UNPACKDIR}/testfile ${D}${sysconfdir}/selftest-replaceme-patched\n', 272 '}\n'] 273 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/etc/selftest-replaceme-patched', self.testfile, '', expectedlines, ['testfile']) 274 for line in output.splitlines(): 275 if 'WARNING: ' in line: 276 self.assertIn('add-file.patch', line, 'Unexpected warning found in output:\n%s' % line) 277 break 278 else: 279 self.fail('Patch warning not found in output:\n%s' % output) 280 281 def test_recipetool_appendfile_script(self): 282 # Now, a file that's in SRC_URI but installed by a script (so no mention in do_install) 283 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 284 '\n', 285 'SRC_URI += "file://testfile"\n', 286 '\n', 287 'do_install:append() {\n', 288 ' install -d ${D}${datadir}\n', 289 ' install -m 0644 ${UNPACKDIR}/testfile ${D}${datadir}/selftest-replaceme-scripted\n', 290 '}\n'] 291 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-scripted', self.testfile, '', expectedlines, ['testfile']) 292 self.assertNotIn('WARNING: ', output) 293 294 def test_recipetool_appendfile_inst_func(self): 295 # A file that's installed from a function called by do_install 296 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 297 '\n'] 298 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-func', self.testfile, '', expectedlines, ['selftest-replaceme-inst-func']) 299 self.assertNotIn('WARNING: ', output) 300 301 def test_recipetool_appendfile_postinstall(self): 302 # A file that's created by a postinstall script (and explicitly mentioned in it) 303 # First try without specifying recipe 304 self._try_recipetool_appendfile_fail('/usr/share/selftest-replaceme-postinst', self.testfile, ['File /usr/share/selftest-replaceme-postinst may be written out in a pre/postinstall script of the following recipes:', 'selftest-recipetool-appendfile']) 305 # Now specify recipe 306 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 307 '\n', 308 'SRC_URI += "file://testfile"\n', 309 '\n', 310 'do_install:append() {\n', 311 ' install -d ${D}${datadir}\n', 312 ' install -m 0644 ${UNPACKDIR}/testfile ${D}${datadir}/selftest-replaceme-postinst\n', 313 '}\n'] 314 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-postinst', self.testfile, '-r selftest-recipetool-appendfile', expectedlines, ['testfile']) 315 316 def test_recipetool_appendfile_extlayer(self): 317 # Try creating a bbappend in a layer that's not in bblayers.conf and has a different structure 318 exttemplayerdir = os.path.join(self.tempdir, 'extlayer') 319 self._create_temp_layer(exttemplayerdir, False, 'oeselftestextlayer', recipepathspec='metadata/recipes/recipes-*/*') 320 result = runCmd('recipetool appendfile %s /usr/share/selftest-replaceme-orig %s' % (exttemplayerdir, self.testfile)) 321 self.assertNotIn('Traceback', result.output) 322 createdfiles = [] 323 for root, _, files in os.walk(exttemplayerdir): 324 for f in files: 325 createdfiles.append(os.path.relpath(os.path.join(root, f), exttemplayerdir)) 326 createdfiles.remove('conf/layer.conf') 327 expectedfiles = ['metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile.bbappend', 328 'metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile/selftest-replaceme-orig'] 329 self.assertEqual(sorted(createdfiles), sorted(expectedfiles)) 330 331 def test_recipetool_appendfile_wildcard(self): 332 333 def try_appendfile_wc(options): 334 result = runCmd('recipetool appendfile %s /etc/profile %s %s' % (self.templayerdir, self.testfile, options)) 335 self.assertNotIn('Traceback', result.output) 336 bbappendfile = None 337 for root, _, files in os.walk(self.templayerdir): 338 for f in files: 339 if f.endswith('.bbappend'): 340 bbappendfile = f 341 break 342 if not bbappendfile: 343 self.fail('No bbappend file created') 344 runCmd('rm -rf %s/recipes-*' % self.templayerdir) 345 return bbappendfile 346 347 # Check without wildcard option 348 recipefn = os.path.basename(get_bb_var('FILE', 'base-files')) 349 filename = try_appendfile_wc('') 350 self.assertEqual(filename, recipefn.replace('.bb', '.bbappend')) 351 # Now check with wildcard option 352 filename = try_appendfile_wc('-w') 353 self.assertEqual(filename, recipefn.split('_')[0] + '_%.bbappend') 354 355 356class RecipetoolCreateTests(RecipetoolBase): 357 358 def test_recipetool_create(self): 359 # Try adding a recipe 360 tempsrc = os.path.join(self.tempdir, 'srctree') 361 os.makedirs(tempsrc) 362 recipefile = os.path.join(self.tempdir, 'logrotate_3.12.3.bb') 363 srcuri = 'https://github.com/logrotate/logrotate/releases/download/3.12.3/logrotate-3.12.3.tar.xz' 364 result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc)) 365 self.assertTrue(os.path.isfile(recipefile)) 366 checkvars = {} 367 checkvars['LICENSE'] = 'GPL-2.0-only' 368 checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263' 369 checkvars['SRC_URI'] = 'https://github.com/logrotate/logrotate/releases/download/${PV}/logrotate-${PV}.tar.xz' 370 checkvars['SRC_URI[sha256sum]'] = '2e6a401cac9024db2288297e3be1a8ab60e7401ba8e91225218aaf4a27e82a07' 371 self._test_recipe_contents(recipefile, checkvars, []) 372 373 def test_recipetool_create_autotools(self): 374 if 'x11' not in get_bb_var('DISTRO_FEATURES'): 375 self.skipTest('Test requires x11 as distro feature') 376 # Ensure we have the right data in shlibs/pkgdata 377 bitbake('libpng pango libx11 libxext jpeg libcheck') 378 # Try adding a recipe 379 tempsrc = os.path.join(self.tempdir, 'srctree') 380 os.makedirs(tempsrc) 381 recipefile = os.path.join(self.tempdir, 'libmatchbox.bb') 382 srcuri = 'git://git.yoctoproject.org/libmatchbox;protocol=https' 383 result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri + ";rev=9f7cf8895ae2d39c465c04cc78e918c157420269", '-x', tempsrc]) 384 self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) 385 checkvars = {} 386 checkvars['LICENSE'] = 'LGPL-2.1-only' 387 checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7fbc338309ac38fefcd64b04bb903e34' 388 checkvars['S'] = '${WORKDIR}/git' 389 checkvars['PV'] = '1.11+git' 390 checkvars['SRC_URI'] = srcuri + ';branch=master' 391 checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxext', 'pango']) 392 inherits = ['autotools', 'pkgconfig'] 393 self._test_recipe_contents(recipefile, checkvars, inherits) 394 395 def test_recipetool_create_simple(self): 396 # Try adding a recipe 397 temprecipe = os.path.join(self.tempdir, 'recipe') 398 os.makedirs(temprecipe) 399 pv = '1.7.4.1' 400 srcuri = 'http://www.dest-unreach.org/socat/download/Archive/socat-%s.tar.bz2' % pv 401 result = runCmd('recipetool create %s -o %s' % (srcuri, temprecipe)) 402 dirlist = os.listdir(temprecipe) 403 if len(dirlist) > 1: 404 self.fail('recipetool created more than just one file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) 405 if len(dirlist) < 1 or not os.path.isfile(os.path.join(temprecipe, dirlist[0])): 406 self.fail('recipetool did not create recipe file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) 407 self.assertEqual(dirlist[0], 'socat_%s.bb' % pv, 'Recipe file incorrectly named') 408 checkvars = {} 409 checkvars['LICENSE'] = set(['Unknown', 'GPL-2.0-only']) 410 checkvars['LIC_FILES_CHKSUM'] = set(['file://COPYING.OpenSSL;md5=5c9bccc77f67a8328ef4ebaf468116f4', 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263']) 411 # We don't check DEPENDS since they are variable for this recipe depending on what's in the sysroot 412 checkvars['S'] = None 413 checkvars['SRC_URI'] = srcuri.replace(pv, '${PV}') 414 inherits = ['autotools'] 415 self._test_recipe_contents(os.path.join(temprecipe, dirlist[0]), checkvars, inherits) 416 417 def test_recipetool_create_cmake(self): 418 temprecipe = os.path.join(self.tempdir, 'recipe') 419 os.makedirs(temprecipe) 420 recipefile = os.path.join(temprecipe, 'taglib_1.11.1.bb') 421 srcuri = 'http://taglib.github.io/releases/taglib-1.11.1.tar.gz' 422 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 423 self.assertTrue(os.path.isfile(recipefile)) 424 checkvars = {} 425 checkvars['LICENSE'] = set(['LGPL-2.1-only', 'MPL-1.1-only']) 426 checkvars['SRC_URI'] = 'http://taglib.github.io/releases/taglib-${PV}.tar.gz' 427 checkvars['SRC_URI[sha256sum]'] = 'b6d1a5a610aae6ff39d93de5efd0fdc787aa9e9dc1e7026fa4c961b26563526b' 428 checkvars['DEPENDS'] = set(['boost', 'zlib']) 429 inherits = ['cmake'] 430 self._test_recipe_contents(recipefile, checkvars, inherits) 431 432 def test_recipetool_create_npm(self): 433 collections = get_bb_var('BBFILE_COLLECTIONS').split() 434 if "openembedded-layer" not in collections: 435 self.skipTest("Test needs meta-oe for nodejs") 436 437 temprecipe = os.path.join(self.tempdir, 'recipe') 438 os.makedirs(temprecipe) 439 recipefile = os.path.join(temprecipe, 'savoirfairelinux-node-server-example_1.0.0.bb') 440 shrinkwrap = os.path.join(temprecipe, 'savoirfairelinux-node-server-example', 'npm-shrinkwrap.json') 441 srcuri = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 442 result = runCmd('recipetool create -o %s \'%s\'' % (temprecipe, srcuri)) 443 self.assertTrue(os.path.isfile(recipefile)) 444 self.assertTrue(os.path.isfile(shrinkwrap)) 445 checkvars = {} 446 checkvars['SUMMARY'] = 'Node Server Example' 447 checkvars['HOMEPAGE'] = 'https://github.com/savoirfairelinux/node-server-example#readme' 448 checkvars['LICENSE'] = 'BSD-3-Clause & ISC & MIT & Unknown' 449 urls = [] 450 urls.append('npm://registry.npmjs.org/;package=@savoirfairelinux/node-server-example;version=${PV}') 451 urls.append('npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json') 452 checkvars['SRC_URI'] = set(urls) 453 checkvars['S'] = '${WORKDIR}/npm' 454 checkvars['LICENSE:${PN}'] = 'MIT' 455 checkvars['LICENSE:${PN}-base64'] = 'Unknown' 456 checkvars['LICENSE:${PN}-accepts'] = 'MIT' 457 checkvars['LICENSE:${PN}-inherits'] = 'ISC' 458 inherits = ['npm'] 459 self._test_recipe_contents(recipefile, checkvars, inherits) 460 461 def test_recipetool_create_github(self): 462 # Basic test to see if github URL mangling works. Deliberately use an 463 # older release of Meson at present so we don't need a toml parser. 464 temprecipe = os.path.join(self.tempdir, 'recipe') 465 os.makedirs(temprecipe) 466 recipefile = os.path.join(temprecipe, 'python3-meson_git.bb') 467 srcuri = 'https://github.com/mesonbuild/meson;rev=0.52.1' 468 cmd = ['recipetool', 'create', '-o', temprecipe, srcuri] 469 result = runCmd(cmd) 470 self.assertTrue(os.path.isfile(recipefile), msg="recipe %s not created for command %s, output %s" % (recipefile, " ".join(cmd), result.output)) 471 checkvars = {} 472 checkvars['LICENSE'] = set(['Apache-2.0', "Unknown"]) 473 checkvars['SRC_URI'] = 'git://github.com/mesonbuild/meson;protocol=https;branch=0.52' 474 inherits = ['setuptools3'] 475 self._test_recipe_contents(recipefile, checkvars, inherits) 476 477 def test_recipetool_create_python3_setuptools(self): 478 # Test creating python3 package from tarball (using setuptools3 class) 479 # Use the --no-pypi switch to avoid creating a pypi enabled recipe and 480 # and check the created recipe as if it was a more general tarball 481 temprecipe = os.path.join(self.tempdir, 'recipe') 482 os.makedirs(temprecipe) 483 pn = 'python-magic' 484 pv = '0.4.15' 485 recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, pv)) 486 srcuri = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-%s.tar.gz' % pv 487 result = runCmd('recipetool create --no-pypi -o %s %s' % (temprecipe, srcuri)) 488 self.assertTrue(os.path.isfile(recipefile)) 489 checkvars = {} 490 checkvars['LICENSE'] = set(['MIT']) 491 checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88' 492 checkvars['SRC_URI'] = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-${PV}.tar.gz' 493 checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5' 494 inherits = ['setuptools3'] 495 self._test_recipe_contents(recipefile, checkvars, inherits) 496 497 def test_recipetool_create_python3_setuptools_pypi_tarball(self): 498 # Test creating python3 package from tarball (using setuptools3 and pypi classes) 499 temprecipe = os.path.join(self.tempdir, 'recipe') 500 os.makedirs(temprecipe) 501 pn = 'python-magic' 502 pv = '0.4.15' 503 recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, pv)) 504 srcuri = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-%s.tar.gz' % pv 505 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 506 self.assertTrue(os.path.isfile(recipefile)) 507 checkvars = {} 508 checkvars['LICENSE'] = set(['MIT']) 509 checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88' 510 checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5' 511 checkvars['PYPI_PACKAGE'] = pn 512 inherits = ['setuptools3', 'pypi'] 513 self._test_recipe_contents(recipefile, checkvars, inherits) 514 515 def test_recipetool_create_python3_setuptools_pypi(self): 516 # Test creating python3 package from pypi url (using setuptools3 and pypi classes) 517 # Intentionnaly using setuptools3 class here instead of any of the pep517 class 518 # to avoid the toml dependency and allows this test to run on host autobuilders 519 # with older version of python 520 temprecipe = os.path.join(self.tempdir, 'recipe') 521 os.makedirs(temprecipe) 522 pn = 'python-magic' 523 pv = '0.4.15' 524 recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, pv)) 525 # First specify the required version in the url 526 srcuri = 'https://pypi.org/project/%s/%s' % (pn, pv) 527 runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 528 self.assertTrue(os.path.isfile(recipefile)) 529 checkvars = {} 530 checkvars['LICENSE'] = set(['MIT']) 531 checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88' 532 checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5' 533 checkvars['PYPI_PACKAGE'] = pn 534 inherits = ['setuptools3', "pypi"] 535 self._test_recipe_contents(recipefile, checkvars, inherits) 536 537 # Now specify the version as a recipetool parameter 538 runCmd('rm -rf %s' % recipefile) 539 self.assertFalse(os.path.isfile(recipefile)) 540 srcuri = 'https://pypi.org/project/%s' % pn 541 runCmd('recipetool create -o %s %s --version %s' % (temprecipe, srcuri, pv)) 542 self.assertTrue(os.path.isfile(recipefile)) 543 checkvars = {} 544 checkvars['LICENSE'] = set(['MIT']) 545 checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88' 546 checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5' 547 checkvars['PYPI_PACKAGE'] = pn 548 inherits = ['setuptools3', "pypi"] 549 self._test_recipe_contents(recipefile, checkvars, inherits) 550 551 # Now, try to grab latest version of the package, so we cannot guess the name of the recipe, 552 # unless hardcoding the latest version but it means we will need to update the test for each release, 553 # so use a regexp 554 runCmd('rm -rf %s' % recipefile) 555 self.assertFalse(os.path.isfile(recipefile)) 556 recipefile_re = r'%s_(.*)\.bb' % pn 557 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 558 dirlist = os.listdir(temprecipe) 559 if len(dirlist) > 1: 560 self.fail('recipetool created more than just one file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) 561 if len(dirlist) < 1 or not os.path.isfile(os.path.join(temprecipe, dirlist[0])): 562 self.fail('recipetool did not create recipe file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) 563 import re 564 match = re.match(recipefile_re, dirlist[0]) 565 self.assertTrue(match) 566 latest_pv = match.group(1) 567 self.assertTrue(latest_pv != pv) 568 recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, latest_pv)) 569 # Do not check LIC_FILES_CHKSUM and SRC_URI checksum here to avoid having updating the test on each release 570 checkvars = {} 571 checkvars['LICENSE'] = set(['MIT']) 572 checkvars['PYPI_PACKAGE'] = pn 573 inherits = ['setuptools3', "pypi"] 574 self._test_recipe_contents(recipefile, checkvars, inherits) 575 576 def test_recipetool_create_python3_pep517_setuptools_build_meta(self): 577 # This test require python 3.11 or above for the tomllib module or tomli module to be installed 578 needTomllib(self) 579 580 # Test creating python3 package from tarball (using setuptools.build_meta class) 581 temprecipe = os.path.join(self.tempdir, 'recipe') 582 os.makedirs(temprecipe) 583 pn = 'webcolors' 584 pv = '1.13' 585 recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) 586 srcuri = 'https://files.pythonhosted.org/packages/a1/fb/f95560c6a5d4469d9c49e24cf1b5d4d21ffab5608251c6020a965fb7791c/%s-%s.tar.gz' % (pn, pv) 587 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 588 self.assertTrue(os.path.isfile(recipefile)) 589 checkvars = {} 590 checkvars['SUMMARY'] = 'A library for working with the color formats defined by HTML and CSS.' 591 checkvars['LICENSE'] = set(['BSD-3-Clause']) 592 checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=702b1ef12cf66832a88f24c8f2ee9c19' 593 checkvars['SRC_URI[sha256sum]'] = 'c225b674c83fa923be93d235330ce0300373d02885cef23238813b0d5668304a' 594 inherits = ['python_setuptools_build_meta', 'pypi'] 595 596 self._test_recipe_contents(recipefile, checkvars, inherits) 597 598 def test_recipetool_create_python3_pep517_poetry_core_masonry_api(self): 599 # This test require python 3.11 or above for the tomllib module or tomli module to be installed 600 needTomllib(self) 601 602 # Test creating python3 package from tarball (using poetry.core.masonry.api class) 603 temprecipe = os.path.join(self.tempdir, 'recipe') 604 os.makedirs(temprecipe) 605 pn = 'iso8601' 606 pv = '2.1.0' 607 recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) 608 srcuri = 'https://files.pythonhosted.org/packages/b9/f3/ef59cee614d5e0accf6fd0cbba025b93b272e626ca89fb70a3e9187c5d15/%s-%s.tar.gz' % (pn, pv) 609 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 610 self.assertTrue(os.path.isfile(recipefile)) 611 checkvars = {} 612 checkvars['SUMMARY'] = 'Simple module to parse ISO 8601 dates' 613 checkvars['LICENSE'] = set(['MIT']) 614 checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=aab31f2ef7ba214a5a341eaa47a7f367' 615 checkvars['SRC_URI[sha256sum]'] = '6b1d3829ee8921c4301998c909f7829fa9ed3cbdac0d3b16af2d743aed1ba8df' 616 inherits = ['python_poetry_core', 'pypi'] 617 618 self._test_recipe_contents(recipefile, checkvars, inherits) 619 620 def test_recipetool_create_python3_pep517_flit_core_buildapi(self): 621 # This test require python 3.11 or above for the tomllib module or tomli module to be installed 622 needTomllib(self) 623 624 # Test creating python3 package from tarball (using flit_core.buildapi class) 625 temprecipe = os.path.join(self.tempdir, 'recipe') 626 os.makedirs(temprecipe) 627 pn = 'typing-extensions' 628 pv = '4.8.0' 629 recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) 630 srcuri = 'https://files.pythonhosted.org/packages/1f/7a/8b94bb016069caa12fc9f587b28080ac33b4fbb8ca369b98bc0a4828543e/typing_extensions-%s.tar.gz' % pv 631 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 632 self.assertTrue(os.path.isfile(recipefile)) 633 checkvars = {} 634 checkvars['SUMMARY'] = 'Backported and Experimental Type Hints for Python 3.8+' 635 checkvars['LICENSE'] = set(['PSF-2.0']) 636 checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=fcf6b249c2641540219a727f35d8d2c2' 637 checkvars['SRC_URI[sha256sum]'] = 'df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef' 638 inherits = ['python_flit_core', 'pypi'] 639 640 self._test_recipe_contents(recipefile, checkvars, inherits) 641 642 def test_recipetool_create_python3_pep517_hatchling(self): 643 # This test require python 3.11 or above for the tomllib module or tomli module to be installed 644 needTomllib(self) 645 646 # Test creating python3 package from tarball (using hatchling class) 647 temprecipe = os.path.join(self.tempdir, 'recipe') 648 os.makedirs(temprecipe) 649 pn = 'jsonschema' 650 pv = '4.19.1' 651 recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) 652 srcuri = 'https://files.pythonhosted.org/packages/e4/43/087b24516db11722c8687e0caf0f66c7785c0b1c51b0ab951dfde924e3f5/jsonschema-%s.tar.gz' % pv 653 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 654 self.assertTrue(os.path.isfile(recipefile)) 655 checkvars = {} 656 checkvars['SUMMARY'] = 'An implementation of JSON Schema validation for Python' 657 checkvars['HOMEPAGE'] = 'https://github.com/python-jsonschema/jsonschema' 658 checkvars['LICENSE'] = set(['MIT']) 659 checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7a60a81c146ec25599a3e1dabb8610a8 file://json/LICENSE;md5=9d4de43111d33570c8fe49b4cb0e01af' 660 checkvars['SRC_URI[sha256sum]'] = 'ec84cc37cfa703ef7cd4928db24f9cb31428a5d0fa77747b8b51a847458e0bbf' 661 inherits = ['python_hatchling', 'pypi'] 662 663 self._test_recipe_contents(recipefile, checkvars, inherits) 664 665 def test_recipetool_create_python3_pep517_maturin(self): 666 # This test require python 3.11 or above for the tomllib module or tomli module to be installed 667 needTomllib(self) 668 669 # Test creating python3 package from tarball (using maturin class) 670 temprecipe = os.path.join(self.tempdir, 'recipe') 671 os.makedirs(temprecipe) 672 pn = 'pydantic-core' 673 pv = '2.14.5' 674 recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) 675 srcuri = 'https://files.pythonhosted.org/packages/64/26/cffb93fe9c6b5a91c497f37fae14a4b073ecbc47fc36a9979c7aa888b245/pydantic_core-%s.tar.gz' % pv 676 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 677 self.assertTrue(os.path.isfile(recipefile)) 678 checkvars = {} 679 checkvars['HOMEPAGE'] = 'https://github.com/pydantic/pydantic-core' 680 checkvars['LICENSE'] = set(['MIT']) 681 checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=ab599c188b4a314d2856b3a55030c75c' 682 checkvars['SRC_URI[sha256sum]'] = '6d30226dfc816dd0fdf120cae611dd2215117e4f9b124af8c60ab9093b6e8e71' 683 inherits = ['python_maturin', 'pypi'] 684 685 self._test_recipe_contents(recipefile, checkvars, inherits) 686 687 def test_recipetool_create_python3_pep517_mesonpy(self): 688 # This test require python 3.11 or above for the tomllib module or tomli module to be installed 689 needTomllib(self) 690 691 # Test creating python3 package from tarball (using mesonpy class) 692 temprecipe = os.path.join(self.tempdir, 'recipe') 693 os.makedirs(temprecipe) 694 pn = 'siphash24' 695 pv = '1.4' 696 recipefile = os.path.join(temprecipe, 'python3-%s_%s.bb' % (pn, pv)) 697 srcuri = 'https://files.pythonhosted.org/packages/c2/32/b934a70592f314afcfa86c7f7e388804a8061be65b822e2aa07e573b6477/%s-%s.tar.gz' % (pn, pv) 698 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 699 self.assertTrue(os.path.isfile(recipefile)) 700 checkvars = {} 701 checkvars['SRC_URI[sha256sum]'] = '7fd65e39b2a7c8c4ddc3a168a687f4610751b0ac2ebb518783c0cdfc30bec4a0' 702 inherits = ['python_mesonpy', 'pypi'] 703 704 self._test_recipe_contents(recipefile, checkvars, inherits) 705 706 def test_recipetool_create_github_tarball(self): 707 # Basic test to ensure github URL mangling doesn't apply to release tarballs. 708 # Deliberately use an older release of Meson at present so we don't need a toml parser. 709 temprecipe = os.path.join(self.tempdir, 'recipe') 710 os.makedirs(temprecipe) 711 pv = '0.52.1' 712 recipefile = os.path.join(temprecipe, 'python3-meson_%s.bb' % pv) 713 srcuri = 'https://github.com/mesonbuild/meson/releases/download/%s/meson-%s.tar.gz' % (pv, pv) 714 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 715 self.assertTrue(os.path.isfile(recipefile)) 716 checkvars = {} 717 checkvars['LICENSE'] = set(['Apache-2.0']) 718 checkvars['SRC_URI'] = 'https://github.com/mesonbuild/meson/releases/download/${PV}/meson-${PV}.tar.gz' 719 inherits = ['setuptools3'] 720 self._test_recipe_contents(recipefile, checkvars, inherits) 721 722 def _test_recipetool_create_git(self, srcuri, branch=None): 723 # Basic test to check http git URL mangling works 724 temprecipe = os.path.join(self.tempdir, 'recipe') 725 os.makedirs(temprecipe) 726 name = srcuri.split(';')[0].split('/')[-1] 727 recipefile = os.path.join(temprecipe, name + '_git.bb') 728 options = ' -B %s' % branch if branch else '' 729 result = runCmd('recipetool create -o %s%s "%s"' % (temprecipe, options, srcuri)) 730 self.assertTrue(os.path.isfile(recipefile)) 731 checkvars = {} 732 checkvars['SRC_URI'] = srcuri 733 for scheme in ['http', 'https']: 734 if srcuri.startswith(scheme + ":"): 735 checkvars['SRC_URI'] = 'git%s;protocol=%s' % (srcuri[len(scheme):], scheme) 736 if ';branch=' not in srcuri: 737 checkvars['SRC_URI'] += ';branch=' + (branch or 'master') 738 self._test_recipe_contents(recipefile, checkvars, []) 739 740 def test_recipetool_create_git_http(self): 741 self._test_recipetool_create_git('http://git.yoctoproject.org/git/matchbox-keyboard') 742 743 def test_recipetool_create_git_srcuri_master(self): 744 self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;branch=master;protocol=https') 745 746 def test_recipetool_create_git_srcuri_branch(self): 747 self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;branch=matchbox-keyboard-0-1;protocol=https') 748 749 def test_recipetool_create_git_srcbranch(self): 750 self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;protocol=https', 'matchbox-keyboard-0-1') 751 752 def _go_urifiy(self, url, version, modulepath = None, pathmajor = None, subdir = None): 753 modulepath = ",path='%s'" % modulepath if len(modulepath) else '' 754 pathmajor = ",pathmajor='%s'" % pathmajor if len(pathmajor) else '' 755 subdir = ",subdir='%s'" % subdir if len(subdir) else '' 756 return "${@go_src_uri('%s','%s'%s%s%s)}" % (url, version, modulepath, pathmajor, subdir) 757 758 def test_recipetool_create_go(self): 759 # Basic test to check go recipe generation 760 temprecipe = os.path.join(self.tempdir, 'recipe') 761 os.makedirs(temprecipe) 762 763 recipefile = os.path.join(temprecipe, 'recipetool-go-test_git.bb') 764 deps_require_file = os.path.join(temprecipe, 'recipetool-go-test', 'recipetool-go-test-modules.inc') 765 lics_require_file = os.path.join(temprecipe, 'recipetool-go-test', 'recipetool-go-test-licenses.inc') 766 modules_txt_file = os.path.join(temprecipe, 'recipetool-go-test', 'modules.txt') 767 768 srcuri = 'https://git.yoctoproject.org/recipetool-go-test.git' 769 srcrev = "c3e213c01b6c1406b430df03ef0d1ae77de5d2f7" 770 srcbranch = "main" 771 772 result = runCmd('recipetool create -o %s %s -S %s -B %s' % (temprecipe, srcuri, srcrev, srcbranch)) 773 774 self.maxDiff = None 775 inherits = ['go-vendor'] 776 777 checkvars = {} 778 checkvars['GO_IMPORT'] = "git.yoctoproject.org/recipetool-go-test" 779 checkvars['SRC_URI'] = {'git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https', 780 'file://modules.txt'} 781 checkvars['LIC_FILES_CHKSUM'] = { 782 'file://src/${GO_IMPORT}/LICENSE;md5=4e3933dd47afbf115e484d11385fb3bd', 783 'file://src/${GO_IMPORT}/is/LICENSE;md5=62beaee5a116dd1e80161667b1df39ab' 784 } 785 786 self._test_recipe_contents(recipefile, checkvars, inherits) 787 self.assertNotIn('Traceback', result.output) 788 789 checkvars = {} 790 checkvars['VENDORED_LIC_FILES_CHKSUM'] = set( 791 ['file://src/${GO_IMPORT}/vendor/github.com/godbus/dbus/v5/LICENSE;md5=09042bd5c6c96a2b9e45ddf1bc517eed', 792 'file://src/${GO_IMPORT}/vendor/github.com/matryer/is/LICENSE;md5=62beaee5a116dd1e80161667b1df39ab']) 793 self.assertTrue(os.path.isfile(lics_require_file)) 794 self._test_recipe_contents(lics_require_file, checkvars, []) 795 796 # make sure that dependencies don't mention local directory ./matryer/is 797 dependencies = \ 798 [ ('github.com/godbus/dbus','v5.1.0', 'github.com/godbus/dbus/v5', '/v5', ''), 799 ] 800 801 src_uri = set() 802 for d in dependencies: 803 src_uri.add(self._go_urifiy(*d)) 804 805 checkvars = {} 806 checkvars['GO_DEPENDENCIES_SRC_URI'] = src_uri 807 808 self.assertTrue(os.path.isfile(deps_require_file)) 809 self._test_recipe_contents(deps_require_file, checkvars, []) 810 811class RecipetoolTests(RecipetoolBase): 812 813 @classmethod 814 def setUpClass(cls): 815 import sys 816 817 super(RecipetoolTests, cls).setUpClass() 818 bb_vars = get_bb_vars(['BBPATH']) 819 cls.bbpath = bb_vars['BBPATH'] 820 libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib', 'recipetool') 821 sys.path.insert(0, libpath) 822 823 def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths): 824 dstdir = basedstdir 825 self.assertTrue(os.path.exists(dstdir)) 826 for p in paths: 827 dstdir = os.path.join(dstdir, p) 828 if not os.path.exists(dstdir): 829 try: 830 os.makedirs(dstdir) 831 except PermissionError: 832 return False 833 except OSError as e: 834 if e.errno == errno.EROFS: 835 return False 836 else: 837 raise e 838 if p == "lib": 839 # Can race with other tests 840 self.add_command_to_tearDown('rmdir --ignore-fail-on-non-empty %s' % dstdir) 841 else: 842 self.track_for_cleanup(dstdir) 843 dstfile = os.path.join(dstdir, os.path.basename(srcfile)) 844 if srcfile != dstfile: 845 try: 846 shutil.copy(srcfile, dstfile) 847 except PermissionError: 848 return False 849 self.track_for_cleanup(dstfile) 850 return True 851 852 def test_recipetool_load_plugin(self): 853 """Test that recipetool loads only the first found plugin in BBPATH.""" 854 855 recipetool = runCmd("which recipetool") 856 fromname = runCmd("recipetool --quiet pluginfile") 857 srcfile = fromname.output 858 searchpath = self.bbpath.split(':') + [os.path.dirname(recipetool.output)] 859 plugincontent = [] 860 with open(srcfile) as fh: 861 plugincontent = fh.readlines() 862 try: 863 self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found') 864 searchpath = [ 865 path for path in searchpath 866 if self._copy_file_with_cleanup(srcfile, path, 'lib', 'recipetool') 867 ] 868 result = runCmd("recipetool --quiet count") 869 self.assertEqual(result.output, '1') 870 result = runCmd("recipetool --quiet multiloaded") 871 self.assertEqual(result.output, "no") 872 for path in searchpath: 873 result = runCmd("recipetool --quiet bbdir") 874 self.assertEqual(os.path.realpath(result.output), os.path.realpath(path)) 875 os.unlink(os.path.join(result.output, 'lib', 'recipetool', 'bbpath.py')) 876 finally: 877 with open(srcfile, 'w') as fh: 878 fh.writelines(plugincontent) 879 880 def test_recipetool_handle_license_vars(self): 881 from create import handle_license_vars 882 from unittest.mock import Mock 883 884 commonlicdir = get_bb_var('COMMON_LICENSE_DIR') 885 886 class DataConnectorCopy(bb.tinfoil.TinfoilDataStoreConnector): 887 pass 888 889 d = DataConnectorCopy 890 d.getVar = Mock(return_value=commonlicdir) 891 d.expand = Mock(side_effect=lambda x: x) 892 893 srctree = tempfile.mkdtemp(prefix='recipetoolqa') 894 self.track_for_cleanup(srctree) 895 896 # Multiple licenses 897 licenses = ['MIT', 'ISC', 'BSD-3-Clause', 'Apache-2.0'] 898 for licence in licenses: 899 shutil.copy(os.path.join(commonlicdir, licence), os.path.join(srctree, 'LICENSE.' + licence)) 900 # Duplicate license 901 shutil.copy(os.path.join(commonlicdir, 'MIT'), os.path.join(srctree, 'LICENSE')) 902 903 extravalues = { 904 # Duplicate and missing licenses 905 'LICENSE': 'Zlib & BSD-2-Clause & Zlib', 906 'LIC_FILES_CHKSUM': [ 907 'file://README.md;md5=0123456789abcdef0123456789abcd' 908 ] 909 } 910 lines_before = [] 911 handled = [] 912 licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d) 913 expected_lines_before = [ 914 '# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is', 915 '# your responsibility to verify that the values are complete and correct.', 916 '# NOTE: Original package / source metadata indicates license is: BSD-2-Clause & Zlib', 917 '#', 918 '# NOTE: multiple licenses have been detected; they have been separated with &', 919 '# in the LICENSE value for now since it is a reasonable assumption that all', 920 '# of the licenses apply. If instead there is a choice between the multiple', 921 '# licenses then you should change the value to separate the licenses with |', 922 '# instead of &. If there is any doubt, check the accompanying documentation', 923 '# to determine which situation is applicable.', 924 'LICENSE = "Apache-2.0 & BSD-2-Clause & BSD-3-Clause & ISC & MIT & Zlib"', 925 'LIC_FILES_CHKSUM = "file://LICENSE;md5=0835ade698e0bcf8506ecda2f7b4f302 \\\n' 926 ' file://LICENSE.Apache-2.0;md5=89aea4e17d99a7cacdbeed46a0096b10 \\\n' 927 ' file://LICENSE.BSD-3-Clause;md5=550794465ba0ec5312d6919e203a55f9 \\\n' 928 ' file://LICENSE.ISC;md5=f3b90e78ea0cffb20bf5cca7947a896d \\\n' 929 ' file://LICENSE.MIT;md5=0835ade698e0bcf8506ecda2f7b4f302 \\\n' 930 ' file://README.md;md5=0123456789abcdef0123456789abcd"', 931 '' 932 ] 933 self.assertEqual(lines_before, expected_lines_before) 934 expected_licvalues = [ 935 ('MIT', 'LICENSE', '0835ade698e0bcf8506ecda2f7b4f302'), 936 ('Apache-2.0', 'LICENSE.Apache-2.0', '89aea4e17d99a7cacdbeed46a0096b10'), 937 ('BSD-3-Clause', 'LICENSE.BSD-3-Clause', '550794465ba0ec5312d6919e203a55f9'), 938 ('ISC', 'LICENSE.ISC', 'f3b90e78ea0cffb20bf5cca7947a896d'), 939 ('MIT', 'LICENSE.MIT', '0835ade698e0bcf8506ecda2f7b4f302') 940 ] 941 self.assertEqual(handled, [('license', expected_licvalues)]) 942 self.assertEqual(extravalues, {}) 943 self.assertEqual(licvalues, expected_licvalues) 944 945 946 def test_recipetool_split_pkg_licenses(self): 947 from create import split_pkg_licenses 948 licvalues = [ 949 # Duplicate licenses 950 ('BSD-2-Clause', 'x/COPYING', None), 951 ('BSD-2-Clause', 'x/LICENSE', None), 952 # Multiple licenses 953 ('MIT', 'x/a/LICENSE.MIT', None), 954 ('ISC', 'x/a/LICENSE.ISC', None), 955 # Alternative licenses 956 ('(MIT | ISC)', 'x/b/LICENSE', None), 957 # Alternative licenses without brackets 958 ('MIT | BSD-2-Clause', 'x/c/LICENSE', None), 959 # Multi licenses with alternatives 960 ('MIT', 'x/d/COPYING', None), 961 ('MIT | BSD-2-Clause', 'x/d/LICENSE', None), 962 # Multi licenses with alternatives and brackets 963 ('Apache-2.0 & ((MIT | ISC) & BSD-3-Clause)', 'x/e/LICENSE', None) 964 ] 965 packages = { 966 '${PN}': '', 967 'a': 'x/a', 968 'b': 'x/b', 969 'c': 'x/c', 970 'd': 'x/d', 971 'e': 'x/e', 972 'f': 'x/f', 973 'g': 'x/g', 974 } 975 fallback_licenses = { 976 # Ignored 977 'a': 'BSD-3-Clause', 978 # Used 979 'f': 'BSD-3-Clause' 980 } 981 outlines = [] 982 outlicenses = split_pkg_licenses(licvalues, packages, outlines, fallback_licenses) 983 expected_outlicenses = { 984 '${PN}': ['BSD-2-Clause'], 985 'a': ['ISC', 'MIT'], 986 'b': ['(ISC | MIT)'], 987 'c': ['(BSD-2-Clause | MIT)'], 988 'd': ['(BSD-2-Clause | MIT)', 'MIT'], 989 'e': ['(ISC | MIT)', 'Apache-2.0', 'BSD-3-Clause'], 990 'f': ['BSD-3-Clause'], 991 'g': ['Unknown'] 992 } 993 self.assertEqual(outlicenses, expected_outlicenses) 994 expected_outlines = [ 995 'LICENSE:${PN} = "BSD-2-Clause"', 996 'LICENSE:a = "ISC & MIT"', 997 'LICENSE:b = "(ISC | MIT)"', 998 'LICENSE:c = "(BSD-2-Clause | MIT)"', 999 'LICENSE:d = "(BSD-2-Clause | MIT) & MIT"', 1000 'LICENSE:e = "(ISC | MIT) & Apache-2.0 & BSD-3-Clause"', 1001 'LICENSE:f = "BSD-3-Clause"', 1002 'LICENSE:g = "Unknown"' 1003 ] 1004 self.assertEqual(outlines, expected_outlines) 1005 1006 1007class RecipetoolAppendsrcBase(RecipetoolBase): 1008 def _try_recipetool_appendsrcfile(self, testrecipe, newfile, destfile, options, expectedlines, expectedfiles): 1009 cmd = 'recipetool appendsrcfile %s %s %s %s %s' % (options, self.templayerdir, testrecipe, newfile, destfile) 1010 return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) 1011 1012 def _try_recipetool_appendsrcfiles(self, testrecipe, newfiles, expectedlines=None, expectedfiles=None, destdir=None, options=''): 1013 1014 if destdir: 1015 options += ' -D %s' % destdir 1016 1017 if expectedfiles is None: 1018 expectedfiles = [os.path.basename(f) for f in newfiles] 1019 1020 cmd = 'recipetool appendsrcfiles %s %s %s %s' % (options, self.templayerdir, testrecipe, ' '.join(newfiles)) 1021 return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) 1022 1023 def _try_recipetool_appendsrcfile_fail(self, testrecipe, newfile, destfile, checkerror): 1024 cmd = 'recipetool appendsrcfile %s %s %s %s' % (self.templayerdir, testrecipe, newfile, destfile or '') 1025 result = runCmd(cmd, ignore_status=True) 1026 self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd) 1027 self.assertNotIn('Traceback', result.output) 1028 for errorstr in checkerror: 1029 self.assertIn(errorstr, result.output) 1030 1031 @staticmethod 1032 def _get_first_file_uri(recipe): 1033 '''Return the first file:// in SRC_URI for the specified recipe.''' 1034 src_uri = get_bb_var('SRC_URI', recipe).split() 1035 for uri in src_uri: 1036 p = urllib.parse.urlparse(uri) 1037 if p.scheme == 'file': 1038 return p.netloc + p.path, uri 1039 1040 def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, remove=None, machine=None , options=''): 1041 if newfile is None: 1042 newfile = self.testfile 1043 1044 if srcdir: 1045 if destdir: 1046 expected_subdir = os.path.join(srcdir, destdir) 1047 else: 1048 expected_subdir = srcdir 1049 else: 1050 options += " -W" 1051 expected_subdir = destdir 1052 1053 if filename: 1054 if destdir: 1055 destpath = os.path.join(destdir, filename) 1056 else: 1057 destpath = filename 1058 else: 1059 filename = os.path.basename(newfile) 1060 if destdir: 1061 destpath = destdir + os.sep 1062 else: 1063 destpath = '.' + os.sep 1064 1065 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 1066 '\n'] 1067 1068 override = "" 1069 if machine: 1070 options += ' -m %s' % machine 1071 override = ':append:%s' % machine 1072 expectedlines.extend(['PACKAGE_ARCH = "${MACHINE_ARCH}"\n', 1073 '\n']) 1074 1075 if remove: 1076 for entry in remove: 1077 if machine: 1078 entry_remove_line = 'SRC_URI:remove:%s = " %s"\n' % (machine, entry) 1079 else: 1080 entry_remove_line = 'SRC_URI:remove = "%s"\n' % entry 1081 1082 expectedlines.extend([entry_remove_line, 1083 '\n']) 1084 1085 if has_src_uri: 1086 uri = 'file://%s' % filename 1087 if expected_subdir: 1088 uri += ';subdir=%s' % expected_subdir 1089 if machine: 1090 src_uri_line = 'SRC_URI%s = " %s"\n' % (override, uri) 1091 else: 1092 src_uri_line = 'SRC_URI += "%s"\n' % uri 1093 1094 expectedlines.extend([src_uri_line, '\n']) 1095 1096 with open("/tmp/tmp.txt", "w") as file: 1097 print(expectedlines, file=file) 1098 1099 if machine: 1100 filename = '%s/%s' % (machine, filename) 1101 1102 return self._try_recipetool_appendsrcfile(testrecipe, newfile, destpath, options, expectedlines, [filename]) 1103 1104 def _test_appendsrcfiles(self, testrecipe, newfiles, expectedfiles=None, destdir=None, options=''): 1105 if expectedfiles is None: 1106 expectedfiles = [os.path.basename(n) for n in newfiles] 1107 1108 self._try_recipetool_appendsrcfiles(testrecipe, newfiles, expectedfiles=expectedfiles, destdir=destdir, options=options) 1109 1110 bb_vars = get_bb_vars(['SRC_URI', 'FILE', 'FILESEXTRAPATHS'], testrecipe) 1111 src_uri = bb_vars['SRC_URI'].split() 1112 for f in expectedfiles: 1113 if destdir: 1114 self.assertIn('file://%s;subdir=%s' % (f, destdir), src_uri) 1115 else: 1116 self.assertIn('file://%s' % f, src_uri) 1117 1118 recipefile = bb_vars['FILE'] 1119 bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) 1120 filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe) 1121 filesextrapaths = bb_vars['FILESEXTRAPATHS'].split(':') 1122 self.assertIn(filesdir, filesextrapaths) 1123 1124 1125 1126 1127class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): 1128 1129 def test_recipetool_appendsrcfile_basic(self): 1130 self._test_appendsrcfile('base-files', 'a-file') 1131 1132 def test_recipetool_appendsrcfile_basic_wildcard(self): 1133 testrecipe = 'base-files' 1134 self._test_appendsrcfile(testrecipe, 'a-file', options='-w') 1135 recipefile = get_bb_var('FILE', testrecipe) 1136 bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) 1137 self.assertEqual(os.path.basename(bbappendfile), '%s_%%.bbappend' % testrecipe) 1138 1139 def test_recipetool_appendsrcfile_subdir_basic(self): 1140 self._test_appendsrcfile('base-files', 'a-file', 'tmp') 1141 1142 def test_recipetool_appendsrcfile_subdir_basic_dirdest(self): 1143 self._test_appendsrcfile('base-files', destdir='tmp') 1144 1145 def test_recipetool_appendsrcfile_srcdir_basic(self): 1146 testrecipe = 'bash' 1147 bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe) 1148 srcdir = bb_vars['S'] 1149 workdir = bb_vars['WORKDIR'] 1150 subdir = os.path.relpath(srcdir, workdir) 1151 self._test_appendsrcfile(testrecipe, 'a-file', srcdir=subdir) 1152 1153 def test_recipetool_appendsrcfile_existing_in_src_uri(self): 1154 testrecipe = 'base-files' 1155 filepath,_ = self._get_first_file_uri(testrecipe) 1156 self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) 1157 self._test_appendsrcfile(testrecipe, filepath, has_src_uri=False) 1158 1159 def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self, machine=None): 1160 testrecipe = 'base-files' 1161 subdir = 'tmp' 1162 filepath, srcuri_entry = self._get_first_file_uri(testrecipe) 1163 self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) 1164 1165 self._test_appendsrcfile(testrecipe, filepath, subdir, machine=machine, remove=[srcuri_entry]) 1166 1167 def test_recipetool_appendsrcfile_machine(self): 1168 # A very basic test 1169 self._test_appendsrcfile('base-files', 'a-file', machine='mymachine') 1170 1171 # Force cleaning the output of previous test 1172 self.tearDownLocal() 1173 1174 # A more complex test: existing entry in src_uri with different param 1175 self.test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(machine='mymachine') 1176 1177 def test_recipetool_appendsrcfile_update_recipe_basic(self): 1178 testrecipe = "mtd-utils-selftest" 1179 recipefile = get_bb_var('FILE', testrecipe) 1180 self.assertIn('meta-selftest', recipefile, 'This test expect %s recipe to be in meta-selftest') 1181 cmd = 'recipetool appendsrcfile -W -u meta-selftest %s %s' % (testrecipe, self.testfile) 1182 result = runCmd(cmd) 1183 self.assertNotIn('Traceback', result.output) 1184 self.add_command_to_tearDown('cd %s; rm -f %s/%s; git checkout .' % (os.path.dirname(recipefile), testrecipe, os.path.basename(self.testfile))) 1185 1186 expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), 1187 ('??', '.*/%s/%s$' % (testrecipe, os.path.basename(self.testfile)))] 1188 self._check_repo_status(os.path.dirname(recipefile), expected_status) 1189 result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile)) 1190 removelines = [] 1191 addlines = [ 1192 'file://%s \\\\' % os.path.basename(self.testfile), 1193 ] 1194 self._check_diff(result.output, addlines, removelines) 1195 1196 def test_recipetool_appendsrcfile_replace_file_srcdir(self): 1197 testrecipe = 'bash' 1198 filepath = 'Makefile.in' 1199 bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe) 1200 srcdir = bb_vars['S'] 1201 workdir = bb_vars['WORKDIR'] 1202 subdir = os.path.relpath(srcdir, workdir) 1203 1204 self._test_appendsrcfile(testrecipe, filepath, srcdir=subdir) 1205 bitbake('%s:do_unpack' % testrecipe) 1206 with open(self.testfile, 'r') as testfile: 1207 with open(os.path.join(srcdir, filepath), 'r') as makefilein: 1208 self.assertEqual(testfile.read(), makefilein.read()) 1209 1210 def test_recipetool_appendsrcfiles_basic(self, destdir=None): 1211 newfiles = [self.testfile] 1212 for i in range(1, 5): 1213 testfile = os.path.join(self.tempdir, 'testfile%d' % i) 1214 with open(testfile, 'w') as f: 1215 f.write('Test file %d\n' % i) 1216 newfiles.append(testfile) 1217 self._test_appendsrcfiles('gcc', newfiles, destdir=destdir, options='-W') 1218 1219 def test_recipetool_appendsrcfiles_basic_subdir(self): 1220 self.test_recipetool_appendsrcfiles_basic(destdir='testdir') 1221