1# 2# SPDX-License-Identifier: MIT 3# 4 5import os 6import shutil 7import glob 8import subprocess 9import tempfile 10 11from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer 12from oeqa.selftest.cases.sstate import SStateBase 13import oe 14 15import bb.siggen 16 17class SStateTests(SStateBase): 18 def test_autorev_sstate_works(self): 19 # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV} 20 # when PV does not contain SRCPV 21 22 tempdir = tempfile.mkdtemp(prefix='sstate_autorev') 23 tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir') 24 self.track_for_cleanup(tempdir) 25 self.track_for_cleanup(tempdldir) 26 create_temp_layer(tempdir, 'selftestrecipetool') 27 self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir) 28 self.append_config("DL_DIR = \"%s\"" % tempdldir) 29 runCmd('bitbake-layers add-layer %s' % tempdir) 30 31 # Use dbus-wait as a local git repo we can add a commit between two builds in 32 pn = 'dbus-wait' 33 srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517' 34 url = 'git://git.yoctoproject.org/dbus-wait' 35 result = runCmd('git clone %s noname' % url, cwd=tempdir) 36 srcdir = os.path.join(tempdir, 'noname') 37 result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir) 38 self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') 39 40 recipefile = os.path.join(tempdir, "recipes-test", "dbus-wait-test", 'dbus-wait-test_git.bb') 41 os.makedirs(os.path.dirname(recipefile)) 42 srcuri = 'git://' + srcdir + ';protocol=file;branch=master' 43 result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri]) 44 self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) 45 46 with open(recipefile, 'a') as f: 47 f.write('SRCREV = "${AUTOREV}"\n') 48 f.write('PV = "1.0"\n') 49 50 bitbake("dbus-wait-test -c fetch") 51 with open(os.path.join(srcdir, "bar.txt"), "w") as f: 52 f.write("foo") 53 result = runCmd('git add bar.txt; git commit -asm "add bar"', cwd=srcdir) 54 bitbake("dbus-wait-test -c unpack") 55 56 57 # Test sstate files creation and their location 58 def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True): 59 self.config_sstate(temp_sstate_location, [self.sstate_path]) 60 61 if self.temp_sstate_location: 62 bitbake(['-cclean'] + targets) 63 else: 64 bitbake(['-ccleansstate'] + targets) 65 66 bitbake(targets) 67 file_tracker = [] 68 results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) 69 if distro_nonspecific: 70 for r in results: 71 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo", "_fetch.tar.zst.siginfo", "_unpack.tar.zst.siginfo", "_patch.tar.zst.siginfo")): 72 continue 73 file_tracker.append(r) 74 else: 75 file_tracker = results 76 77 if should_pass: 78 self.assertTrue(file_tracker , msg="Could not find sstate files for: %s" % ', '.join(map(str, targets))) 79 else: 80 self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker))) 81 82 def test_sstate_creation_distro_specific_pass(self): 83 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) 84 85 def test_sstate_creation_distro_specific_fail(self): 86 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) 87 88 def test_sstate_creation_distro_nonspecific_pass(self): 89 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) 90 91 def test_sstate_creation_distro_nonspecific_fail(self): 92 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) 93 94 # Test the sstate files deletion part of the do_cleansstate task 95 def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True): 96 self.config_sstate(temp_sstate_location, [self.sstate_path]) 97 98 bitbake(['-ccleansstate'] + targets) 99 100 bitbake(targets) 101 archives_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific) 102 self.assertTrue(archives_created, msg="Could not find sstate .tar.zst files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_created))) 103 104 siginfo_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific) 105 self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created))) 106 107 bitbake(['-ccleansstate'] + targets) 108 archives_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific) 109 self.assertTrue(not archives_removed, msg="do_cleansstate didn't remove .tar.zst sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_removed))) 110 111 def test_cleansstate_task_distro_specific_nonspecific(self): 112 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] 113 targets.append('linux-libc-headers') 114 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) 115 116 def test_cleansstate_task_distro_nonspecific(self): 117 self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) 118 119 def test_cleansstate_task_distro_specific(self): 120 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] 121 targets.append('linux-libc-headers') 122 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) 123 124 125 # Test rebuilding of distro-specific sstate files 126 def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True): 127 self.config_sstate(temp_sstate_location, [self.sstate_path]) 128 129 bitbake(['-ccleansstate'] + targets) 130 131 bitbake(targets) 132 results = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=False, distro_nonspecific=True) 133 filtered_results = [] 134 for r in results: 135 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo")): 136 continue 137 filtered_results.append(r) 138 self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results))) 139 file_tracker_1 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False) 140 self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets))) 141 142 self.track_for_cleanup(self.distro_specific_sstate + "_old") 143 shutil.copytree(self.distro_specific_sstate, self.distro_specific_sstate + "_old") 144 shutil.rmtree(self.distro_specific_sstate) 145 146 bitbake(['-cclean'] + targets) 147 bitbake(targets) 148 file_tracker_2 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False) 149 self.assertTrue(len(file_tracker_2) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets))) 150 151 not_recreated = [x for x in file_tracker_1 if x not in file_tracker_2] 152 self.assertTrue(not_recreated == [], msg="The following sstate files were not recreated: %s" % ', '.join(map(str, not_recreated))) 153 154 created_once = [x for x in file_tracker_2 if x not in file_tracker_1] 155 self.assertTrue(created_once == [], msg="The following sstate files were created only in the second run: %s" % ', '.join(map(str, created_once))) 156 157 def test_rebuild_distro_specific_sstate_cross_native_targets(self): 158 self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True) 159 160 def test_rebuild_distro_specific_sstate_cross_target(self): 161 self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch], temp_sstate_location=True) 162 163 def test_rebuild_distro_specific_sstate_native_target(self): 164 self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True) 165 166 167 # Test the sstate-cache-management script. Each element in the global_config list is used with the corresponding element in the target_config list 168 # global_config elements are expected to not generate any sstate files that would be removed by sstate-cache-management.sh (such as changing the value of MACHINE) 169 def run_test_sstate_cache_management_script(self, target, global_config=[''], target_config=[''], ignore_patterns=[]): 170 self.assertTrue(global_config) 171 self.assertTrue(target_config) 172 self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements') 173 self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path]) 174 175 # If buildhistory is enabled, we need to disable version-going-backwards 176 # QA checks for this test. It may report errors otherwise. 177 self.append_config('ERROR_QA:remove = "version-going-backwards"') 178 179 # For now this only checks if random sstate tasks are handled correctly as a group. 180 # In the future we should add control over what tasks we check for. 181 182 sstate_archs_list = [] 183 expected_remaining_sstate = [] 184 for idx in range(len(target_config)): 185 self.append_config(global_config[idx]) 186 self.append_recipeinc(target, target_config[idx]) 187 sstate_arch = get_bb_var('SSTATE_PKGARCH', target) 188 if not sstate_arch in sstate_archs_list: 189 sstate_archs_list.append(sstate_arch) 190 if target_config[idx] == target_config[-1]: 191 target_sstate_before_build = self.search_sstate(target + r'.*?\.tar.zst$') 192 bitbake("-cclean %s" % target) 193 result = bitbake(target, ignore_status=True) 194 if target_config[idx] == target_config[-1]: 195 target_sstate_after_build = self.search_sstate(target + r'.*?\.tar.zst$') 196 expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)] 197 self.remove_config(global_config[idx]) 198 self.remove_recipeinc(target, target_config[idx]) 199 self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output)) 200 201 runCmd("sstate-cache-management.sh -y --cache-dir=%s --remove-duplicated --extra-archs=%s" % (self.sstate_path, ','.join(map(str, sstate_archs_list)))) 202 actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tar.zst$') if not any(pattern in x for pattern in ignore_patterns)] 203 204 actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate] 205 self.assertFalse(actual_not_expected, msg="Files should have been removed but were not: %s" % ', '.join(map(str, actual_not_expected))) 206 expected_not_actual = [x for x in expected_remaining_sstate if x not in actual_remaining_sstate] 207 self.assertFalse(expected_not_actual, msg="Extra files were removed: %s" ', '.join(map(str, expected_not_actual))) 208 209 def test_sstate_cache_management_script_using_pr_1(self): 210 global_config = [] 211 target_config = [] 212 global_config.append('') 213 target_config.append('PR = "0"') 214 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 215 216 def test_sstate_cache_management_script_using_pr_2(self): 217 global_config = [] 218 target_config = [] 219 global_config.append('') 220 target_config.append('PR = "0"') 221 global_config.append('') 222 target_config.append('PR = "1"') 223 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 224 225 def test_sstate_cache_management_script_using_pr_3(self): 226 global_config = [] 227 target_config = [] 228 global_config.append('MACHINE = "qemux86-64"') 229 target_config.append('PR = "0"') 230 global_config.append(global_config[0]) 231 target_config.append('PR = "1"') 232 global_config.append('MACHINE = "qemux86"') 233 target_config.append('PR = "1"') 234 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 235 236 def test_sstate_cache_management_script_using_machine(self): 237 global_config = [] 238 target_config = [] 239 global_config.append('MACHINE = "qemux86-64"') 240 target_config.append('') 241 global_config.append('MACHINE = "qemux86"') 242 target_config.append('') 243 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 244 245 def test_sstate_32_64_same_hash(self): 246 """ 247 The sstate checksums for both native and target should not vary whether 248 they're built on a 32 or 64 bit system. Rather than requiring two different 249 build machines and running a builds, override the variables calling uname() 250 manually and check using bitbake -S. 251 """ 252 253 self.write_config(""" 254MACHINE = "qemux86" 255TMPDIR = "${TOPDIR}/tmp-sstatesamehash" 256TCLIBCAPPEND = "" 257BUILD_ARCH = "x86_64" 258BUILD_OS = "linux" 259SDKMACHINE = "x86_64" 260PACKAGE_CLASSES = "package_rpm package_ipk package_deb" 261BB_SIGNATURE_HANDLER = "OEBasicHash" 262""") 263 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 264 bitbake("core-image-weston -S none") 265 self.write_config(""" 266MACHINE = "qemux86" 267TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" 268TCLIBCAPPEND = "" 269BUILD_ARCH = "i686" 270BUILD_OS = "linux" 271SDKMACHINE = "i686" 272PACKAGE_CLASSES = "package_rpm package_ipk package_deb" 273BB_SIGNATURE_HANDLER = "OEBasicHash" 274""") 275 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 276 bitbake("core-image-weston -S none") 277 278 def get_files(d): 279 f = [] 280 for root, dirs, files in os.walk(d): 281 if "core-image-weston" in root: 282 # SDKMACHINE changing will change 283 # do_rootfs/do_testimage/do_build stamps of images which 284 # is safe to ignore. 285 continue 286 f.extend(os.path.join(root, name) for name in files) 287 return f 288 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") 289 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") 290 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash").replace("i686-linux", "x86_64-linux").replace("i686" + self.target_vendor + "-linux", "x86_64" + self.target_vendor + "-linux", ) for x in files2] 291 self.maxDiff = None 292 self.assertCountEqual(files1, files2) 293 294 295 def test_sstate_nativelsbstring_same_hash(self): 296 """ 297 The sstate checksums should be independent of whichever NATIVELSBSTRING is 298 detected. Rather than requiring two different build machines and running 299 builds, override the variables manually and check using bitbake -S. 300 """ 301 302 self.write_config(""" 303TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 304TCLIBCAPPEND = \"\" 305NATIVELSBSTRING = \"DistroA\" 306BB_SIGNATURE_HANDLER = "OEBasicHash" 307""") 308 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 309 bitbake("core-image-weston -S none") 310 self.write_config(""" 311TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 312TCLIBCAPPEND = \"\" 313NATIVELSBSTRING = \"DistroB\" 314BB_SIGNATURE_HANDLER = "OEBasicHash" 315""") 316 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 317 bitbake("core-image-weston -S none") 318 319 def get_files(d): 320 f = [] 321 for root, dirs, files in os.walk(d): 322 f.extend(os.path.join(root, name) for name in files) 323 return f 324 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") 325 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") 326 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 327 self.maxDiff = None 328 self.assertCountEqual(files1, files2) 329 330 def test_sstate_allarch_samesigs(self): 331 """ 332 The sstate checksums of allarch packages should be independent of whichever 333 MACHINE is set. Check this using bitbake -S. 334 Also, rather than duplicate the test, check nativesdk stamps are the same between 335 the two MACHINE values. 336 """ 337 338 configA = """ 339TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 340TCLIBCAPPEND = \"\" 341MACHINE = \"qemux86-64\" 342BB_SIGNATURE_HANDLER = "OEBasicHash" 343""" 344 #OLDEST_KERNEL is arch specific so set to a different value here for testing 345 configB = """ 346TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 347TCLIBCAPPEND = \"\" 348MACHINE = \"qemuarm\" 349OLDEST_KERNEL = \"3.3.0\" 350BB_SIGNATURE_HANDLER = "OEBasicHash" 351""" 352 self.sstate_common_samesigs(configA, configB, allarch=True) 353 354 def test_sstate_nativesdk_samesigs_multilib(self): 355 """ 356 check nativesdk stamps are the same between the two MACHINE values. 357 """ 358 359 configA = """ 360TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 361TCLIBCAPPEND = \"\" 362MACHINE = \"qemux86-64\" 363require conf/multilib.conf 364MULTILIBS = \"multilib:lib32\" 365DEFAULTTUNE:virtclass-multilib-lib32 = \"x86\" 366BB_SIGNATURE_HANDLER = "OEBasicHash" 367""" 368 configB = """ 369TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 370TCLIBCAPPEND = \"\" 371MACHINE = \"qemuarm\" 372require conf/multilib.conf 373MULTILIBS = \"\" 374BB_SIGNATURE_HANDLER = "OEBasicHash" 375""" 376 self.sstate_common_samesigs(configA, configB) 377 378 def sstate_common_samesigs(self, configA, configB, allarch=False): 379 380 self.write_config(configA) 381 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 382 bitbake("world meta-toolchain -S none") 383 self.write_config(configB) 384 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 385 bitbake("world meta-toolchain -S none") 386 387 def get_files(d): 388 f = {} 389 for root, dirs, files in os.walk(d): 390 for name in files: 391 if "meta-environment" in root or "cross-canadian" in root: 392 continue 393 if "do_build" not in name: 394 # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79 395 (_, task, _, shash) = name.rsplit(".", 3) 396 f[os.path.join(os.path.basename(root), task)] = shash 397 return f 398 399 nativesdkdir = os.path.basename(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")[0]) 400 401 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir) 402 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir) 403 self.maxDiff = None 404 self.assertEqual(files1, files2) 405 406 if allarch: 407 allarchdir = os.path.basename(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/all-*-linux")[0]) 408 409 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + allarchdir) 410 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + allarchdir) 411 self.assertEqual(files1, files2) 412 413 def test_sstate_sametune_samesigs(self): 414 """ 415 The sstate checksums of two identical machines (using the same tune) should be the 416 same, apart from changes within the machine specific stamps directory. We use the 417 qemux86copy machine to test this. Also include multilibs in the test. 418 """ 419 420 self.write_config(""" 421TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 422TCLIBCAPPEND = \"\" 423MACHINE = \"qemux86\" 424require conf/multilib.conf 425MULTILIBS = "multilib:lib32" 426DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 427BB_SIGNATURE_HANDLER = "OEBasicHash" 428""") 429 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 430 bitbake("world meta-toolchain -S none") 431 self.write_config(""" 432TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 433TCLIBCAPPEND = \"\" 434MACHINE = \"qemux86copy\" 435require conf/multilib.conf 436MULTILIBS = "multilib:lib32" 437DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 438BB_SIGNATURE_HANDLER = "OEBasicHash" 439""") 440 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 441 bitbake("world meta-toolchain -S none") 442 443 def get_files(d): 444 f = [] 445 for root, dirs, files in os.walk(d): 446 for name in files: 447 if "meta-environment" in root or "cross-canadian" in root: 448 continue 449 if "qemux86copy-" in root or "qemux86-" in root: 450 continue 451 if "do_build" not in name and "do_populate_sdk" not in name: 452 f.append(os.path.join(root, name)) 453 return f 454 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") 455 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") 456 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 457 self.maxDiff = None 458 self.assertCountEqual(files1, files2) 459 460 461 def test_sstate_multilib_or_not_native_samesigs(self): 462 """The sstate checksums of two native recipes (and their dependencies) 463 where the target is using multilib in one but not the other 464 should be the same. We use the qemux86copy machine to test 465 this. 466 """ 467 468 self.write_config(""" 469TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 470TCLIBCAPPEND = \"\" 471MACHINE = \"qemux86\" 472require conf/multilib.conf 473MULTILIBS = "multilib:lib32" 474DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 475BB_SIGNATURE_HANDLER = "OEBasicHash" 476""") 477 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 478 bitbake("binutils-native -S none") 479 self.write_config(""" 480TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 481TCLIBCAPPEND = \"\" 482MACHINE = \"qemux86copy\" 483BB_SIGNATURE_HANDLER = "OEBasicHash" 484""") 485 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 486 bitbake("binutils-native -S none") 487 488 def get_files(d): 489 f = [] 490 for root, dirs, files in os.walk(d): 491 for name in files: 492 f.append(os.path.join(root, name)) 493 return f 494 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") 495 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") 496 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 497 self.maxDiff = None 498 self.assertCountEqual(files1, files2) 499 500 501 def test_sstate_noop_samesigs(self): 502 """ 503 The sstate checksums of two builds with these variables changed or 504 classes inherits should be the same. 505 """ 506 507 self.write_config(""" 508TMPDIR = "${TOPDIR}/tmp-sstatesamehash" 509TCLIBCAPPEND = "" 510BB_NUMBER_THREADS = "${@oe.utils.cpu_count()}" 511PARALLEL_MAKE = "-j 1" 512DL_DIR = "${TOPDIR}/download1" 513TIME = "111111" 514DATE = "20161111" 515INHERIT:remove = "buildstats-summary buildhistory uninative" 516http_proxy = "" 517BB_SIGNATURE_HANDLER = "OEBasicHash" 518""") 519 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 520 self.track_for_cleanup(self.topdir + "/download1") 521 bitbake("world meta-toolchain -S none") 522 self.write_config(""" 523TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" 524TCLIBCAPPEND = "" 525BB_NUMBER_THREADS = "${@oe.utils.cpu_count()+1}" 526PARALLEL_MAKE = "-j 2" 527DL_DIR = "${TOPDIR}/download2" 528TIME = "222222" 529DATE = "20161212" 530# Always remove uninative as we're changing proxies 531INHERIT:remove = "uninative" 532INHERIT += "buildstats-summary buildhistory" 533http_proxy = "http://example.com/" 534BB_SIGNATURE_HANDLER = "OEBasicHash" 535""") 536 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 537 self.track_for_cleanup(self.topdir + "/download2") 538 bitbake("world meta-toolchain -S none") 539 540 def get_files(d): 541 f = {} 542 for root, dirs, files in os.walk(d): 543 for name in files: 544 name, shash = name.rsplit('.', 1) 545 # Extract just the machine and recipe name 546 base = os.sep.join(root.rsplit(os.sep, 2)[-2:] + [name]) 547 f[base] = shash 548 return f 549 550 def compare_sigfiles(files, files1, files2, compare=False): 551 for k in files: 552 if k in files1 and k in files2: 553 print("%s differs:" % k) 554 if compare: 555 sigdatafile1 = self.topdir + "/tmp-sstatesamehash/stamps/" + k + "." + files1[k] 556 sigdatafile2 = self.topdir + "/tmp-sstatesamehash2/stamps/" + k + "." + files2[k] 557 output = bb.siggen.compare_sigfiles(sigdatafile1, sigdatafile2) 558 if output: 559 print('\n'.join(output)) 560 elif k in files1 and k not in files2: 561 print("%s in files1" % k) 562 elif k not in files1 and k in files2: 563 print("%s in files2" % k) 564 else: 565 assert "shouldn't reach here" 566 567 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") 568 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") 569 # Remove items that are identical in both sets 570 for k,v in files1.items() & files2.items(): 571 del files1[k] 572 del files2[k] 573 if not files1 and not files2: 574 # No changes, so we're done 575 return 576 577 files = list(files1.keys() | files2.keys()) 578 # this is an expensive computation, thus just compare the first 'max_sigfiles_to_compare' k files 579 max_sigfiles_to_compare = 20 580 first, rest = files[:max_sigfiles_to_compare], files[max_sigfiles_to_compare:] 581 compare_sigfiles(first, files1, files2, compare=True) 582 compare_sigfiles(rest, files1, files2, compare=False) 583 584 self.fail("sstate hashes not identical.") 585 586 def test_sstate_movelayer_samesigs(self): 587 """ 588 The sstate checksums of two builds with the same oe-core layer in two 589 different locations should be the same. 590 """ 591 core_layer = os.path.join( 592 self.tc.td["COREBASE"], 'meta') 593 copy_layer_1 = self.topdir + "/meta-copy1/meta" 594 copy_layer_2 = self.topdir + "/meta-copy2/meta" 595 596 oe.path.copytree(core_layer, copy_layer_1) 597 self.write_config(""" 598TMPDIR = "${TOPDIR}/tmp-sstatesamehash" 599""") 600 bblayers_conf = 'BBLAYERS += "%s"\nBBLAYERS:remove = "%s"' % (copy_layer_1, core_layer) 601 self.write_bblayers_config(bblayers_conf) 602 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 603 bitbake("bash -S none") 604 605 oe.path.copytree(core_layer, copy_layer_2) 606 self.write_config(""" 607TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" 608""") 609 bblayers_conf = 'BBLAYERS += "%s"\nBBLAYERS:remove = "%s"' % (copy_layer_2, core_layer) 610 self.write_bblayers_config(bblayers_conf) 611 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 612 bitbake("bash -S none") 613 614 def get_files(d): 615 f = [] 616 for root, dirs, files in os.walk(d): 617 for name in files: 618 f.append(os.path.join(root, name)) 619 return f 620 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") 621 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") 622 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 623 self.maxDiff = None 624 self.assertCountEqual(files1, files2) 625 626