1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: MIT 5# 6 7import os 8import shutil 9import glob 10import subprocess 11import tempfile 12import datetime 13import re 14 15from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, get_bb_vars 16from oeqa.selftest.case import OESelftestTestCase 17from oeqa.core.decorator import OETestTag 18 19import oe 20import bb.siggen 21 22# Set to True to preserve stamp files after test execution for debugging failures 23keep_temp_files = False 24 25class SStateBase(OESelftestTestCase): 26 27 def setUpLocal(self): 28 super(SStateBase, self).setUpLocal() 29 self.temp_sstate_location = None 30 needed_vars = ['SSTATE_DIR', 'NATIVELSBSTRING', 'TCLIBC', 'TUNE_ARCH', 31 'TOPDIR', 'TARGET_VENDOR', 'TARGET_OS'] 32 bb_vars = get_bb_vars(needed_vars) 33 self.sstate_path = bb_vars['SSTATE_DIR'] 34 self.hostdistro = bb_vars['NATIVELSBSTRING'] 35 self.tclibc = bb_vars['TCLIBC'] 36 self.tune_arch = bb_vars['TUNE_ARCH'] 37 self.topdir = bb_vars['TOPDIR'] 38 self.target_vendor = bb_vars['TARGET_VENDOR'] 39 self.target_os = bb_vars['TARGET_OS'] 40 self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) 41 42 def track_for_cleanup(self, path): 43 if not keep_temp_files: 44 super().track_for_cleanup(path) 45 46 # Creates a special sstate configuration with the option to add sstate mirrors 47 def config_sstate(self, temp_sstate_location=False, add_local_mirrors=[]): 48 self.temp_sstate_location = temp_sstate_location 49 50 if self.temp_sstate_location: 51 temp_sstate_path = os.path.join(self.builddir, "temp_sstate_%s" % datetime.datetime.now().strftime('%Y%m%d%H%M%S')) 52 config_temp_sstate = "SSTATE_DIR = \"%s\"" % temp_sstate_path 53 self.append_config(config_temp_sstate) 54 self.track_for_cleanup(temp_sstate_path) 55 bb_vars = get_bb_vars(['SSTATE_DIR', 'NATIVELSBSTRING']) 56 self.sstate_path = bb_vars['SSTATE_DIR'] 57 self.hostdistro = bb_vars['NATIVELSBSTRING'] 58 self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) 59 60 if add_local_mirrors: 61 config_set_sstate_if_not_set = 'SSTATE_MIRRORS ?= ""' 62 self.append_config(config_set_sstate_if_not_set) 63 for local_mirror in add_local_mirrors: 64 self.assertFalse(os.path.join(local_mirror) == os.path.join(self.sstate_path), msg='Cannot add the current sstate path as a sstate mirror') 65 config_sstate_mirror = "SSTATE_MIRRORS += \"file://.* file:///%s/PATH\"" % local_mirror 66 self.append_config(config_sstate_mirror) 67 68 # Returns a list containing sstate files 69 def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True): 70 result = [] 71 for root, dirs, files in os.walk(self.sstate_path): 72 if distro_specific and re.search(r"%s/%s/[a-z0-9]{2}/[a-z0-9]{2}$" % (self.sstate_path, self.hostdistro), root): 73 for f in files: 74 if re.search(filename_regex, f): 75 result.append(f) 76 if distro_nonspecific and re.search(r"%s/[a-z0-9]{2}/[a-z0-9]{2}$" % self.sstate_path, root): 77 for f in files: 78 if re.search(filename_regex, f): 79 result.append(f) 80 return result 81 82 # Test sstate files creation and their location and directory perms 83 def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True): 84 self.config_sstate(temp_sstate_location, [self.sstate_path]) 85 86 if self.temp_sstate_location: 87 bitbake(['-cclean'] + targets) 88 else: 89 bitbake(['-ccleansstate'] + targets) 90 91 # We need to test that the env umask have does not effect sstate directory creation 92 # So, first, we'll get the current umask and set it to something we know incorrect 93 # See: sstate_task_postfunc for correct umask of os.umask(0o002) 94 import os 95 def current_umask(): 96 current_umask = os.umask(0) 97 os.umask(current_umask) 98 return current_umask 99 100 orig_umask = current_umask() 101 # Set it to a umask we know will be 'wrong' 102 os.umask(0o022) 103 104 bitbake(targets) 105 file_tracker = [] 106 results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) 107 if distro_nonspecific: 108 for r in results: 109 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo", "_fetch.tar.zst.siginfo", "_unpack.tar.zst.siginfo", "_patch.tar.zst.siginfo")): 110 continue 111 file_tracker.append(r) 112 else: 113 file_tracker = results 114 115 if should_pass: 116 self.assertTrue(file_tracker , msg="Could not find sstate files for: %s" % ', '.join(map(str, targets))) 117 else: 118 self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker))) 119 120 # Now we'll walk the tree to check the mode and see if things are incorrect. 121 badperms = [] 122 for root, dirs, files in os.walk(self.sstate_path): 123 for directory in dirs: 124 if (os.stat(os.path.join(root, directory)).st_mode & 0o777) != 0o775: 125 badperms.append(os.path.join(root, directory)) 126 127 # Return to original umask 128 os.umask(orig_umask) 129 130 if should_pass: 131 self.assertTrue(badperms , msg="Found sstate directories with the wrong permissions: %s (found %s)" % (', '.join(map(str, targets)), str(badperms))) 132 133 # Test the sstate files deletion part of the do_cleansstate task 134 def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True): 135 self.config_sstate(temp_sstate_location, [self.sstate_path]) 136 137 bitbake(['-ccleansstate'] + targets) 138 139 bitbake(targets) 140 archives_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific) 141 self.assertTrue(archives_created, msg="Could not find sstate .tar.zst files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_created))) 142 143 siginfo_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific) 144 self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created))) 145 146 bitbake(['-ccleansstate'] + targets) 147 archives_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific) 148 self.assertTrue(not archives_removed, msg="do_cleansstate didn't remove .tar.zst sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_removed))) 149 150 # Test rebuilding of distro-specific sstate files 151 def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True): 152 self.config_sstate(temp_sstate_location, [self.sstate_path]) 153 154 bitbake(['-ccleansstate'] + targets) 155 156 bitbake(targets) 157 results = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=False, distro_nonspecific=True) 158 filtered_results = [] 159 for r in results: 160 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo")): 161 continue 162 filtered_results.append(r) 163 self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results))) 164 file_tracker_1 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False) 165 self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets))) 166 167 self.track_for_cleanup(self.distro_specific_sstate + "_old") 168 shutil.copytree(self.distro_specific_sstate, self.distro_specific_sstate + "_old") 169 shutil.rmtree(self.distro_specific_sstate) 170 171 bitbake(['-cclean'] + targets) 172 bitbake(targets) 173 file_tracker_2 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False) 174 self.assertTrue(len(file_tracker_2) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets))) 175 176 not_recreated = [x for x in file_tracker_1 if x not in file_tracker_2] 177 self.assertTrue(not_recreated == [], msg="The following sstate files were not recreated: %s" % ', '.join(map(str, not_recreated))) 178 179 created_once = [x for x in file_tracker_2 if x not in file_tracker_1] 180 self.assertTrue(created_once == [], msg="The following sstate files were created only in the second run: %s" % ', '.join(map(str, created_once))) 181 182 def sstate_common_samesigs(self, configA, configB, allarch=False): 183 184 self.write_config(configA) 185 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 186 bitbake("world meta-toolchain -S none") 187 self.write_config(configB) 188 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 189 bitbake("world meta-toolchain -S none") 190 191 def get_files(d, result): 192 for root, dirs, files in os.walk(d): 193 for name in files: 194 if "meta-environment" in root or "cross-canadian" in root: 195 continue 196 if "do_build" not in name: 197 # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79 198 (_, task, _, shash) = name.rsplit(".", 3) 199 result[os.path.join(os.path.basename(root), task)] = shash 200 201 files1 = {} 202 files2 = {} 203 subdirs = sorted(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")) 204 if allarch: 205 subdirs.extend(sorted(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/all-*-linux"))) 206 207 for subdir in subdirs: 208 nativesdkdir = os.path.basename(subdir) 209 get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir, files1) 210 get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir, files2) 211 212 self.maxDiff = None 213 self.assertEqual(files1, files2) 214 215class SStateTests(SStateBase): 216 def test_autorev_sstate_works(self): 217 # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV} 218 219 tempdir = tempfile.mkdtemp(prefix='sstate_autorev') 220 tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir') 221 self.track_for_cleanup(tempdir) 222 self.track_for_cleanup(tempdldir) 223 create_temp_layer(tempdir, 'selftestrecipetool') 224 self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir) 225 self.append_config("DL_DIR = \"%s\"" % tempdldir) 226 runCmd('bitbake-layers add-layer %s' % tempdir) 227 228 # Use dbus-wait as a local git repo we can add a commit between two builds in 229 pn = 'dbus-wait' 230 srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517' 231 url = 'git://git.yoctoproject.org/dbus-wait' 232 result = runCmd('git clone %s noname' % url, cwd=tempdir) 233 srcdir = os.path.join(tempdir, 'noname') 234 result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir) 235 self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') 236 237 recipefile = os.path.join(tempdir, "recipes-test", "dbus-wait-test", 'dbus-wait-test_git.bb') 238 os.makedirs(os.path.dirname(recipefile)) 239 srcuri = 'git://' + srcdir + ';protocol=file;branch=master' 240 result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri]) 241 self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) 242 243 with open(recipefile, 'a') as f: 244 f.write('SRCREV = "${AUTOREV}"\n') 245 f.write('PV = "1.0"\n') 246 247 bitbake("dbus-wait-test -c fetch") 248 with open(os.path.join(srcdir, "bar.txt"), "w") as f: 249 f.write("foo") 250 result = runCmd('git add bar.txt; git commit -asm "add bar"', cwd=srcdir) 251 bitbake("dbus-wait-test -c unpack") 252 253class SStateCreation(SStateBase): 254 def test_sstate_creation_distro_specific_pass(self): 255 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) 256 257 def test_sstate_creation_distro_specific_fail(self): 258 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) 259 260 def test_sstate_creation_distro_nonspecific_pass(self): 261 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) 262 263 def test_sstate_creation_distro_nonspecific_fail(self): 264 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) 265 266class SStateCleanup(SStateBase): 267 def test_cleansstate_task_distro_specific_nonspecific(self): 268 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] 269 targets.append('linux-libc-headers') 270 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) 271 272 def test_cleansstate_task_distro_nonspecific(self): 273 self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) 274 275 def test_cleansstate_task_distro_specific(self): 276 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] 277 targets.append('linux-libc-headers') 278 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) 279 280class SStateDistroTests(SStateBase): 281 def test_rebuild_distro_specific_sstate_cross_native_targets(self): 282 self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True) 283 284 def test_rebuild_distro_specific_sstate_cross_target(self): 285 self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch], temp_sstate_location=True) 286 287 def test_rebuild_distro_specific_sstate_native_target(self): 288 self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True) 289 290class SStateCacheManagement(SStateBase): 291 # Test the sstate-cache-management script. Each element in the global_config list is used with the corresponding element in the target_config list 292 # global_config elements are expected to not generate any sstate files that would be removed by sstate-cache-management.py (such as changing the value of MACHINE) 293 def run_test_sstate_cache_management_script(self, target, global_config=[''], target_config=[''], ignore_patterns=[]): 294 self.assertTrue(global_config) 295 self.assertTrue(target_config) 296 self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements') 297 298 for idx in range(len(target_config)): 299 self.append_config(global_config[idx]) 300 self.append_recipeinc(target, target_config[idx]) 301 bitbake(target) 302 self.remove_config(global_config[idx]) 303 self.remove_recipeinc(target, target_config[idx]) 304 305 self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path]) 306 307 # For now this only checks if random sstate tasks are handled correctly as a group. 308 # In the future we should add control over what tasks we check for. 309 310 expected_remaining_sstate = [] 311 for idx in range(len(target_config)): 312 self.append_config(global_config[idx]) 313 self.append_recipeinc(target, target_config[idx]) 314 if target_config[idx] == target_config[-1]: 315 target_sstate_before_build = self.search_sstate(target + r'.*?\.tar.zst$') 316 bitbake("-cclean %s" % target) 317 result = bitbake(target, ignore_status=True) 318 if target_config[idx] == target_config[-1]: 319 target_sstate_after_build = self.search_sstate(target + r'.*?\.tar.zst$') 320 expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)] 321 self.remove_config(global_config[idx]) 322 self.remove_recipeinc(target, target_config[idx]) 323 self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output)) 324 325 runCmd("sstate-cache-management.py -y --cache-dir=%s --remove-duplicated" % (self.sstate_path)) 326 actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tar.zst$') if not any(pattern in x for pattern in ignore_patterns)] 327 328 actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate] 329 self.assertFalse(actual_not_expected, msg="Files should have been removed but were not: %s" % ', '.join(map(str, actual_not_expected))) 330 expected_not_actual = [x for x in expected_remaining_sstate if x not in actual_remaining_sstate] 331 self.assertFalse(expected_not_actual, msg="Extra files were removed: %s" ', '.join(map(str, expected_not_actual))) 332 333 def test_sstate_cache_management_script_using_pr_1(self): 334 global_config = [] 335 target_config = [] 336 global_config.append('') 337 target_config.append('PR = "0"') 338 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 339 340 def test_sstate_cache_management_script_using_pr_2(self): 341 global_config = [] 342 target_config = [] 343 global_config.append('') 344 target_config.append('PR = "0"') 345 global_config.append('') 346 target_config.append('PR = "1"') 347 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 348 349 def test_sstate_cache_management_script_using_pr_3(self): 350 global_config = [] 351 target_config = [] 352 global_config.append('MACHINE = "qemux86-64"') 353 target_config.append('PR = "0"') 354 global_config.append(global_config[0]) 355 target_config.append('PR = "1"') 356 global_config.append('MACHINE = "qemux86"') 357 target_config.append('PR = "1"') 358 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 359 360 def test_sstate_cache_management_script_using_machine(self): 361 global_config = [] 362 target_config = [] 363 global_config.append('MACHINE = "qemux86-64"') 364 target_config.append('') 365 global_config.append('MACHINE = "qemux86"') 366 target_config.append('') 367 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 368 369class SStateHashSameSigs(SStateBase): 370 def sstate_hashtest(self, sdkmachine): 371 372 self.write_config(""" 373MACHINE = "qemux86" 374TMPDIR = "${TOPDIR}/tmp-sstatesamehash" 375BUILD_ARCH = "x86_64" 376BUILD_OS = "linux" 377SDKMACHINE = "x86_64" 378PACKAGE_CLASSES = "package_rpm package_ipk package_deb" 379BB_SIGNATURE_HANDLER = "OEBasicHash" 380""") 381 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 382 bitbake("core-image-weston -S none") 383 self.write_config(""" 384MACHINE = "qemux86" 385TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" 386BUILD_ARCH = "i686" 387BUILD_OS = "linux" 388SDKMACHINE = "%s" 389PACKAGE_CLASSES = "package_rpm package_ipk package_deb" 390BB_SIGNATURE_HANDLER = "OEBasicHash" 391""" % sdkmachine) 392 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 393 bitbake("core-image-weston -S none") 394 395 def get_files(d): 396 f = [] 397 for root, dirs, files in os.walk(d): 398 if "core-image-weston" in root: 399 # SDKMACHINE changing will change 400 # do_rootfs/do_testimage/do_build stamps of images which 401 # is safe to ignore. 402 continue 403 f.extend(os.path.join(root, name) for name in files) 404 return f 405 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") 406 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") 407 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash").replace("i686-linux", "x86_64-linux").replace("i686" + self.target_vendor + "-linux", "x86_64" + self.target_vendor + "-linux", ) for x in files2] 408 self.maxDiff = None 409 self.assertCountEqual(files1, files2) 410 411 def test_sstate_32_64_same_hash(self): 412 """ 413 The sstate checksums for both native and target should not vary whether 414 they're built on a 32 or 64 bit system. Rather than requiring two different 415 build machines and running a builds, override the variables calling uname() 416 manually and check using bitbake -S. 417 """ 418 self.sstate_hashtest("i686") 419 420 def test_sstate_sdk_arch_same_hash(self): 421 """ 422 Similarly, test an arm SDK has the same hashes 423 """ 424 self.sstate_hashtest("aarch64") 425 426 def test_sstate_nativelsbstring_same_hash(self): 427 """ 428 The sstate checksums should be independent of whichever NATIVELSBSTRING is 429 detected. Rather than requiring two different build machines and running 430 builds, override the variables manually and check using bitbake -S. 431 """ 432 433 self.write_config(""" 434TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 435NATIVELSBSTRING = \"DistroA\" 436BB_SIGNATURE_HANDLER = "OEBasicHash" 437""") 438 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 439 bitbake("core-image-weston -S none") 440 self.write_config(""" 441TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 442NATIVELSBSTRING = \"DistroB\" 443BB_SIGNATURE_HANDLER = "OEBasicHash" 444""") 445 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 446 bitbake("core-image-weston -S none") 447 448 def get_files(d): 449 f = [] 450 for root, dirs, files in os.walk(d): 451 f.extend(os.path.join(root, name) for name in files) 452 return f 453 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") 454 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") 455 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 456 self.maxDiff = None 457 self.assertCountEqual(files1, files2) 458 459class SStateHashSameSigs2(SStateBase): 460 def test_sstate_allarch_samesigs(self): 461 """ 462 The sstate checksums of allarch packages should be independent of whichever 463 MACHINE is set. Check this using bitbake -S. 464 Also, rather than duplicate the test, check nativesdk stamps are the same between 465 the two MACHINE values. 466 """ 467 468 configA = """ 469TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 470MACHINE = \"qemux86-64\" 471BB_SIGNATURE_HANDLER = "OEBasicHash" 472""" 473 #OLDEST_KERNEL is arch specific so set to a different value here for testing 474 configB = """ 475TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 476MACHINE = \"qemuarm\" 477OLDEST_KERNEL = \"3.3.0\" 478BB_SIGNATURE_HANDLER = "OEBasicHash" 479ERROR_QA:append = " somenewoption" 480WARN_QA:append = " someotheroption" 481""" 482 self.sstate_common_samesigs(configA, configB, allarch=True) 483 484 def test_sstate_nativesdk_samesigs_multilib(self): 485 """ 486 check nativesdk stamps are the same between the two MACHINE values. 487 """ 488 489 configA = """ 490TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 491MACHINE = \"qemux86-64\" 492require conf/multilib.conf 493MULTILIBS = \"multilib:lib32\" 494DEFAULTTUNE:virtclass-multilib-lib32 = \"x86\" 495BB_SIGNATURE_HANDLER = "OEBasicHash" 496""" 497 configB = """ 498TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 499MACHINE = \"qemuarm\" 500require conf/multilib.conf 501MULTILIBS = \"\" 502BB_SIGNATURE_HANDLER = "OEBasicHash" 503""" 504 self.sstate_common_samesigs(configA, configB) 505 506class SStateHashSameSigs3(SStateBase): 507 def test_sstate_sametune_samesigs(self): 508 """ 509 The sstate checksums of two identical machines (using the same tune) should be the 510 same, apart from changes within the machine specific stamps directory. We use the 511 qemux86copy machine to test this. Also include multilibs in the test. 512 """ 513 514 self.write_config(""" 515TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 516MACHINE = \"qemux86\" 517require conf/multilib.conf 518MULTILIBS = "multilib:lib32" 519DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 520BB_SIGNATURE_HANDLER = "OEBasicHash" 521""") 522 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 523 bitbake("world meta-toolchain -S none") 524 self.write_config(""" 525TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 526MACHINE = \"qemux86copy\" 527require conf/multilib.conf 528MULTILIBS = "multilib:lib32" 529DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 530BB_SIGNATURE_HANDLER = "OEBasicHash" 531""") 532 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 533 bitbake("world meta-toolchain -S none") 534 535 def get_files(d): 536 f = [] 537 for root, dirs, files in os.walk(d): 538 for name in files: 539 if "meta-environment" in root or "cross-canadian" in root or 'meta-ide-support' in root: 540 continue 541 if "qemux86copy-" in root or "qemux86-" in root: 542 continue 543 if "do_build" not in name and "do_populate_sdk" not in name: 544 f.append(os.path.join(root, name)) 545 return f 546 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") 547 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") 548 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 549 self.maxDiff = None 550 self.assertCountEqual(files1, files2) 551 552 553 def test_sstate_multilib_or_not_native_samesigs(self): 554 """The sstate checksums of two native recipes (and their dependencies) 555 where the target is using multilib in one but not the other 556 should be the same. We use the qemux86copy machine to test 557 this. 558 """ 559 560 self.write_config(""" 561TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 562MACHINE = \"qemux86\" 563require conf/multilib.conf 564MULTILIBS = "multilib:lib32" 565DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 566BB_SIGNATURE_HANDLER = "OEBasicHash" 567""") 568 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 569 bitbake("binutils-native -S none") 570 self.write_config(""" 571TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 572MACHINE = \"qemux86copy\" 573BB_SIGNATURE_HANDLER = "OEBasicHash" 574""") 575 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 576 bitbake("binutils-native -S none") 577 578 def get_files(d): 579 f = [] 580 for root, dirs, files in os.walk(d): 581 for name in files: 582 f.append(os.path.join(root, name)) 583 return f 584 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") 585 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") 586 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 587 self.maxDiff = None 588 self.assertCountEqual(files1, files2) 589 590class SStateHashSameSigs4(SStateBase): 591 def test_sstate_noop_samesigs(self): 592 """ 593 The sstate checksums of two builds with these variables changed or 594 classes inherits should be the same. 595 """ 596 597 self.write_config(""" 598TMPDIR = "${TOPDIR}/tmp-sstatesamehash" 599BB_NUMBER_THREADS = "${@oe.utils.cpu_count()}" 600PARALLEL_MAKE = "-j 1" 601DL_DIR = "${TOPDIR}/download1" 602TIME = "111111" 603DATE = "20161111" 604INHERIT:remove = "buildstats-summary buildhistory uninative" 605http_proxy = "" 606BB_SIGNATURE_HANDLER = "OEBasicHash" 607""") 608 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 609 self.track_for_cleanup(self.topdir + "/download1") 610 bitbake("world meta-toolchain -S none") 611 self.write_config(""" 612TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" 613BB_NUMBER_THREADS = "${@oe.utils.cpu_count()+1}" 614PARALLEL_MAKE = "-j 2" 615DL_DIR = "${TOPDIR}/download2" 616TIME = "222222" 617DATE = "20161212" 618# Always remove uninative as we're changing proxies 619INHERIT:remove = "uninative" 620INHERIT += "buildstats-summary buildhistory" 621http_proxy = "http://example.com/" 622BB_SIGNATURE_HANDLER = "OEBasicHash" 623""") 624 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 625 self.track_for_cleanup(self.topdir + "/download2") 626 bitbake("world meta-toolchain -S none") 627 628 def get_files(d): 629 f = {} 630 for root, dirs, files in os.walk(d): 631 for name in files: 632 name, shash = name.rsplit('.', 1) 633 # Extract just the machine and recipe name 634 base = os.sep.join(root.rsplit(os.sep, 2)[-2:] + [name]) 635 f[base] = shash 636 return f 637 638 def compare_sigfiles(files, files1, files2, compare=False): 639 for k in files: 640 if k in files1 and k in files2: 641 print("%s differs:" % k) 642 if compare: 643 sigdatafile1 = self.topdir + "/tmp-sstatesamehash/stamps/" + k + "." + files1[k] 644 sigdatafile2 = self.topdir + "/tmp-sstatesamehash2/stamps/" + k + "." + files2[k] 645 output = bb.siggen.compare_sigfiles(sigdatafile1, sigdatafile2) 646 if output: 647 print('\n'.join(output)) 648 elif k in files1 and k not in files2: 649 print("%s in files1" % k) 650 elif k not in files1 and k in files2: 651 print("%s in files2" % k) 652 else: 653 assert "shouldn't reach here" 654 655 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") 656 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") 657 # Remove items that are identical in both sets 658 for k,v in files1.items() & files2.items(): 659 del files1[k] 660 del files2[k] 661 if not files1 and not files2: 662 # No changes, so we're done 663 return 664 665 files = list(files1.keys() | files2.keys()) 666 # this is an expensive computation, thus just compare the first 'max_sigfiles_to_compare' k files 667 max_sigfiles_to_compare = 20 668 first, rest = files[:max_sigfiles_to_compare], files[max_sigfiles_to_compare:] 669 compare_sigfiles(first, files1, files2, compare=True) 670 compare_sigfiles(rest, files1, files2, compare=False) 671 672 self.fail("sstate hashes not identical.") 673 674 def test_sstate_movelayer_samesigs(self): 675 """ 676 The sstate checksums of two builds with the same oe-core layer in two 677 different locations should be the same. 678 """ 679 core_layer = os.path.join( 680 self.tc.td["COREBASE"], 'meta') 681 copy_layer_1 = self.topdir + "/meta-copy1/meta" 682 copy_layer_2 = self.topdir + "/meta-copy2/meta" 683 684 oe.path.copytree(core_layer, copy_layer_1) 685 os.symlink(os.path.dirname(core_layer) + "/scripts", self.topdir + "/meta-copy1/scripts") 686 self.write_config(""" 687TMPDIR = "${TOPDIR}/tmp-sstatesamehash" 688""") 689 bblayers_conf = 'BBLAYERS += "%s"\nBBLAYERS:remove = "%s"' % (copy_layer_1, core_layer) 690 self.write_bblayers_config(bblayers_conf) 691 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 692 bitbake("bash -S none") 693 694 oe.path.copytree(core_layer, copy_layer_2) 695 os.symlink(os.path.dirname(core_layer) + "/scripts", self.topdir + "/meta-copy2/scripts") 696 self.write_config(""" 697TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" 698""") 699 bblayers_conf = 'BBLAYERS += "%s"\nBBLAYERS:remove = "%s"' % (copy_layer_2, core_layer) 700 self.write_bblayers_config(bblayers_conf) 701 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 702 bitbake("bash -S none") 703 704 def get_files(d): 705 f = [] 706 for root, dirs, files in os.walk(d): 707 for name in files: 708 f.append(os.path.join(root, name)) 709 return f 710 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") 711 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") 712 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 713 self.maxDiff = None 714 self.assertCountEqual(files1, files2) 715 716class SStateFindSiginfo(SStateBase): 717 def test_sstate_compare_sigfiles_and_find_siginfo(self): 718 """ 719 Test the functionality of the find_siginfo: basic function and callback in compare_sigfiles 720 """ 721 self.write_config(""" 722TMPDIR = \"${TOPDIR}/tmp-sstates-findsiginfo\" 723MACHINE = \"qemux86-64\" 724require conf/multilib.conf 725MULTILIBS = "multilib:lib32" 726DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 727BB_SIGNATURE_HANDLER = "OEBasicHash" 728""") 729 self.track_for_cleanup(self.topdir + "/tmp-sstates-findsiginfo") 730 731 pns = ["binutils", "binutils-native", "lib32-binutils"] 732 target_configs = [ 733""" 734TMPVAL1 = "tmpval1" 735TMPVAL2 = "tmpval2" 736do_tmptask1() { 737 echo ${TMPVAL1} 738} 739do_tmptask2() { 740 echo ${TMPVAL2} 741} 742addtask do_tmptask1 743addtask tmptask2 before do_tmptask1 744""", 745""" 746TMPVAL3 = "tmpval3" 747TMPVAL4 = "tmpval4" 748do_tmptask1() { 749 echo ${TMPVAL3} 750} 751do_tmptask2() { 752 echo ${TMPVAL4} 753} 754addtask do_tmptask1 755addtask tmptask2 before do_tmptask1 756""" 757 ] 758 759 for target_config in target_configs: 760 self.write_recipeinc("binutils", target_config) 761 for pn in pns: 762 bitbake("%s -c do_tmptask1 -S none" % pn) 763 self.delete_recipeinc("binutils") 764 765 with bb.tinfoil.Tinfoil() as tinfoil: 766 tinfoil.prepare(config_only=True) 767 768 def find_siginfo(pn, taskname, sigs=None): 769 result = None 770 command_complete = False 771 tinfoil.set_event_mask(["bb.event.FindSigInfoResult", 772 "bb.command.CommandCompleted"]) 773 ret = tinfoil.run_command("findSigInfo", pn, taskname, sigs) 774 if ret: 775 while result is None or not command_complete: 776 event = tinfoil.wait_event(1) 777 if event: 778 if isinstance(event, bb.command.CommandCompleted): 779 command_complete = True 780 elif isinstance(event, bb.event.FindSigInfoResult): 781 result = event.result 782 return result 783 784 def recursecb(key, hash1, hash2): 785 nonlocal recursecb_count 786 recursecb_count += 1 787 hashes = [hash1, hash2] 788 hashfiles = find_siginfo(key, None, hashes) 789 self.assertCountEqual(hashes, hashfiles) 790 bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb) 791 792 for pn in pns: 793 recursecb_count = 0 794 matches = find_siginfo(pn, "do_tmptask1") 795 self.assertGreaterEqual(len(matches), 2) 796 latesthashes = sorted(matches.keys(), key=lambda h: matches[h]['time'])[-2:] 797 bb.siggen.compare_sigfiles(matches[latesthashes[-2]]['path'], matches[latesthashes[-1]]['path'], recursecb) 798 self.assertEqual(recursecb_count,1) 799 800class SStatePrintdiff(SStateBase): 801 def run_test_printdiff_changerecipe(self, target, change_recipe, change_bbtask, change_content, expected_sametmp_output, expected_difftmp_output): 802 import time 803 self.write_config(""" 804TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-sametmp-{}" 805""".format(time.time())) 806 # Use runall do_build to ensure any indirect sstate is created, e.g. tzcode-native on both x86 and 807 # aarch64 hosts since only allarch target recipes depend upon it and it may not be built otherwise. 808 # A bitbake -c cleansstate tzcode-native would cause some of these tests to error for example. 809 bitbake("--runall build --runall deploy_source_date_epoch {}".format(target)) 810 bitbake("-S none {}".format(target)) 811 bitbake(change_bbtask) 812 self.write_recipeinc(change_recipe, change_content) 813 result_sametmp = bitbake("-S printdiff {}".format(target)) 814 815 self.write_config(""" 816TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-difftmp-{}" 817""".format(time.time())) 818 result_difftmp = bitbake("-S printdiff {}".format(target)) 819 820 self.delete_recipeinc(change_recipe) 821 for item in expected_sametmp_output: 822 self.assertIn(item, result_sametmp.output, msg = "Item {} not found in output:\n{}".format(item, result_sametmp.output)) 823 for item in expected_difftmp_output: 824 self.assertIn(item, result_difftmp.output, msg = "Item {} not found in output:\n{}".format(item, result_difftmp.output)) 825 826 def run_test_printdiff_changeconfig(self, target, change_bbtasks, change_content, expected_sametmp_output, expected_difftmp_output): 827 import time 828 self.write_config(""" 829TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-sametmp-{}" 830""".format(time.time())) 831 bitbake("--runall build --runall deploy_source_date_epoch {}".format(target)) 832 bitbake("-S none {}".format(target)) 833 bitbake(" ".join(change_bbtasks)) 834 self.append_config(change_content) 835 result_sametmp = bitbake("-S printdiff {}".format(target)) 836 837 self.write_config(""" 838TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-difftmp-{}" 839""".format(time.time())) 840 self.append_config(change_content) 841 result_difftmp = bitbake("-S printdiff {}".format(target)) 842 843 for item in expected_sametmp_output: 844 self.assertIn(item, result_sametmp.output, msg = "Item {} not found in output:\n{}".format(item, result_sametmp.output)) 845 for item in expected_difftmp_output: 846 self.assertIn(item, result_difftmp.output, msg = "Item {} not found in output:\n{}".format(item, result_difftmp.output)) 847 848 849 # Check if printdiff walks the full dependency chain from the image target to where the change is in a specific recipe 850 def test_image_minimal_vs_perlcross(self): 851 expected_output = ("Task perlcross-native:do_install couldn't be used from the cache because:", 852"We need hash", 853"most recent matching task was") 854 expected_sametmp_output = expected_output + ( 855"Variable do_install value changed", 856'+ echo "this changes the task signature"') 857 expected_difftmp_output = expected_output 858 859 self.run_test_printdiff_changerecipe("core-image-minimal", "perlcross", "-c do_install perlcross-native", 860""" 861do_install:append() { 862 echo "this changes the task signature" 863} 864""", 865expected_sametmp_output, expected_difftmp_output) 866 867 # Check if changes to gcc-source (which uses tmp/work-shared) are correctly discovered 868 def test_gcc_runtime_vs_gcc_source(self): 869 gcc_source_pn = 'gcc-source-%s' % get_bb_vars(['PV'], 'gcc')['PV'] 870 871 expected_output = ("Task {}:do_preconfigure couldn't be used from the cache because:".format(gcc_source_pn), 872"We need hash", 873"most recent matching task was") 874 expected_sametmp_output = expected_output + ( 875"Variable do_preconfigure value changed", 876'+ print("this changes the task signature")') 877 expected_difftmp_output = expected_output 878 879 self.run_test_printdiff_changerecipe("gcc-runtime", "gcc-source", "-c do_preconfigure {}".format(gcc_source_pn), 880""" 881python do_preconfigure:append() { 882 print("this changes the task signature") 883} 884""", 885expected_sametmp_output, expected_difftmp_output) 886 887 # Check if changing a really base task definiton is reported against multiple core recipes using it 888 def test_image_minimal_vs_base_do_configure(self): 889 change_bbtasks = ('zstd-native:do_configure', 890'texinfo-dummy-native:do_configure', 891'ldconfig-native:do_configure', 892'gettext-minimal-native:do_configure', 893'tzcode-native:do_configure', 894'makedevs-native:do_configure', 895'pigz-native:do_configure', 896'update-rc.d-native:do_configure', 897'unzip-native:do_configure', 898'gnu-config-native:do_configure') 899 900 expected_output = ["Task {} couldn't be used from the cache because:".format(t) for t in change_bbtasks] + [ 901"We need hash", 902"most recent matching task was"] 903 904 expected_sametmp_output = expected_output + [ 905"Variable base_do_configure value changed", 906'+ echo "this changes base_do_configure() definiton "'] 907 expected_difftmp_output = expected_output 908 909 self.run_test_printdiff_changeconfig("core-image-minimal",change_bbtasks, 910""" 911INHERIT += "base-do-configure-modified" 912""", 913expected_sametmp_output, expected_difftmp_output) 914 915class SStateCheckObjectPresence(SStateBase): 916 def check_bb_output(self, output, targets, exceptions, check_cdn): 917 def is_exception(object, exceptions): 918 for e in exceptions: 919 if re.search(e, object): 920 return True 921 return False 922 923 # sstate is checked for existence of these, but they never get written out to begin with 924 exceptions += ["{}.*image_qa".format(t) for t in targets.split()] 925 exceptions += ["{}.*deploy_source_date_epoch".format(t) for t in targets.split()] 926 exceptions += ["{}.*image_complete".format(t) for t in targets.split()] 927 exceptions += ["linux-yocto.*shared_workdir"] 928 # these get influnced by IMAGE_FSTYPES tweaks in yocto-autobuilder-helper's config.json (on x86-64) 929 # additionally, they depend on noexec (thus, absent stamps) package, install, etc. image tasks, 930 # which makes tracing other changes difficult 931 exceptions += ["{}.*create_.*spdx".format(t) for t in targets.split()] 932 933 output_l = output.splitlines() 934 for l in output_l: 935 if l.startswith("Sstate summary"): 936 for idx, item in enumerate(l.split()): 937 if item == 'Missed': 938 missing_objects = int(l.split()[idx+1]) 939 break 940 else: 941 self.fail("Did not find missing objects amount in sstate summary: {}".format(l)) 942 break 943 else: 944 self.fail("Did not find 'Sstate summary' line in bitbake output") 945 946 failed_urls = [] 947 failed_urls_extrainfo = [] 948 for l in output_l: 949 if "SState: Unsuccessful fetch test for" in l and check_cdn: 950 missing_object = l.split()[6] 951 elif "SState: Looked for but didn't find file" in l and not check_cdn: 952 missing_object = l.split()[8] 953 else: 954 missing_object = None 955 if missing_object: 956 if not is_exception(missing_object, exceptions): 957 failed_urls.append(missing_object) 958 else: 959 missing_objects -= 1 960 961 if "urlopen failed for" in l and not is_exception(l, exceptions): 962 failed_urls_extrainfo.append(l) 963 964 self.assertEqual(len(failed_urls), missing_objects, "Amount of reported missing objects does not match failed URLs: {}\nFailed URLs:\n{}\nFetcher diagnostics:\n{}".format(missing_objects, "\n".join(failed_urls), "\n".join(failed_urls_extrainfo))) 965 self.assertEqual(len(failed_urls), 0, "Missing objects in the cache:\n{}\nFetcher diagnostics:\n{}".format("\n".join(failed_urls), "\n".join(failed_urls_extrainfo))) 966 967@OETestTag("yocto-mirrors") 968class SStateMirrors(SStateCheckObjectPresence): 969 def run_test(self, machine, targets, exceptions, check_cdn = True, ignore_errors = False): 970 if check_cdn: 971 self.config_sstate(True) 972 self.append_config(""" 973MACHINE = "{}" 974BB_HASHSERVE_UPSTREAM = "hashserv.yoctoproject.org:8686" 975SSTATE_MIRRORS ?= "file://.* http://cdn.jsdelivr.net/yocto/sstate/all/PATH;downloadfilename=PATH" 976""".format(machine)) 977 else: 978 self.append_config(""" 979MACHINE = "{}" 980""".format(machine)) 981 result = bitbake("-DD -n {}".format(targets)) 982 bitbake("-S none {}".format(targets)) 983 if ignore_errors: 984 return 985 self.check_bb_output(result.output, targets, exceptions, check_cdn) 986 987 def test_cdn_mirror_qemux86_64(self): 988 exceptions = [] 989 self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions) 990 991 def test_cdn_mirror_qemuarm64(self): 992 exceptions = [] 993 self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions) 994 995 def test_local_cache_qemux86_64(self): 996 exceptions = [] 997 self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False) 998 999 def test_local_cache_qemuarm64(self): 1000 exceptions = [] 1001 self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False) 1002