1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: MIT 5# 6 7import os 8import shutil 9import glob 10import subprocess 11import tempfile 12import datetime 13import re 14 15from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, get_bb_vars 16from oeqa.selftest.case import OESelftestTestCase 17from oeqa.core.decorator import OETestTag 18 19import oe 20import bb.siggen 21 22# Set to True to preserve stamp files after test execution for debugging failures 23keep_temp_files = False 24 25class SStateBase(OESelftestTestCase): 26 27 def setUpLocal(self): 28 super(SStateBase, self).setUpLocal() 29 self.temp_sstate_location = None 30 needed_vars = ['SSTATE_DIR', 'NATIVELSBSTRING', 'TCLIBC', 'TUNE_ARCH', 31 'TOPDIR', 'TARGET_VENDOR', 'TARGET_OS'] 32 bb_vars = get_bb_vars(needed_vars) 33 self.sstate_path = bb_vars['SSTATE_DIR'] 34 self.hostdistro = bb_vars['NATIVELSBSTRING'] 35 self.tclibc = bb_vars['TCLIBC'] 36 self.tune_arch = bb_vars['TUNE_ARCH'] 37 self.topdir = bb_vars['TOPDIR'] 38 self.target_vendor = bb_vars['TARGET_VENDOR'] 39 self.target_os = bb_vars['TARGET_OS'] 40 self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) 41 42 def track_for_cleanup(self, path): 43 if not keep_temp_files: 44 super().track_for_cleanup(path) 45 46 # Creates a special sstate configuration with the option to add sstate mirrors 47 def config_sstate(self, temp_sstate_location=False, add_local_mirrors=[]): 48 self.temp_sstate_location = temp_sstate_location 49 50 if self.temp_sstate_location: 51 temp_sstate_path = os.path.join(self.builddir, "temp_sstate_%s" % datetime.datetime.now().strftime('%Y%m%d%H%M%S')) 52 config_temp_sstate = "SSTATE_DIR = \"%s\"" % temp_sstate_path 53 self.append_config(config_temp_sstate) 54 self.track_for_cleanup(temp_sstate_path) 55 bb_vars = get_bb_vars(['SSTATE_DIR', 'NATIVELSBSTRING']) 56 self.sstate_path = bb_vars['SSTATE_DIR'] 57 self.hostdistro = bb_vars['NATIVELSBSTRING'] 58 self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) 59 60 if add_local_mirrors: 61 config_set_sstate_if_not_set = 'SSTATE_MIRRORS ?= ""' 62 self.append_config(config_set_sstate_if_not_set) 63 for local_mirror in add_local_mirrors: 64 self.assertFalse(os.path.join(local_mirror) == os.path.join(self.sstate_path), msg='Cannot add the current sstate path as a sstate mirror') 65 config_sstate_mirror = "SSTATE_MIRRORS += \"file://.* file:///%s/PATH\"" % local_mirror 66 self.append_config(config_sstate_mirror) 67 68 # Returns a list containing sstate files 69 def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True): 70 result = [] 71 for root, dirs, files in os.walk(self.sstate_path): 72 if distro_specific and re.search(r"%s/%s/[a-z0-9]{2}/[a-z0-9]{2}$" % (self.sstate_path, self.hostdistro), root): 73 for f in files: 74 if re.search(filename_regex, f): 75 result.append(f) 76 if distro_nonspecific and re.search(r"%s/[a-z0-9]{2}/[a-z0-9]{2}$" % self.sstate_path, root): 77 for f in files: 78 if re.search(filename_regex, f): 79 result.append(f) 80 return result 81 82 # Test sstate files creation and their location and directory perms 83 def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True): 84 self.config_sstate(temp_sstate_location, [self.sstate_path]) 85 86 if self.temp_sstate_location: 87 bitbake(['-cclean'] + targets) 88 else: 89 bitbake(['-ccleansstate'] + targets) 90 91 # We need to test that the env umask have does not effect sstate directory creation 92 # So, first, we'll get the current umask and set it to something we know incorrect 93 # See: sstate_task_postfunc for correct umask of os.umask(0o002) 94 import os 95 def current_umask(): 96 current_umask = os.umask(0) 97 os.umask(current_umask) 98 return current_umask 99 100 orig_umask = current_umask() 101 # Set it to a umask we know will be 'wrong' 102 os.umask(0o022) 103 104 bitbake(targets) 105 file_tracker = [] 106 results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) 107 if distro_nonspecific: 108 for r in results: 109 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo", "_fetch.tar.zst.siginfo", "_unpack.tar.zst.siginfo", "_patch.tar.zst.siginfo")): 110 continue 111 file_tracker.append(r) 112 else: 113 file_tracker = results 114 115 if should_pass: 116 self.assertTrue(file_tracker , msg="Could not find sstate files for: %s" % ', '.join(map(str, targets))) 117 else: 118 self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker))) 119 120 # Now we'll walk the tree to check the mode and see if things are incorrect. 121 badperms = [] 122 for root, dirs, files in os.walk(self.sstate_path): 123 for directory in dirs: 124 if (os.stat(os.path.join(root, directory)).st_mode & 0o777) != 0o775: 125 badperms.append(os.path.join(root, directory)) 126 127 # Return to original umask 128 os.umask(orig_umask) 129 130 if should_pass: 131 self.assertTrue(badperms , msg="Found sstate directories with the wrong permissions: %s (found %s)" % (', '.join(map(str, targets)), str(badperms))) 132 133 # Test the sstate files deletion part of the do_cleansstate task 134 def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True): 135 self.config_sstate(temp_sstate_location, [self.sstate_path]) 136 137 bitbake(['-ccleansstate'] + targets) 138 139 bitbake(targets) 140 archives_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific) 141 self.assertTrue(archives_created, msg="Could not find sstate .tar.zst files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_created))) 142 143 siginfo_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific) 144 self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created))) 145 146 bitbake(['-ccleansstate'] + targets) 147 archives_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific) 148 self.assertTrue(not archives_removed, msg="do_cleansstate didn't remove .tar.zst sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_removed))) 149 150 # Test rebuilding of distro-specific sstate files 151 def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True): 152 self.config_sstate(temp_sstate_location, [self.sstate_path]) 153 154 bitbake(['-ccleansstate'] + targets) 155 156 bitbake(targets) 157 results = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=False, distro_nonspecific=True) 158 filtered_results = [] 159 for r in results: 160 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo")): 161 continue 162 filtered_results.append(r) 163 self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results))) 164 file_tracker_1 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False) 165 self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets))) 166 167 self.track_for_cleanup(self.distro_specific_sstate + "_old") 168 shutil.copytree(self.distro_specific_sstate, self.distro_specific_sstate + "_old") 169 shutil.rmtree(self.distro_specific_sstate) 170 171 bitbake(['-cclean'] + targets) 172 bitbake(targets) 173 file_tracker_2 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False) 174 self.assertTrue(len(file_tracker_2) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets))) 175 176 not_recreated = [x for x in file_tracker_1 if x not in file_tracker_2] 177 self.assertTrue(not_recreated == [], msg="The following sstate files were not recreated: %s" % ', '.join(map(str, not_recreated))) 178 179 created_once = [x for x in file_tracker_2 if x not in file_tracker_1] 180 self.assertTrue(created_once == [], msg="The following sstate files were created only in the second run: %s" % ', '.join(map(str, created_once))) 181 182 def sstate_common_samesigs(self, configA, configB, allarch=False): 183 184 self.write_config(configA) 185 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 186 bitbake("world meta-toolchain -S none") 187 self.write_config(configB) 188 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 189 bitbake("world meta-toolchain -S none") 190 191 def get_files(d, result): 192 for root, dirs, files in os.walk(d): 193 for name in files: 194 if "meta-environment" in root or "cross-canadian" in root: 195 continue 196 if "do_build" not in name: 197 # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79 198 (_, task, _, shash) = name.rsplit(".", 3) 199 result[os.path.join(os.path.basename(root), task)] = shash 200 201 files1 = {} 202 files2 = {} 203 subdirs = sorted(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")) 204 if allarch: 205 subdirs.extend(sorted(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/all-*-linux"))) 206 207 for subdir in subdirs: 208 nativesdkdir = os.path.basename(subdir) 209 get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir, files1) 210 get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir, files2) 211 212 self.maxDiff = None 213 self.assertEqual(files1, files2) 214 215class SStateTests(SStateBase): 216 def test_autorev_sstate_works(self): 217 # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV} 218 219 tempdir = tempfile.mkdtemp(prefix='sstate_autorev') 220 tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir') 221 self.track_for_cleanup(tempdir) 222 self.track_for_cleanup(tempdldir) 223 create_temp_layer(tempdir, 'selftestrecipetool') 224 self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir) 225 self.append_config("DL_DIR = \"%s\"" % tempdldir) 226 runCmd('bitbake-layers add-layer %s' % tempdir) 227 228 # Use dbus-wait as a local git repo we can add a commit between two builds in 229 pn = 'dbus-wait' 230 srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517' 231 url = 'git://git.yoctoproject.org/dbus-wait' 232 result = runCmd('git clone %s noname' % url, cwd=tempdir) 233 srcdir = os.path.join(tempdir, 'noname') 234 result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir) 235 self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') 236 237 recipefile = os.path.join(tempdir, "recipes-test", "dbus-wait-test", 'dbus-wait-test_git.bb') 238 os.makedirs(os.path.dirname(recipefile)) 239 srcuri = 'git://' + srcdir + ';protocol=file;branch=master' 240 result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri]) 241 self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) 242 243 with open(recipefile, 'a') as f: 244 f.write('SRCREV = "${AUTOREV}"\n') 245 f.write('PV = "1.0"\n') 246 247 bitbake("dbus-wait-test -c fetch") 248 with open(os.path.join(srcdir, "bar.txt"), "w") as f: 249 f.write("foo") 250 result = runCmd('git add bar.txt; git commit -asm "add bar"', cwd=srcdir) 251 bitbake("dbus-wait-test -c unpack") 252 253class SStateCreation(SStateBase): 254 def test_sstate_creation_distro_specific_pass(self): 255 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) 256 257 def test_sstate_creation_distro_specific_fail(self): 258 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) 259 260 def test_sstate_creation_distro_nonspecific_pass(self): 261 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) 262 263 def test_sstate_creation_distro_nonspecific_fail(self): 264 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) 265 266class SStateCleanup(SStateBase): 267 def test_cleansstate_task_distro_specific_nonspecific(self): 268 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] 269 targets.append('linux-libc-headers') 270 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) 271 272 def test_cleansstate_task_distro_nonspecific(self): 273 self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) 274 275 def test_cleansstate_task_distro_specific(self): 276 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] 277 targets.append('linux-libc-headers') 278 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) 279 280class SStateDistroTests(SStateBase): 281 def test_rebuild_distro_specific_sstate_cross_native_targets(self): 282 self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True) 283 284 def test_rebuild_distro_specific_sstate_cross_target(self): 285 self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch], temp_sstate_location=True) 286 287 def test_rebuild_distro_specific_sstate_native_target(self): 288 self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True) 289 290class SStateCacheManagement(SStateBase): 291 # Test the sstate-cache-management script. Each element in the global_config list is used with the corresponding element in the target_config list 292 # global_config elements are expected to not generate any sstate files that would be removed by sstate-cache-management.py (such as changing the value of MACHINE) 293 def run_test_sstate_cache_management_script(self, target, global_config=[''], target_config=[''], ignore_patterns=[]): 294 self.assertTrue(global_config) 295 self.assertTrue(target_config) 296 self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements') 297 298 for idx in range(len(target_config)): 299 self.append_config(global_config[idx]) 300 self.append_recipeinc(target, target_config[idx]) 301 bitbake(target) 302 self.remove_config(global_config[idx]) 303 self.remove_recipeinc(target, target_config[idx]) 304 305 self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path]) 306 307 # For now this only checks if random sstate tasks are handled correctly as a group. 308 # In the future we should add control over what tasks we check for. 309 310 expected_remaining_sstate = [] 311 for idx in range(len(target_config)): 312 self.append_config(global_config[idx]) 313 self.append_recipeinc(target, target_config[idx]) 314 if target_config[idx] == target_config[-1]: 315 target_sstate_before_build = self.search_sstate(target + r'.*?\.tar.zst$') 316 bitbake("-cclean %s" % target) 317 result = bitbake(target, ignore_status=True) 318 if target_config[idx] == target_config[-1]: 319 target_sstate_after_build = self.search_sstate(target + r'.*?\.tar.zst$') 320 expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)] 321 self.remove_config(global_config[idx]) 322 self.remove_recipeinc(target, target_config[idx]) 323 self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output)) 324 325 runCmd("sstate-cache-management.py -y --cache-dir=%s --remove-duplicated" % (self.sstate_path)) 326 actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tar.zst$') if not any(pattern in x for pattern in ignore_patterns)] 327 328 actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate] 329 self.assertFalse(actual_not_expected, msg="Files should have been removed but were not: %s" % ', '.join(map(str, actual_not_expected))) 330 expected_not_actual = [x for x in expected_remaining_sstate if x not in actual_remaining_sstate] 331 self.assertFalse(expected_not_actual, msg="Extra files were removed: %s" ', '.join(map(str, expected_not_actual))) 332 333 def test_sstate_cache_management_script_using_pr_1(self): 334 global_config = [] 335 target_config = [] 336 global_config.append('') 337 target_config.append('PR = "0"') 338 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 339 340 def test_sstate_cache_management_script_using_pr_2(self): 341 global_config = [] 342 target_config = [] 343 global_config.append('') 344 target_config.append('PR = "0"') 345 global_config.append('') 346 target_config.append('PR = "1"') 347 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 348 349 def test_sstate_cache_management_script_using_pr_3(self): 350 global_config = [] 351 target_config = [] 352 global_config.append('MACHINE = "qemux86-64"') 353 target_config.append('PR = "0"') 354 global_config.append(global_config[0]) 355 target_config.append('PR = "1"') 356 global_config.append('MACHINE = "qemux86"') 357 target_config.append('PR = "1"') 358 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 359 360 def test_sstate_cache_management_script_using_machine(self): 361 global_config = [] 362 target_config = [] 363 global_config.append('MACHINE = "qemux86-64"') 364 target_config.append('') 365 global_config.append('MACHINE = "qemux86"') 366 target_config.append('') 367 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 368 369class SStateHashSameSigs(SStateBase): 370 def test_sstate_32_64_same_hash(self): 371 """ 372 The sstate checksums for both native and target should not vary whether 373 they're built on a 32 or 64 bit system. Rather than requiring two different 374 build machines and running a builds, override the variables calling uname() 375 manually and check using bitbake -S. 376 """ 377 378 self.write_config(""" 379MACHINE = "qemux86" 380TMPDIR = "${TOPDIR}/tmp-sstatesamehash" 381TCLIBCAPPEND = "" 382BUILD_ARCH = "x86_64" 383BUILD_OS = "linux" 384SDKMACHINE = "x86_64" 385PACKAGE_CLASSES = "package_rpm package_ipk package_deb" 386BB_SIGNATURE_HANDLER = "OEBasicHash" 387""") 388 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 389 bitbake("core-image-weston -S none") 390 self.write_config(""" 391MACHINE = "qemux86" 392TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" 393TCLIBCAPPEND = "" 394BUILD_ARCH = "i686" 395BUILD_OS = "linux" 396SDKMACHINE = "i686" 397PACKAGE_CLASSES = "package_rpm package_ipk package_deb" 398BB_SIGNATURE_HANDLER = "OEBasicHash" 399""") 400 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 401 bitbake("core-image-weston -S none") 402 403 def get_files(d): 404 f = [] 405 for root, dirs, files in os.walk(d): 406 if "core-image-weston" in root: 407 # SDKMACHINE changing will change 408 # do_rootfs/do_testimage/do_build stamps of images which 409 # is safe to ignore. 410 continue 411 f.extend(os.path.join(root, name) for name in files) 412 return f 413 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") 414 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") 415 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash").replace("i686-linux", "x86_64-linux").replace("i686" + self.target_vendor + "-linux", "x86_64" + self.target_vendor + "-linux", ) for x in files2] 416 self.maxDiff = None 417 self.assertCountEqual(files1, files2) 418 419 420 def test_sstate_nativelsbstring_same_hash(self): 421 """ 422 The sstate checksums should be independent of whichever NATIVELSBSTRING is 423 detected. Rather than requiring two different build machines and running 424 builds, override the variables manually and check using bitbake -S. 425 """ 426 427 self.write_config(""" 428TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 429TCLIBCAPPEND = \"\" 430NATIVELSBSTRING = \"DistroA\" 431BB_SIGNATURE_HANDLER = "OEBasicHash" 432""") 433 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 434 bitbake("core-image-weston -S none") 435 self.write_config(""" 436TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 437TCLIBCAPPEND = \"\" 438NATIVELSBSTRING = \"DistroB\" 439BB_SIGNATURE_HANDLER = "OEBasicHash" 440""") 441 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 442 bitbake("core-image-weston -S none") 443 444 def get_files(d): 445 f = [] 446 for root, dirs, files in os.walk(d): 447 f.extend(os.path.join(root, name) for name in files) 448 return f 449 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") 450 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") 451 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 452 self.maxDiff = None 453 self.assertCountEqual(files1, files2) 454 455class SStateHashSameSigs2(SStateBase): 456 def test_sstate_allarch_samesigs(self): 457 """ 458 The sstate checksums of allarch packages should be independent of whichever 459 MACHINE is set. Check this using bitbake -S. 460 Also, rather than duplicate the test, check nativesdk stamps are the same between 461 the two MACHINE values. 462 """ 463 464 configA = """ 465TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 466TCLIBCAPPEND = \"\" 467MACHINE = \"qemux86-64\" 468BB_SIGNATURE_HANDLER = "OEBasicHash" 469""" 470 #OLDEST_KERNEL is arch specific so set to a different value here for testing 471 configB = """ 472TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 473TCLIBCAPPEND = \"\" 474MACHINE = \"qemuarm\" 475OLDEST_KERNEL = \"3.3.0\" 476BB_SIGNATURE_HANDLER = "OEBasicHash" 477""" 478 self.sstate_common_samesigs(configA, configB, allarch=True) 479 480 def test_sstate_nativesdk_samesigs_multilib(self): 481 """ 482 check nativesdk stamps are the same between the two MACHINE values. 483 """ 484 485 configA = """ 486TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 487TCLIBCAPPEND = \"\" 488MACHINE = \"qemux86-64\" 489require conf/multilib.conf 490MULTILIBS = \"multilib:lib32\" 491DEFAULTTUNE:virtclass-multilib-lib32 = \"x86\" 492BB_SIGNATURE_HANDLER = "OEBasicHash" 493""" 494 configB = """ 495TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 496TCLIBCAPPEND = \"\" 497MACHINE = \"qemuarm\" 498require conf/multilib.conf 499MULTILIBS = \"\" 500BB_SIGNATURE_HANDLER = "OEBasicHash" 501""" 502 self.sstate_common_samesigs(configA, configB) 503 504class SStateHashSameSigs3(SStateBase): 505 def test_sstate_sametune_samesigs(self): 506 """ 507 The sstate checksums of two identical machines (using the same tune) should be the 508 same, apart from changes within the machine specific stamps directory. We use the 509 qemux86copy machine to test this. Also include multilibs in the test. 510 """ 511 512 self.write_config(""" 513TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 514TCLIBCAPPEND = \"\" 515MACHINE = \"qemux86\" 516require conf/multilib.conf 517MULTILIBS = "multilib:lib32" 518DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 519BB_SIGNATURE_HANDLER = "OEBasicHash" 520""") 521 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 522 bitbake("world meta-toolchain -S none") 523 self.write_config(""" 524TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 525TCLIBCAPPEND = \"\" 526MACHINE = \"qemux86copy\" 527require conf/multilib.conf 528MULTILIBS = "multilib:lib32" 529DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 530BB_SIGNATURE_HANDLER = "OEBasicHash" 531""") 532 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 533 bitbake("world meta-toolchain -S none") 534 535 def get_files(d): 536 f = [] 537 for root, dirs, files in os.walk(d): 538 for name in files: 539 if "meta-environment" in root or "cross-canadian" in root or 'meta-ide-support' in root: 540 continue 541 if "qemux86copy-" in root or "qemux86-" in root: 542 continue 543 if "do_build" not in name and "do_populate_sdk" not in name: 544 f.append(os.path.join(root, name)) 545 return f 546 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") 547 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") 548 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 549 self.maxDiff = None 550 self.assertCountEqual(files1, files2) 551 552 553 def test_sstate_multilib_or_not_native_samesigs(self): 554 """The sstate checksums of two native recipes (and their dependencies) 555 where the target is using multilib in one but not the other 556 should be the same. We use the qemux86copy machine to test 557 this. 558 """ 559 560 self.write_config(""" 561TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" 562TCLIBCAPPEND = \"\" 563MACHINE = \"qemux86\" 564require conf/multilib.conf 565MULTILIBS = "multilib:lib32" 566DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 567BB_SIGNATURE_HANDLER = "OEBasicHash" 568""") 569 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 570 bitbake("binutils-native -S none") 571 self.write_config(""" 572TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 573TCLIBCAPPEND = \"\" 574MACHINE = \"qemux86copy\" 575BB_SIGNATURE_HANDLER = "OEBasicHash" 576""") 577 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 578 bitbake("binutils-native -S none") 579 580 def get_files(d): 581 f = [] 582 for root, dirs, files in os.walk(d): 583 for name in files: 584 f.append(os.path.join(root, name)) 585 return f 586 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") 587 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") 588 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 589 self.maxDiff = None 590 self.assertCountEqual(files1, files2) 591 592class SStateHashSameSigs4(SStateBase): 593 def test_sstate_noop_samesigs(self): 594 """ 595 The sstate checksums of two builds with these variables changed or 596 classes inherits should be the same. 597 """ 598 599 self.write_config(""" 600TMPDIR = "${TOPDIR}/tmp-sstatesamehash" 601TCLIBCAPPEND = "" 602BB_NUMBER_THREADS = "${@oe.utils.cpu_count()}" 603PARALLEL_MAKE = "-j 1" 604DL_DIR = "${TOPDIR}/download1" 605TIME = "111111" 606DATE = "20161111" 607INHERIT:remove = "buildstats-summary buildhistory uninative" 608http_proxy = "" 609BB_SIGNATURE_HANDLER = "OEBasicHash" 610""") 611 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 612 self.track_for_cleanup(self.topdir + "/download1") 613 bitbake("world meta-toolchain -S none") 614 self.write_config(""" 615TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" 616TCLIBCAPPEND = "" 617BB_NUMBER_THREADS = "${@oe.utils.cpu_count()+1}" 618PARALLEL_MAKE = "-j 2" 619DL_DIR = "${TOPDIR}/download2" 620TIME = "222222" 621DATE = "20161212" 622# Always remove uninative as we're changing proxies 623INHERIT:remove = "uninative" 624INHERIT += "buildstats-summary buildhistory" 625http_proxy = "http://example.com/" 626BB_SIGNATURE_HANDLER = "OEBasicHash" 627""") 628 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 629 self.track_for_cleanup(self.topdir + "/download2") 630 bitbake("world meta-toolchain -S none") 631 632 def get_files(d): 633 f = {} 634 for root, dirs, files in os.walk(d): 635 for name in files: 636 name, shash = name.rsplit('.', 1) 637 # Extract just the machine and recipe name 638 base = os.sep.join(root.rsplit(os.sep, 2)[-2:] + [name]) 639 f[base] = shash 640 return f 641 642 def compare_sigfiles(files, files1, files2, compare=False): 643 for k in files: 644 if k in files1 and k in files2: 645 print("%s differs:" % k) 646 if compare: 647 sigdatafile1 = self.topdir + "/tmp-sstatesamehash/stamps/" + k + "." + files1[k] 648 sigdatafile2 = self.topdir + "/tmp-sstatesamehash2/stamps/" + k + "." + files2[k] 649 output = bb.siggen.compare_sigfiles(sigdatafile1, sigdatafile2) 650 if output: 651 print('\n'.join(output)) 652 elif k in files1 and k not in files2: 653 print("%s in files1" % k) 654 elif k not in files1 and k in files2: 655 print("%s in files2" % k) 656 else: 657 assert "shouldn't reach here" 658 659 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/") 660 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/") 661 # Remove items that are identical in both sets 662 for k,v in files1.items() & files2.items(): 663 del files1[k] 664 del files2[k] 665 if not files1 and not files2: 666 # No changes, so we're done 667 return 668 669 files = list(files1.keys() | files2.keys()) 670 # this is an expensive computation, thus just compare the first 'max_sigfiles_to_compare' k files 671 max_sigfiles_to_compare = 20 672 first, rest = files[:max_sigfiles_to_compare], files[max_sigfiles_to_compare:] 673 compare_sigfiles(first, files1, files2, compare=True) 674 compare_sigfiles(rest, files1, files2, compare=False) 675 676 self.fail("sstate hashes not identical.") 677 678 def test_sstate_movelayer_samesigs(self): 679 """ 680 The sstate checksums of two builds with the same oe-core layer in two 681 different locations should be the same. 682 """ 683 core_layer = os.path.join( 684 self.tc.td["COREBASE"], 'meta') 685 copy_layer_1 = self.topdir + "/meta-copy1/meta" 686 copy_layer_2 = self.topdir + "/meta-copy2/meta" 687 688 oe.path.copytree(core_layer, copy_layer_1) 689 os.symlink(os.path.dirname(core_layer) + "/scripts", self.topdir + "/meta-copy1/scripts") 690 self.write_config(""" 691TMPDIR = "${TOPDIR}/tmp-sstatesamehash" 692""") 693 bblayers_conf = 'BBLAYERS += "%s"\nBBLAYERS:remove = "%s"' % (copy_layer_1, core_layer) 694 self.write_bblayers_config(bblayers_conf) 695 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 696 bitbake("bash -S none") 697 698 oe.path.copytree(core_layer, copy_layer_2) 699 os.symlink(os.path.dirname(core_layer) + "/scripts", self.topdir + "/meta-copy2/scripts") 700 self.write_config(""" 701TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" 702""") 703 bblayers_conf = 'BBLAYERS += "%s"\nBBLAYERS:remove = "%s"' % (copy_layer_2, core_layer) 704 self.write_bblayers_config(bblayers_conf) 705 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 706 bitbake("bash -S none") 707 708 def get_files(d): 709 f = [] 710 for root, dirs, files in os.walk(d): 711 for name in files: 712 f.append(os.path.join(root, name)) 713 return f 714 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") 715 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") 716 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] 717 self.maxDiff = None 718 self.assertCountEqual(files1, files2) 719 720class SStateFindSiginfo(SStateBase): 721 def test_sstate_compare_sigfiles_and_find_siginfo(self): 722 """ 723 Test the functionality of the find_siginfo: basic function and callback in compare_sigfiles 724 """ 725 self.write_config(""" 726TMPDIR = \"${TOPDIR}/tmp-sstates-findsiginfo\" 727TCLIBCAPPEND = \"\" 728MACHINE = \"qemux86-64\" 729require conf/multilib.conf 730MULTILIBS = "multilib:lib32" 731DEFAULTTUNE:virtclass-multilib-lib32 = "x86" 732BB_SIGNATURE_HANDLER = "OEBasicHash" 733""") 734 self.track_for_cleanup(self.topdir + "/tmp-sstates-findsiginfo") 735 736 pns = ["binutils", "binutils-native", "lib32-binutils"] 737 target_configs = [ 738""" 739TMPVAL1 = "tmpval1" 740TMPVAL2 = "tmpval2" 741do_tmptask1() { 742 echo ${TMPVAL1} 743} 744do_tmptask2() { 745 echo ${TMPVAL2} 746} 747addtask do_tmptask1 748addtask tmptask2 before do_tmptask1 749""", 750""" 751TMPVAL3 = "tmpval3" 752TMPVAL4 = "tmpval4" 753do_tmptask1() { 754 echo ${TMPVAL3} 755} 756do_tmptask2() { 757 echo ${TMPVAL4} 758} 759addtask do_tmptask1 760addtask tmptask2 before do_tmptask1 761""" 762 ] 763 764 for target_config in target_configs: 765 self.write_recipeinc("binutils", target_config) 766 for pn in pns: 767 bitbake("%s -c do_tmptask1 -S none" % pn) 768 self.delete_recipeinc("binutils") 769 770 with bb.tinfoil.Tinfoil() as tinfoil: 771 tinfoil.prepare(config_only=True) 772 773 def find_siginfo(pn, taskname, sigs=None): 774 result = None 775 command_complete = False 776 tinfoil.set_event_mask(["bb.event.FindSigInfoResult", 777 "bb.command.CommandCompleted"]) 778 ret = tinfoil.run_command("findSigInfo", pn, taskname, sigs) 779 if ret: 780 while result is None or not command_complete: 781 event = tinfoil.wait_event(1) 782 if event: 783 if isinstance(event, bb.command.CommandCompleted): 784 command_complete = True 785 elif isinstance(event, bb.event.FindSigInfoResult): 786 result = event.result 787 return result 788 789 def recursecb(key, hash1, hash2): 790 nonlocal recursecb_count 791 recursecb_count += 1 792 hashes = [hash1, hash2] 793 hashfiles = find_siginfo(key, None, hashes) 794 self.assertCountEqual(hashes, hashfiles) 795 bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb) 796 797 for pn in pns: 798 recursecb_count = 0 799 matches = find_siginfo(pn, "do_tmptask1") 800 self.assertGreaterEqual(len(matches), 2) 801 latesthashes = sorted(matches.keys(), key=lambda h: matches[h]['time'])[-2:] 802 bb.siggen.compare_sigfiles(matches[latesthashes[-2]]['path'], matches[latesthashes[-1]]['path'], recursecb) 803 self.assertEqual(recursecb_count,1) 804 805class SStatePrintdiff(SStateBase): 806 def run_test_printdiff_changerecipe(self, target, change_recipe, change_bbtask, change_content, expected_sametmp_output, expected_difftmp_output): 807 import time 808 self.write_config(""" 809TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-sametmp-{}" 810""".format(time.time())) 811 # Use runall do_build to ensure any indirect sstate is created, e.g. tzcode-native on both x86 and 812 # aarch64 hosts since only allarch target recipes depend upon it and it may not be built otherwise. 813 # A bitbake -c cleansstate tzcode-native would cause some of these tests to error for example. 814 bitbake("--runall build --runall deploy_source_date_epoch {}".format(target)) 815 bitbake("-S none {}".format(target)) 816 bitbake(change_bbtask) 817 self.write_recipeinc(change_recipe, change_content) 818 result_sametmp = bitbake("-S printdiff {}".format(target)) 819 820 self.write_config(""" 821TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-difftmp-{}" 822""".format(time.time())) 823 result_difftmp = bitbake("-S printdiff {}".format(target)) 824 825 self.delete_recipeinc(change_recipe) 826 for item in expected_sametmp_output: 827 self.assertIn(item, result_sametmp.output, msg = "Item {} not found in output:\n{}".format(item, result_sametmp.output)) 828 for item in expected_difftmp_output: 829 self.assertIn(item, result_difftmp.output, msg = "Item {} not found in output:\n{}".format(item, result_difftmp.output)) 830 831 def run_test_printdiff_changeconfig(self, target, change_bbtasks, change_content, expected_sametmp_output, expected_difftmp_output): 832 import time 833 self.write_config(""" 834TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-sametmp-{}" 835""".format(time.time())) 836 bitbake("--runall build --runall deploy_source_date_epoch {}".format(target)) 837 bitbake("-S none {}".format(target)) 838 bitbake(" ".join(change_bbtasks)) 839 self.append_config(change_content) 840 result_sametmp = bitbake("-S printdiff {}".format(target)) 841 842 self.write_config(""" 843TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-difftmp-{}" 844""".format(time.time())) 845 self.append_config(change_content) 846 result_difftmp = bitbake("-S printdiff {}".format(target)) 847 848 for item in expected_sametmp_output: 849 self.assertIn(item, result_sametmp.output, msg = "Item {} not found in output:\n{}".format(item, result_sametmp.output)) 850 for item in expected_difftmp_output: 851 self.assertIn(item, result_difftmp.output, msg = "Item {} not found in output:\n{}".format(item, result_difftmp.output)) 852 853 854 # Check if printdiff walks the full dependency chain from the image target to where the change is in a specific recipe 855 def test_image_minimal_vs_perlcross(self): 856 expected_output = ("Task perlcross-native:do_install couldn't be used from the cache because:", 857"We need hash", 858"most recent matching task was") 859 expected_sametmp_output = expected_output + ( 860"Variable do_install value changed", 861'+ echo "this changes the task signature"') 862 expected_difftmp_output = expected_output 863 864 self.run_test_printdiff_changerecipe("core-image-minimal", "perlcross", "-c do_install perlcross-native", 865""" 866do_install:append() { 867 echo "this changes the task signature" 868} 869""", 870expected_sametmp_output, expected_difftmp_output) 871 872 # Check if changes to gcc-source (which uses tmp/work-shared) are correctly discovered 873 def test_gcc_runtime_vs_gcc_source(self): 874 gcc_source_pn = 'gcc-source-%s' % get_bb_vars(['PV'], 'gcc')['PV'] 875 876 expected_output = ("Task {}:do_preconfigure couldn't be used from the cache because:".format(gcc_source_pn), 877"We need hash", 878"most recent matching task was") 879 expected_sametmp_output = expected_output + ( 880"Variable do_preconfigure value changed", 881'+ print("this changes the task signature")') 882 expected_difftmp_output = expected_output 883 884 self.run_test_printdiff_changerecipe("gcc-runtime", "gcc-source", "-c do_preconfigure {}".format(gcc_source_pn), 885""" 886python do_preconfigure:append() { 887 print("this changes the task signature") 888} 889""", 890expected_sametmp_output, expected_difftmp_output) 891 892 # Check if changing a really base task definiton is reported against multiple core recipes using it 893 def test_image_minimal_vs_base_do_configure(self): 894 change_bbtasks = ('zstd-native:do_configure', 895'texinfo-dummy-native:do_configure', 896'ldconfig-native:do_configure', 897'gettext-minimal-native:do_configure', 898'tzcode-native:do_configure', 899'makedevs-native:do_configure', 900'pigz-native:do_configure', 901'update-rc.d-native:do_configure', 902'unzip-native:do_configure', 903'gnu-config-native:do_configure') 904 905 expected_output = ["Task {} couldn't be used from the cache because:".format(t) for t in change_bbtasks] + [ 906"We need hash", 907"most recent matching task was"] 908 909 expected_sametmp_output = expected_output + [ 910"Variable base_do_configure value changed", 911'+ echo "this changes base_do_configure() definiton "'] 912 expected_difftmp_output = expected_output 913 914 self.run_test_printdiff_changeconfig("core-image-minimal",change_bbtasks, 915""" 916INHERIT += "base-do-configure-modified" 917""", 918expected_sametmp_output, expected_difftmp_output) 919 920@OETestTag("yocto-mirrors") 921class SStateMirrors(SStateBase): 922 def check_bb_output(self, output, exceptions, check_cdn): 923 def is_exception(object, exceptions): 924 for e in exceptions: 925 if re.search(e, object): 926 return True 927 return False 928 929 output_l = output.splitlines() 930 for l in output_l: 931 if l.startswith("Sstate summary"): 932 for idx, item in enumerate(l.split()): 933 if item == 'Missed': 934 missing_objects = int(l.split()[idx+1]) 935 break 936 else: 937 self.fail("Did not find missing objects amount in sstate summary: {}".format(l)) 938 break 939 else: 940 self.fail("Did not find 'Sstate summary' line in bitbake output") 941 942 failed_urls = [] 943 failed_urls_extrainfo = [] 944 for l in output_l: 945 if "SState: Unsuccessful fetch test for" in l and check_cdn: 946 missing_object = l.split()[6] 947 elif "SState: Looked for but didn't find file" in l and not check_cdn: 948 missing_object = l.split()[8] 949 else: 950 missing_object = None 951 if missing_object: 952 if not is_exception(missing_object, exceptions): 953 failed_urls.append(missing_object) 954 else: 955 missing_objects -= 1 956 957 if "urlopen failed for" in l and not is_exception(l, exceptions): 958 failed_urls_extrainfo.append(l) 959 960 self.assertEqual(len(failed_urls), missing_objects, "Amount of reported missing objects does not match failed URLs: {}\nFailed URLs:\n{}\nFetcher diagnostics:\n{}".format(missing_objects, "\n".join(failed_urls), "\n".join(failed_urls_extrainfo))) 961 self.assertEqual(len(failed_urls), 0, "Missing objects in the cache:\n{}\nFetcher diagnostics:\n{}".format("\n".join(failed_urls), "\n".join(failed_urls_extrainfo))) 962 963 def run_test(self, machine, targets, exceptions, check_cdn = True, ignore_errors = False): 964 # sstate is checked for existence of these, but they never get written out to begin with 965 exceptions += ["{}.*image_qa".format(t) for t in targets.split()] 966 exceptions += ["{}.*deploy_source_date_epoch".format(t) for t in targets.split()] 967 exceptions += ["{}.*image_complete".format(t) for t in targets.split()] 968 exceptions += ["linux-yocto.*shared_workdir"] 969 # these get influnced by IMAGE_FSTYPES tweaks in yocto-autobuilder-helper's config.json (on x86-64) 970 # additionally, they depend on noexec (thus, absent stamps) package, install, etc. image tasks, 971 # which makes tracing other changes difficult 972 exceptions += ["{}.*create_spdx".format(t) for t in targets.split()] 973 exceptions += ["{}.*create_runtime_spdx".format(t) for t in targets.split()] 974 975 if check_cdn: 976 self.config_sstate(True) 977 self.append_config(""" 978MACHINE = "{}" 979BB_HASHSERVE_UPSTREAM = "hashserv.yocto.io:8687" 980SSTATE_MIRRORS ?= "file://.* http://cdn.jsdelivr.net/yocto/sstate/all/PATH;downloadfilename=PATH" 981""".format(machine)) 982 else: 983 self.append_config(""" 984MACHINE = "{}" 985""".format(machine)) 986 result = bitbake("-DD -n {}".format(targets)) 987 bitbake("-S none {}".format(targets)) 988 if ignore_errors: 989 return 990 self.check_bb_output(result.output, exceptions, check_cdn) 991 992 def test_cdn_mirror_qemux86_64(self): 993 exceptions = [] 994 self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, ignore_errors = True) 995 self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions) 996 997 def test_cdn_mirror_qemuarm64(self): 998 exceptions = [] 999 self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, ignore_errors = True) 1000 self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions) 1001 1002 def test_local_cache_qemux86_64(self): 1003 exceptions = [] 1004 self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False) 1005 1006 def test_local_cache_qemuarm64(self): 1007 exceptions = [] 1008 self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False) 1009