1# 2# SPDX-License-Identifier: MIT 3# 4# Copyright 2019-2020 by Garmin Ltd. or its subsidiaries 5 6from oeqa.selftest.case import OESelftestTestCase 7from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars 8import bb.utils 9import functools 10import multiprocessing 11import textwrap 12import tempfile 13import shutil 14import stat 15import os 16import datetime 17 18exclude_packages = [ 19 ] 20 21def is_excluded(package): 22 package_name = os.path.basename(package) 23 for i in exclude_packages: 24 if package_name.startswith(i): 25 return i 26 return None 27 28MISSING = 'MISSING' 29DIFFERENT = 'DIFFERENT' 30SAME = 'SAME' 31 32@functools.total_ordering 33class CompareResult(object): 34 def __init__(self): 35 self.reference = None 36 self.test = None 37 self.status = 'UNKNOWN' 38 39 def __eq__(self, other): 40 return (self.status, self.test) == (other.status, other.test) 41 42 def __lt__(self, other): 43 return (self.status, self.test) < (other.status, other.test) 44 45class PackageCompareResults(object): 46 def __init__(self, exclusions): 47 self.total = [] 48 self.missing = [] 49 self.different = [] 50 self.different_excluded = [] 51 self.same = [] 52 self.active_exclusions = set() 53 exclude_packages.extend((exclusions or "").split()) 54 55 def add_result(self, r): 56 self.total.append(r) 57 if r.status == MISSING: 58 self.missing.append(r) 59 elif r.status == DIFFERENT: 60 exclusion = is_excluded(r.reference) 61 if exclusion: 62 self.different_excluded.append(r) 63 self.active_exclusions.add(exclusion) 64 else: 65 self.different.append(r) 66 else: 67 self.same.append(r) 68 69 def sort(self): 70 self.total.sort() 71 self.missing.sort() 72 self.different.sort() 73 self.different_excluded.sort() 74 self.same.sort() 75 76 def __str__(self): 77 return 'same=%i different=%i different_excluded=%i missing=%i total=%i\nunused_exclusions=%s' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total), self.unused_exclusions()) 78 79 def unused_exclusions(self): 80 return sorted(set(exclude_packages) - self.active_exclusions) 81 82def compare_file(reference, test, diffutils_sysroot): 83 result = CompareResult() 84 result.reference = reference 85 result.test = test 86 87 if not os.path.exists(reference): 88 result.status = MISSING 89 return result 90 91 r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True, sync=False) 92 93 if r.status: 94 result.status = DIFFERENT 95 return result 96 97 result.status = SAME 98 return result 99 100def run_diffoscope(a_dir, b_dir, html_dir, max_report_size=0, max_diff_block_lines=1024, max_diff_block_lines_saved=0, **kwargs): 101 return runCmd(['diffoscope', '--no-default-limits', '--max-report-size', str(max_report_size), 102 '--max-diff-block-lines-saved', str(max_diff_block_lines_saved), 103 '--max-diff-block-lines', str(max_diff_block_lines), 104 '--exclude-directory-metadata', 'yes', '--html-dir', html_dir, a_dir, b_dir], 105 **kwargs) 106 107class DiffoscopeTests(OESelftestTestCase): 108 diffoscope_test_files = os.path.join(os.path.dirname(os.path.abspath(__file__)), "diffoscope") 109 110 def test_diffoscope(self): 111 bitbake("diffoscope-native -c addto_recipe_sysroot") 112 diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native") 113 114 # Check that diffoscope doesn't return an error when the files compare 115 # the same (a general check that diffoscope is working) 116 with tempfile.TemporaryDirectory() as tmpdir: 117 run_diffoscope('A', 'A', tmpdir, 118 native_sysroot=diffoscope_sysroot, cwd=self.diffoscope_test_files) 119 120 # Check that diffoscope generates an index.html file when the files are 121 # different 122 with tempfile.TemporaryDirectory() as tmpdir: 123 r = run_diffoscope('A', 'B', tmpdir, 124 native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=self.diffoscope_test_files) 125 126 self.assertNotEqual(r.status, 0, msg="diffoscope was successful when an error was expected") 127 self.assertTrue(os.path.exists(os.path.join(tmpdir, 'index.html')), "HTML index not found!") 128 129class ReproducibleTests(OESelftestTestCase): 130 # Test the reproducibility of whatever is built between sstate_targets and targets 131 132 package_classes = ['deb', 'ipk', 'rpm'] 133 134 # Maximum report size, in bytes 135 max_report_size = 250 * 1024 * 1024 136 137 # Maximum diff blocks size, in lines 138 max_diff_block_lines = 1024 139 # Maximum diff blocks size (saved in memory), in lines 140 max_diff_block_lines_saved = max_diff_block_lines 141 142 # targets are the things we want to test the reproducibility of 143 # Have to add the virtual targets manually for now as builds may or may not include them as they're exclude from world 144 targets = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'core-image-weston', 'world', 'virtual/librpc', 'virtual/libsdl2', 'virtual/crypt'] 145 146 # sstate targets are things to pull from sstate to potentially cut build/debugging time 147 sstate_targets = [] 148 149 save_results = False 150 if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ: 151 save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT'] 152 153 # This variable controls if one of the test builds is allowed to pull from 154 # an sstate cache/mirror. The other build is always done clean as a point of 155 # comparison. 156 # If you know that your sstate archives are reproducible, enabling this 157 # will test that and also make the test run faster. If your sstate is not 158 # reproducible, disable this in your derived test class 159 build_from_sstate = True 160 161 def setUpLocal(self): 162 super().setUpLocal() 163 needed_vars = [ 164 'TOPDIR', 165 'TARGET_PREFIX', 166 'BB_NUMBER_THREADS', 167 'BB_HASHSERVE', 168 'OEQA_REPRODUCIBLE_TEST_PACKAGE', 169 'OEQA_REPRODUCIBLE_TEST_TARGET', 170 'OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS', 171 'OEQA_REPRODUCIBLE_EXCLUDED_PACKAGES', 172 'OEQA_REPRODUCIBLE_TEST_LEAF_TARGETS', 173 ] 174 bb_vars = get_bb_vars(needed_vars) 175 for v in needed_vars: 176 setattr(self, v.lower(), bb_vars[v]) 177 178 if bb_vars['OEQA_REPRODUCIBLE_TEST_PACKAGE']: 179 self.package_classes = bb_vars['OEQA_REPRODUCIBLE_TEST_PACKAGE'].split() 180 181 if bb_vars['OEQA_REPRODUCIBLE_TEST_TARGET'] or bb_vars['OEQA_REPRODUCIBLE_TEST_LEAF_TARGETS']: 182 self.targets = (bb_vars['OEQA_REPRODUCIBLE_TEST_TARGET'] or "").split() + (bb_vars['OEQA_REPRODUCIBLE_TEST_LEAF_TARGETS'] or "").split() 183 184 if bb_vars['OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS']: 185 self.sstate_targets = bb_vars['OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS'].split() 186 187 if bb_vars['OEQA_REPRODUCIBLE_TEST_LEAF_TARGETS']: 188 # Setup to build every DEPENDS of leaf recipes using sstate 189 for leaf_recipe in bb_vars['OEQA_REPRODUCIBLE_TEST_LEAF_TARGETS'].split(): 190 self.sstate_targets.extend(get_bb_var('DEPENDS', leaf_recipe).split()) 191 192 self.extraresults = {} 193 self.extraresults.setdefault('reproducible', {}).setdefault('files', {}) 194 195 def compare_packages(self, reference_dir, test_dir, diffutils_sysroot): 196 result = PackageCompareResults(self.oeqa_reproducible_excluded_packages) 197 198 old_cwd = os.getcwd() 199 try: 200 file_result = {} 201 os.chdir(test_dir) 202 with multiprocessing.Pool(processes=int(self.bb_number_threads or 0)) as p: 203 for root, dirs, files in os.walk('.'): 204 async_result = [] 205 for f in files: 206 reference_path = os.path.join(reference_dir, root, f) 207 test_path = os.path.join(test_dir, root, f) 208 async_result.append(p.apply_async(compare_file, (reference_path, test_path, diffutils_sysroot))) 209 210 for a in async_result: 211 result.add_result(a.get()) 212 213 finally: 214 os.chdir(old_cwd) 215 216 result.sort() 217 return result 218 219 def write_package_list(self, package_class, name, packages): 220 self.extraresults['reproducible']['files'].setdefault(package_class, {})[name] = [ 221 p.reference.split("/./")[1] for p in packages] 222 223 def copy_file(self, source, dest): 224 bb.utils.mkdirhier(os.path.dirname(dest)) 225 shutil.copyfile(source, dest) 226 227 def do_test_build(self, name, use_sstate): 228 capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes] 229 230 tmpdir = os.path.join(self.topdir, name, 'tmp') 231 if os.path.exists(tmpdir): 232 bb.utils.remove(tmpdir, recurse=True) 233 config = textwrap.dedent('''\ 234 PACKAGE_CLASSES = "{package_classes}" 235 TMPDIR = "{tmpdir}" 236 LICENSE_FLAGS_ACCEPTED = "commercial" 237 DISTRO_FEATURES:append = ' pam' 238 USERADDEXTENSION = "useradd-staticids" 239 USERADD_ERROR_DYNAMIC = "skip" 240 USERADD_UID_TABLES += "files/static-passwd" 241 USERADD_GID_TABLES += "files/static-group" 242 ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes), 243 tmpdir=tmpdir) 244 245 # Export BB_CONSOLELOG to the calling function and make it constant to 246 # avoid a case where bitbake would get a timestamp-based filename but 247 # oe-selftest would, later, get another. 248 capture_vars.append("BB_CONSOLELOG") 249 config += 'BB_CONSOLELOG = "${LOG_DIR}/cooker/${MACHINE}/console.log"\n' 250 251 # We want different log files for each build, but a persistent bitbake 252 # may reuse the previous log file so restart the bitbake server. 253 bitbake("--kill-server") 254 255 def print_condensed_error_log(logs, context_lines=10, tail_lines=20): 256 """Prints errors with context and the end of the log.""" 257 258 logs = logs.split("\n") 259 for i, line in enumerate(logs): 260 if line.startswith("ERROR"): 261 self.logger.info("Found ERROR (line %d):" % (i + 1)) 262 for l in logs[i-context_lines:i+context_lines]: 263 self.logger.info(" " + l) 264 265 self.logger.info("End of log:") 266 for l in logs[-tail_lines:]: 267 self.logger.info(" " + l) 268 269 bitbake_failure_count = 0 270 if not use_sstate: 271 if self.sstate_targets: 272 self.logger.info("Building prebuild for %s (sstate allowed)..." % (name)) 273 self.write_config(config) 274 try: 275 bitbake("--continue "+' '.join(self.sstate_targets)) 276 except AssertionError as e: 277 bitbake_failure_count += 1 278 self.logger.error("Bitbake failed! but keep going... Log:") 279 print_condensed_error_log(str(e)) 280 281 # This config fragment will disable using shared and the sstate 282 # mirror, forcing a complete build from scratch 283 config += textwrap.dedent('''\ 284 SSTATE_DIR = "${TMPDIR}/sstate" 285 SSTATE_MIRRORS = "file://.*/.*-native.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH file://.*/.*-cross.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH" 286 ''') 287 288 self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT')) 289 self.write_config(config) 290 d = get_bb_vars(capture_vars) 291 try: 292 # targets used to be called images 293 bitbake("--continue "+' '.join(getattr(self, 'images', self.targets))) 294 except AssertionError as e: 295 bitbake_failure_count += 1 296 self.logger.error("Bitbake failed! but keep going... Log:") 297 print_condensed_error_log(str(e)) 298 299 # The calling function expects the existence of the deploy 300 # directories containing the packages. 301 # If bitbake failed to create them, do it manually 302 for c in self.package_classes: 303 deploy = d['DEPLOY_DIR_' + c.upper()] 304 if not os.path.exists(deploy): 305 self.logger.info("Manually creating %s" % deploy) 306 bb.utils.mkdirhier(deploy) 307 308 return (d, bitbake_failure_count) 309 310 def test_reproducible_builds(self): 311 def strip_topdir(s): 312 if s.startswith(self.topdir): 313 return s[len(self.topdir):] 314 return s 315 316 # Build native utilities 317 self.write_config('') 318 bitbake("diffoscope-native diffutils-native jquery-native -c addto_recipe_sysroot") 319 diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native") 320 diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native") 321 jquery_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "jquery-native") 322 323 if self.save_results: 324 os.makedirs(self.save_results, exist_ok=True) 325 datestr = datetime.datetime.now().strftime('%Y%m%d') 326 save_dir = tempfile.mkdtemp(prefix='oe-reproducible-%s-' % datestr, dir=self.save_results) 327 os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) 328 self.logger.info('Non-reproducible packages will be copied to %s', save_dir) 329 330 # The below bug shows that a few reproducible issues are depends on build dir path length. 331 # https://bugzilla.yoctoproject.org/show_bug.cgi?id=15554 332 # So, the reproducibleA & reproducibleB directories are changed to reproducibleA & reproducibleB-extended to have different size. 333 334 fails = [] 335 vars_list = [None, None] 336 337 for i, (name, use_sstate) in enumerate( 338 (('reproducibleA', self.build_from_sstate), 339 ('reproducibleB-extended', False))): 340 (variables, bitbake_failure_count) = self.do_test_build(name, use_sstate) 341 if bitbake_failure_count > 0: 342 self.logger.error('%s build failed. Trying to compute built packages differences but the test will fail.' % name) 343 fails.append("Bitbake %s failure" % name) 344 if self.save_results: 345 failure_log_path = os.path.join(save_dir, "bitbake-%s.log" % name) 346 self.logger.info('Failure log for %s will be copied to %s'% (name, failure_log_path)) 347 self.copy_file(variables["BB_CONSOLELOG"], failure_log_path) 348 vars_list[i] = variables 349 350 vars_A, vars_B = vars_list 351 # NOTE: The temp directories from the reproducible build are purposely 352 # kept after the build so it can be diffed for debugging. 353 354 for c in self.package_classes: 355 with self.subTest(package_class=c): 356 package_class = 'package_' + c 357 358 deploy_A = vars_A['DEPLOY_DIR_' + c.upper()] 359 deploy_B = vars_B['DEPLOY_DIR_' + c.upper()] 360 361 self.logger.info('Checking %s packages for differences...' % c) 362 result = self.compare_packages(deploy_A, deploy_B, diffutils_sysroot) 363 364 self.logger.info('Reproducibility summary for %s: %s' % (c, result)) 365 366 self.write_package_list(package_class, 'missing', result.missing) 367 self.write_package_list(package_class, 'different', result.different) 368 self.write_package_list(package_class, 'different_excluded', result.different_excluded) 369 self.write_package_list(package_class, 'same', result.same) 370 371 if self.save_results: 372 for d in result.different: 373 self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)])) 374 self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)])) 375 376 for d in result.different_excluded: 377 self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)])) 378 self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)])) 379 380 if result.different: 381 fails.append("The following %s packages are different and not in exclusion list:\n%s" % 382 (c, '\n'.join(r.test for r in (result.different)))) 383 384 if result.missing and len(self.sstate_targets) == 0: 385 fails.append("The following %s packages are missing and not in exclusion list:\n%s" % 386 (c, '\n'.join(r.test for r in (result.missing)))) 387 388 # Clean up empty directories 389 if self.save_results: 390 if not os.listdir(save_dir): 391 os.rmdir(save_dir) 392 else: 393 self.logger.info('Running diffoscope') 394 package_dir = os.path.join(save_dir, 'packages') 395 package_html_dir = os.path.join(package_dir, 'diff-html') 396 397 # Copy jquery to improve the diffoscope output usability 398 self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js')) 399 400 run_diffoscope('reproducibleA', 'reproducibleB-extended', package_html_dir, max_report_size=self.max_report_size, 401 max_diff_block_lines_saved=self.max_diff_block_lines_saved, 402 max_diff_block_lines=self.max_diff_block_lines, 403 native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir) 404 405 if fails: 406 self.fail('\n'.join(fails)) 407 408