1#
2# SPDX-License-Identifier: MIT
3#
4# Copyright 2019-2020 by Garmin Ltd. or its subsidiaries
5
6from oeqa.selftest.case import OESelftestTestCase
7from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
8import bb.utils
9import functools
10import multiprocessing
11import textwrap
12import tempfile
13import shutil
14import stat
15import os
16import datetime
17
18exclude_packages = [
19	]
20
21def is_excluded(package):
22    package_name = os.path.basename(package)
23    for i in exclude_packages:
24        if package_name.startswith(i):
25            return i
26    return None
27
28MISSING = 'MISSING'
29DIFFERENT = 'DIFFERENT'
30SAME = 'SAME'
31
32@functools.total_ordering
33class CompareResult(object):
34    def __init__(self):
35        self.reference = None
36        self.test = None
37        self.status = 'UNKNOWN'
38
39    def __eq__(self, other):
40        return (self.status, self.test) == (other.status, other.test)
41
42    def __lt__(self, other):
43        return (self.status, self.test) < (other.status, other.test)
44
45class PackageCompareResults(object):
46    def __init__(self, exclusions):
47        self.total = []
48        self.missing = []
49        self.different = []
50        self.different_excluded = []
51        self.same = []
52        self.active_exclusions = set()
53        exclude_packages.extend((exclusions or "").split())
54
55    def add_result(self, r):
56        self.total.append(r)
57        if r.status == MISSING:
58            self.missing.append(r)
59        elif r.status == DIFFERENT:
60            exclusion = is_excluded(r.reference)
61            if exclusion:
62                self.different_excluded.append(r)
63                self.active_exclusions.add(exclusion)
64            else:
65                self.different.append(r)
66        else:
67            self.same.append(r)
68
69    def sort(self):
70        self.total.sort()
71        self.missing.sort()
72        self.different.sort()
73        self.different_excluded.sort()
74        self.same.sort()
75
76    def __str__(self):
77        return 'same=%i different=%i different_excluded=%i missing=%i total=%i\nunused_exclusions=%s' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total), self.unused_exclusions())
78
79    def unused_exclusions(self):
80        return sorted(set(exclude_packages) - self.active_exclusions)
81
82def compare_file(reference, test, diffutils_sysroot):
83    result = CompareResult()
84    result.reference = reference
85    result.test = test
86
87    if not os.path.exists(reference):
88        result.status = MISSING
89        return result
90
91    r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True, sync=False)
92
93    if r.status:
94        result.status = DIFFERENT
95        return result
96
97    result.status = SAME
98    return result
99
100def run_diffoscope(a_dir, b_dir, html_dir, max_report_size=0, **kwargs):
101    return runCmd(['diffoscope', '--no-default-limits', '--max-report-size', str(max_report_size),
102                   '--exclude-directory-metadata', 'yes', '--html-dir', html_dir, a_dir, b_dir],
103                **kwargs)
104
105class DiffoscopeTests(OESelftestTestCase):
106    diffoscope_test_files = os.path.join(os.path.dirname(os.path.abspath(__file__)), "diffoscope")
107
108    def test_diffoscope(self):
109        bitbake("diffoscope-native -c addto_recipe_sysroot")
110        diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native")
111
112        # Check that diffoscope doesn't return an error when the files compare
113        # the same (a general check that diffoscope is working)
114        with tempfile.TemporaryDirectory() as tmpdir:
115            run_diffoscope('A', 'A', tmpdir,
116                native_sysroot=diffoscope_sysroot, cwd=self.diffoscope_test_files)
117
118        # Check that diffoscope generates an index.html file when the files are
119        # different
120        with tempfile.TemporaryDirectory() as tmpdir:
121            r = run_diffoscope('A', 'B', tmpdir,
122                native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=self.diffoscope_test_files)
123
124            self.assertNotEqual(r.status, 0, msg="diffoscope was successful when an error was expected")
125            self.assertTrue(os.path.exists(os.path.join(tmpdir, 'index.html')), "HTML index not found!")
126
127class ReproducibleTests(OESelftestTestCase):
128    # Test the reproducibility of whatever is built between sstate_targets and targets
129
130    package_classes = ['deb', 'ipk', 'rpm']
131
132    # Maximum report size, in bytes
133    max_report_size = 250 * 1024 * 1024
134
135    # targets are the things we want to test the reproducibility of
136    # Have to add the virtual targets manually for now as builds may or may not include them as they're exclude from world
137    targets = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'core-image-weston', 'world', 'virtual/librpc', 'virtual/libsdl2', 'virtual/crypt']
138
139    # sstate targets are things to pull from sstate to potentially cut build/debugging time
140    sstate_targets = []
141
142    save_results = False
143    if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ:
144        save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT']
145
146    # This variable controls if one of the test builds is allowed to pull from
147    # an sstate cache/mirror. The other build is always done clean as a point of
148    # comparison.
149    # If you know that your sstate archives are reproducible, enabling this
150    # will test that and also make the test run faster. If your sstate is not
151    # reproducible, disable this in your derived test class
152    build_from_sstate = True
153
154    def setUpLocal(self):
155        super().setUpLocal()
156        needed_vars = [
157            'TOPDIR',
158            'TARGET_PREFIX',
159            'BB_NUMBER_THREADS',
160            'BB_HASHSERVE',
161            'OEQA_REPRODUCIBLE_TEST_PACKAGE',
162            'OEQA_REPRODUCIBLE_TEST_TARGET',
163            'OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS',
164            'OEQA_REPRODUCIBLE_EXCLUDED_PACKAGES',
165        ]
166        bb_vars = get_bb_vars(needed_vars)
167        for v in needed_vars:
168            setattr(self, v.lower(), bb_vars[v])
169
170        if bb_vars['OEQA_REPRODUCIBLE_TEST_PACKAGE']:
171            self.package_classes = bb_vars['OEQA_REPRODUCIBLE_TEST_PACKAGE'].split()
172
173        if bb_vars['OEQA_REPRODUCIBLE_TEST_TARGET']:
174            self.targets = bb_vars['OEQA_REPRODUCIBLE_TEST_TARGET'].split()
175
176        if bb_vars['OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS']:
177            self.sstate_targets = bb_vars['OEQA_REPRODUCIBLE_TEST_SSTATE_TARGETS'].split()
178
179        self.extraresults = {}
180        self.extraresults.setdefault('reproducible', {}).setdefault('files', {})
181
182    def compare_packages(self, reference_dir, test_dir, diffutils_sysroot):
183        result = PackageCompareResults(self.oeqa_reproducible_excluded_packages)
184
185        old_cwd = os.getcwd()
186        try:
187            file_result = {}
188            os.chdir(test_dir)
189            with multiprocessing.Pool(processes=int(self.bb_number_threads or 0)) as p:
190                for root, dirs, files in os.walk('.'):
191                    async_result = []
192                    for f in files:
193                        reference_path = os.path.join(reference_dir, root, f)
194                        test_path = os.path.join(test_dir, root, f)
195                        async_result.append(p.apply_async(compare_file, (reference_path, test_path, diffutils_sysroot)))
196
197                    for a in async_result:
198                        result.add_result(a.get())
199
200        finally:
201            os.chdir(old_cwd)
202
203        result.sort()
204        return result
205
206    def write_package_list(self, package_class, name, packages):
207        self.extraresults['reproducible']['files'].setdefault(package_class, {})[name] = [
208                p.reference.split("/./")[1] for p in packages]
209
210    def copy_file(self, source, dest):
211        bb.utils.mkdirhier(os.path.dirname(dest))
212        shutil.copyfile(source, dest)
213
214    def do_test_build(self, name, use_sstate):
215        capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes]
216
217        tmpdir = os.path.join(self.topdir, name, 'tmp')
218        if os.path.exists(tmpdir):
219            bb.utils.remove(tmpdir, recurse=True)
220        config = textwrap.dedent('''\
221            PACKAGE_CLASSES = "{package_classes}"
222            TMPDIR = "{tmpdir}"
223            LICENSE_FLAGS_ACCEPTED = "commercial"
224            DISTRO_FEATURES:append = ' pam'
225            USERADDEXTENSION = "useradd-staticids"
226            USERADD_ERROR_DYNAMIC = "skip"
227            USERADD_UID_TABLES += "files/static-passwd"
228            USERADD_GID_TABLES += "files/static-group"
229            ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes),
230                        tmpdir=tmpdir)
231
232        # Export BB_CONSOLELOG to the calling function and make it constant to
233        # avoid a case where bitbake would get a timestamp-based filename but
234        # oe-selftest would, later, get another.
235        capture_vars.append("BB_CONSOLELOG")
236        config += 'BB_CONSOLELOG = "${LOG_DIR}/cooker/${MACHINE}/console.log"\n'
237
238        # We want different log files for each build, but a persistent bitbake
239        # may reuse the previous log file so restart the bitbake server.
240        bitbake("--kill-server")
241
242        bitbake_failure_count = 0
243        if not use_sstate:
244            if self.sstate_targets:
245               self.logger.info("Building prebuild for %s (sstate allowed)..." % (name))
246               self.write_config(config)
247               try:
248                   bitbake("--continue "+' '.join(self.sstate_targets), limit_exc_output=20)
249               except AssertionError as e:
250                   bitbake_failure_count += 1
251                   self.logger.error("Bitbake failed! but keep going... Log:")
252                   for line in str(e).split("\n"):
253                       self.logger.info("    "+line)
254
255            # This config fragment will disable using shared and the sstate
256            # mirror, forcing a complete build from scratch
257            config += textwrap.dedent('''\
258                SSTATE_DIR = "${TMPDIR}/sstate"
259                SSTATE_MIRRORS = "file://.*/.*-native.*  http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH file://.*/.*-cross.*  http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH"
260                ''')
261
262        self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT'))
263        self.write_config(config)
264        d = get_bb_vars(capture_vars)
265        # targets used to be called images
266        try:
267            bitbake("--continue "+' '.join(getattr(self, 'images', self.targets)), limit_exc_output=20)
268        except AssertionError as e:
269            bitbake_failure_count += 1
270
271            self.logger.error("Bitbake failed! but keep going... Log:")
272            for line in str(e).split("\n"):
273                self.logger.info("    "+line)
274
275            # The calling function expects the existence of the deploy
276            # directories containing the packages.
277            # If bitbake failed to create them, do it manually
278            for c in self.package_classes:
279                deploy = d['DEPLOY_DIR_' + c.upper()]
280                if not os.path.exists(deploy):
281                    self.logger.info("Manually creating %s" % deploy)
282                    bb.utils.mkdirhier(deploy)
283
284        return (d, bitbake_failure_count)
285
286    def test_reproducible_builds(self):
287        def strip_topdir(s):
288            if s.startswith(self.topdir):
289                return s[len(self.topdir):]
290            return s
291
292        # Build native utilities
293        self.write_config('')
294        bitbake("diffoscope-native diffutils-native jquery-native -c addto_recipe_sysroot")
295        diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native")
296        diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native")
297        jquery_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "jquery-native")
298
299        if self.save_results:
300            os.makedirs(self.save_results, exist_ok=True)
301            datestr = datetime.datetime.now().strftime('%Y%m%d')
302            save_dir = tempfile.mkdtemp(prefix='oe-reproducible-%s-' % datestr, dir=self.save_results)
303            os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
304            self.logger.info('Non-reproducible packages will be copied to %s', save_dir)
305
306        # The below bug shows that a few reproducible issues are depends on build dir path length.
307        # https://bugzilla.yoctoproject.org/show_bug.cgi?id=15554
308        # So, the reproducibleA & reproducibleB directories are changed to reproducibleA & reproducibleB-extended to have different size.
309
310        fails = []
311        vars_list = [None, None]
312
313        for i, (name, use_sstate) in enumerate(
314                                 (('reproducibleA', self.build_from_sstate),
315                                 ('reproducibleB-extended', False))):
316            (variables, bitbake_failure_count) = self.do_test_build(name, use_sstate)
317            if bitbake_failure_count > 0:
318                self.logger.error('%s build failed. Trying to compute built packages differences but the test will fail.' % name)
319                fails.append("Bitbake %s failure" % name)
320                if self.save_results:
321                    self.copy_file(variables["BB_CONSOLELOG"], os.path.join(save_dir, "bitbake-%s.log" % name))
322            vars_list[i] = variables
323
324        vars_A, vars_B = vars_list
325        # NOTE: The temp directories from the reproducible build are purposely
326        # kept after the build so it can be diffed for debugging.
327
328        for c in self.package_classes:
329            with self.subTest(package_class=c):
330                package_class = 'package_' + c
331
332                deploy_A = vars_A['DEPLOY_DIR_' + c.upper()]
333                deploy_B = vars_B['DEPLOY_DIR_' + c.upper()]
334
335                self.logger.info('Checking %s packages for differences...' % c)
336                result = self.compare_packages(deploy_A, deploy_B, diffutils_sysroot)
337
338                self.logger.info('Reproducibility summary for %s: %s' % (c, result))
339
340                self.write_package_list(package_class, 'missing', result.missing)
341                self.write_package_list(package_class, 'different', result.different)
342                self.write_package_list(package_class, 'different_excluded', result.different_excluded)
343                self.write_package_list(package_class, 'same', result.same)
344
345                if self.save_results:
346                    for d in result.different:
347                        self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)]))
348                        self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)]))
349
350                    for d in result.different_excluded:
351                        self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)]))
352                        self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)]))
353
354                if result.different:
355                    fails.append("The following %s packages are different and not in exclusion list:\n%s" %
356                            (c, '\n'.join(r.test for r in (result.different))))
357
358                if result.missing and len(self.sstate_targets) == 0:
359                    fails.append("The following %s packages are missing and not in exclusion list:\n%s" %
360                            (c, '\n'.join(r.test for r in (result.missing))))
361
362        # Clean up empty directories
363        if self.save_results:
364            if not os.listdir(save_dir):
365                os.rmdir(save_dir)
366            else:
367                self.logger.info('Running diffoscope')
368                package_dir = os.path.join(save_dir, 'packages')
369                package_html_dir = os.path.join(package_dir, 'diff-html')
370
371                # Copy jquery to improve the diffoscope output usability
372                self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js'))
373
374                run_diffoscope('reproducibleA', 'reproducibleB-extended', package_html_dir, max_report_size=self.max_report_size,
375                        native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir)
376
377        if fails:
378            self.fail('\n'.join(fails))
379
380