1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7import os
8import glob
9import re
10from oeqa.utils.commands import bitbake, get_bb_vars
11from oeqa.selftest.case import OESelftestTestCase
12
13class Archiver(OESelftestTestCase):
14
15    def test_archiver_allows_to_filter_on_recipe_name(self):
16        """
17        Summary:     The archiver should offer the possibility to filter on the recipe. (#6929)
18        Expected:    1. Included recipe (busybox) should be included
19                     2. Excluded recipe (zlib) should be excluded
20        Product:     oe-core
21        Author:      Daniel Istrate <daniel.alexandrux.istrate@intel.com>
22        AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
23        """
24
25        include_recipe = 'selftest-ed'
26        exclude_recipe = 'initscripts'
27
28        features = 'INHERIT += "archiver"\n'
29        features += 'ARCHIVER_MODE[src] = "original"\n'
30        features += 'COPYLEFT_PN_INCLUDE = "%s"\n' % include_recipe
31        features += 'COPYLEFT_PN_EXCLUDE = "%s"\n' % exclude_recipe
32        self.write_config(features)
33
34        bitbake('-c clean %s %s' % (include_recipe, exclude_recipe))
35        bitbake("-c deploy_archives %s %s" % (include_recipe, exclude_recipe))
36
37        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS'])
38        src_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['TARGET_SYS'])
39
40        # Check that include_recipe was included
41        included_present = len(glob.glob(src_path + '/%s-*/*' % include_recipe))
42        self.assertTrue(included_present, 'Recipe %s was not included.' % include_recipe)
43
44        # Check that exclude_recipe was excluded
45        excluded_present = len(glob.glob(src_path + '/%s-*/*' % exclude_recipe))
46        self.assertFalse(excluded_present, 'Recipe %s was not excluded.' % exclude_recipe)
47
48    def test_archiver_filters_by_type(self):
49        """
50        Summary:     The archiver is documented to filter on the recipe type.
51        Expected:    1. included recipe type (target) should be included
52                     2. other types should be excluded
53        Product:     oe-core
54        Author:      André Draszik <adraszik@tycoint.com>
55        """
56
57        target_recipe = 'selftest-ed'
58        native_recipe = 'selftest-ed-native'
59
60        features = 'INHERIT += "archiver"\n'
61        features += 'ARCHIVER_MODE[src] = "original"\n'
62        features += 'COPYLEFT_RECIPE_TYPES = "target"\n'
63        self.write_config(features)
64
65        bitbake('-c clean %s %s' % (target_recipe, native_recipe))
66        bitbake("%s -c deploy_archives %s" % (target_recipe, native_recipe))
67
68        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS', 'BUILD_SYS'])
69        src_path_target = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['TARGET_SYS'])
70        src_path_native = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['BUILD_SYS'])
71
72        # Check that target_recipe was included
73        included_present = len(glob.glob(src_path_target + '/%s-*/*' % target_recipe))
74        self.assertTrue(included_present, 'Recipe %s was not included.' % target_recipe)
75
76        # Check that native_recipe was excluded
77        excluded_present = len(glob.glob(src_path_native + '/%s-*/*' % native_recipe))
78        self.assertFalse(excluded_present, 'Recipe %s was not excluded.' % native_recipe)
79
80    def test_archiver_filters_by_type_and_name(self):
81        """
82        Summary:     Test that the archiver archives by recipe type, taking the
83                     recipe name into account.
84        Expected:    1. included recipe type (target) should be included
85                     2. other types should be excluded
86                     3. recipe by name should be included / excluded,
87                        overriding previous decision by type
88        Product:     oe-core
89        Author:      André Draszik <adraszik@tycoint.com>
90        """
91
92        target_recipes = [ 'initscripts', 'selftest-ed' ]
93        native_recipes = [ 'update-rc.d-native', 'selftest-ed-native' ]
94
95        features = 'INHERIT += "archiver"\n'
96        features += 'ARCHIVER_MODE[src] = "original"\n'
97        features += 'COPYLEFT_RECIPE_TYPES = "target"\n'
98        features += 'COPYLEFT_PN_INCLUDE = "%s"\n' % native_recipes[1]
99        features += 'COPYLEFT_PN_EXCLUDE = "%s"\n' % target_recipes[1]
100        self.write_config(features)
101
102        bitbake('-c clean %s %s' % (' '.join(target_recipes), ' '.join(native_recipes)))
103        bitbake('-c deploy_archives %s %s' % (' '.join(target_recipes), ' '.join(native_recipes)))
104
105        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS', 'BUILD_SYS'])
106        src_path_target = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['TARGET_SYS'])
107        src_path_native = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['BUILD_SYS'])
108
109        # Check that target_recipe[0] and native_recipes[1] were included
110        included_present = len(glob.glob(src_path_target + '/%s-*/*' % target_recipes[0]))
111        self.assertTrue(included_present, 'Recipe %s was not included.' % target_recipes[0])
112
113        included_present = len(glob.glob(src_path_native + '/%s-*/*' % native_recipes[1]))
114        self.assertTrue(included_present, 'Recipe %s was not included.' % native_recipes[1])
115
116        # Check that native_recipes[0] and target_recipes[1] were excluded
117        excluded_present = len(glob.glob(src_path_native + '/%s-*/*' % native_recipes[0]))
118        self.assertFalse(excluded_present, 'Recipe %s was not excluded.' % native_recipes[0])
119
120        excluded_present = len(glob.glob(src_path_target + '/%s-*/*' % target_recipes[1]))
121        self.assertFalse(excluded_present, 'Recipe %s was not excluded.' % target_recipes[1])
122
123    def test_archiver_multiconfig_shared_unpack_and_patch(self):
124        """
125        Test that shared recipes in original mode with diff enabled works in multiconfig,
126        otherwise it will not build when using the same TMP dir.
127        """
128
129        features = 'BBMULTICONFIG = "mc1 mc2"\n'
130        features += 'INHERIT += "archiver"\n'
131        features += 'ARCHIVER_MODE[src] = "original"\n'
132        features += 'ARCHIVER_MODE[diff] = "1"\n'
133        self.write_config(features)
134
135        # We can use any machine in multiconfig as long as they are different
136        self.write_config('MACHINE = "qemuarm"\n', 'mc1')
137        self.write_config('MACHINE = "qemux86"\n', 'mc2')
138
139        task = 'do_unpack_and_patch'
140        # Use gcc-source as it is a shared recipe (appends the pv to the pn)
141        pn = 'gcc-source-%s' % get_bb_vars(['PV'], 'gcc')['PV']
142
143        # Generate the tasks signatures
144        bitbake('mc:mc1:%s mc:mc2:%s -c %s -S lockedsigs' % (pn, pn, task))
145
146        # Check the tasks signatures
147        # To be machine agnostic the tasks needs to generate the same signature for each machine
148        locked_sigs_inc = "%s/locked-sigs.inc" % self.builddir
149        locked_sigs = open(locked_sigs_inc).read()
150        task_sigs = re.findall(r"%s:%s:.*" % (pn, task), locked_sigs)
151        uniq_sigs = set(task_sigs)
152        self.assertFalse(len(uniq_sigs) - 1, \
153            'The task "%s" of the recipe "%s" has different signatures in "%s" for each machine in multiconfig' \
154            % (task, pn, locked_sigs_inc))
155
156    def test_archiver_srpm_mode(self):
157        """
158        Test that in srpm mode, the added recipe dependencies at least exist/work [YOCTO #11121]
159        """
160
161        features = 'INHERIT += "archiver"\n'
162        features += 'ARCHIVER_MODE[srpm] = "1"\n'
163        features += 'PACKAGE_CLASSES = "package_rpm"\n'
164        self.write_config(features)
165
166        bitbake('-n selftest-nopackages selftest-ed')
167
168    def _test_archiver_mode(self, mode, target_file_name, extra_config=None):
169        target = 'selftest-ed-native'
170
171        features = 'INHERIT += "archiver"\n'
172        features +=  'ARCHIVER_MODE[src] = "%s"\n' % (mode)
173        if extra_config:
174            features += extra_config
175        self.write_config(features)
176
177        bitbake('-c clean %s' % (target))
178        bitbake('-c deploy_archives %s' % (target))
179
180        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'BUILD_SYS'])
181        glob_str = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['BUILD_SYS'], '%s-*' % (target))
182        glob_result = glob.glob(glob_str)
183        self.assertTrue(glob_result, 'Missing archiver directory for %s' % (target))
184
185        archive_path = os.path.join(glob_result[0], target_file_name)
186        self.assertTrue(os.path.exists(archive_path), 'Missing archive file %s' % (target_file_name))
187
188    def test_archiver_mode_original(self):
189        """
190        Test that the archiver works with `ARCHIVER_MODE[src] = "original"`.
191        """
192
193        self._test_archiver_mode('original', 'ed-1.14.1.tar.lz')
194
195    def test_archiver_mode_patched(self):
196        """
197        Test that the archiver works with `ARCHIVER_MODE[src] = "patched"`.
198        """
199
200        self._test_archiver_mode('patched', 'selftest-ed-native-1.14.1-r0-patched.tar.xz')
201
202    def test_archiver_mode_configured(self):
203        """
204        Test that the archiver works with `ARCHIVER_MODE[src] = "configured"`.
205        """
206
207        self._test_archiver_mode('configured', 'selftest-ed-native-1.14.1-r0-configured.tar.xz')
208
209    def test_archiver_mode_recipe(self):
210        """
211        Test that the archiver works with `ARCHIVER_MODE[recipe] = "1"`.
212        """
213
214        self._test_archiver_mode('patched', 'selftest-ed-native-1.14.1-r0-recipe.tar.xz',
215                                 'ARCHIVER_MODE[recipe] = "1"\n')
216
217    def test_archiver_mode_diff(self):
218        """
219        Test that the archiver works with `ARCHIVER_MODE[diff] = "1"`.
220        Exclusions controlled by `ARCHIVER_MODE[diff-exclude]` are not yet tested.
221        """
222
223        self._test_archiver_mode('patched', 'selftest-ed-native-1.14.1-r0-diff.gz',
224                                 'ARCHIVER_MODE[diff] = "1"\n')
225
226    def test_archiver_mode_dumpdata(self):
227        """
228        Test that the archiver works with `ARCHIVER_MODE[dumpdata] = "1"`.
229        """
230
231        self._test_archiver_mode('patched', 'selftest-ed-native-1.14.1-r0-showdata.dump',
232                                 'ARCHIVER_MODE[dumpdata] = "1"\n')
233
234    def test_archiver_mode_mirror(self):
235        """
236        Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"`.
237        """
238
239        self._test_archiver_mode('mirror', 'ed-1.14.1.tar.lz',
240                                 'BB_GENERATE_MIRROR_TARBALLS = "1"\n')
241
242    def test_archiver_mode_mirror_excludes(self):
243        """
244        Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"` and
245        correctly excludes an archive when its URL matches
246        `ARCHIVER_MIRROR_EXCLUDE`.
247        """
248
249        target='selftest-ed'
250        target_file_name = 'ed-1.14.1.tar.lz'
251
252        features = 'INHERIT += "archiver"\n'
253        features += 'ARCHIVER_MODE[src] = "mirror"\n'
254        features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
255        features += 'ARCHIVER_MIRROR_EXCLUDE = "${GNU_MIRROR}"\n'
256        self.write_config(features)
257
258        bitbake('-c clean %s' % (target))
259        bitbake('-c deploy_archives %s' % (target))
260
261        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS'])
262        glob_str = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['TARGET_SYS'], '%s-*' % (target))
263        glob_result = glob.glob(glob_str)
264        self.assertTrue(glob_result, 'Missing archiver directory for %s' % (target))
265
266        archive_path = os.path.join(glob_result[0], target_file_name)
267        self.assertFalse(os.path.exists(archive_path), 'Failed to exclude archive file %s' % (target_file_name))
268
269    def test_archiver_mode_mirror_combined(self):
270        """
271        Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"`
272        and `ARCHIVER_MODE[mirror] = "combined"`. Archives for multiple recipes
273        should all end up in the 'mirror' directory.
274        """
275
276        features = 'INHERIT += "archiver"\n'
277        features += 'ARCHIVER_MODE[src] = "mirror"\n'
278        features += 'ARCHIVER_MODE[mirror] = "combined"\n'
279        features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
280        features += 'COPYLEFT_LICENSE_INCLUDE = "*"\n'
281        self.write_config(features)
282
283        for target in ['selftest-ed', 'selftest-hardlink']:
284            bitbake('-c clean %s' % (target))
285            bitbake('-c deploy_archives %s' % (target))
286
287        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
288        for target_file_name in ['ed-1.14.1.tar.lz', 'hello.c']:
289            glob_str = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name)
290            glob_result = glob.glob(glob_str)
291            self.assertTrue(glob_result, 'Missing archive file %s' % (target_file_name))
292
293    def test_archiver_mode_mirror_gitsm(self):
294        """
295        Test that the archiver correctly handles git submodules with
296        `ARCHIVER_MODE[src] = "mirror"`.
297        """
298        features = 'INHERIT += "archiver"\n'
299        features += 'ARCHIVER_MODE[src] = "mirror"\n'
300        features += 'ARCHIVER_MODE[mirror] = "combined"\n'
301        features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
302        features += 'COPYLEFT_LICENSE_INCLUDE = "*"\n'
303        self.write_config(features)
304
305        bitbake('-c clean git-submodule-test')
306        bitbake('-c deploy_archives -f git-submodule-test')
307
308        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
309        for target_file_name in [
310            'git2_git.yoctoproject.org.git-submodule-test.tar.gz',
311            'git2_git.yoctoproject.org.bitbake-gitsm-test1.tar.gz',
312            'git2_git.yoctoproject.org.bitbake-gitsm-test2.tar.gz',
313            'git2_git.openembedded.org.bitbake.tar.gz'
314        ]:
315            target_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name)
316            self.assertTrue(os.path.exists(target_path))
317
318    def test_archiver_mode_mirror_gitsm_shallow(self):
319        """
320        Test that the archiver correctly handles git submodules with
321        `ARCHIVER_MODE[src] = "mirror"`.
322        """
323        features = 'INHERIT += "archiver"\n'
324        features += 'ARCHIVER_MODE[src] = "mirror"\n'
325        features += 'ARCHIVER_MODE[mirror] = "combined"\n'
326        features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
327        features += 'COPYLEFT_LICENSE_INCLUDE = "*"\n'
328        features += 'BB_GIT_SHALLOW = "1"\n'
329        features += 'BB_GENERATE_SHALLOW_TARBALLS = "1"\n'
330        features += 'DL_DIR = "${TOPDIR}/downloads-shallow"\n'
331        self.write_config(features)
332
333        bitbake('-c clean git-submodule-test')
334        bitbake('-c deploy_archives -f git-submodule-test')
335
336        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
337        for target_file_name in [
338            'gitsmshallow_git.yoctoproject.org.git-submodule-test_a2885dd-1_master.tar.gz',
339            'gitsmshallow_git.yoctoproject.org.bitbake-gitsm-test1_bare_120f4c7-1.tar.gz',
340            'gitsmshallow_git.yoctoproject.org.bitbake-gitsm-test2_bare_f66699e-1.tar.gz',
341            'gitsmshallow_git.openembedded.org.bitbake_bare_52a144a-1.tar.gz',
342            'gitsmshallow_git.openembedded.org.bitbake_bare_c39b997-1.tar.gz'
343        ]:
344            target_path = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name)
345            self.assertTrue(os.path.exists(target_path))
346