1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7import errno
8import os
9import re
10import shutil
11import tempfile
12import glob
13import fnmatch
14import unittest
15import json
16
17from oeqa.selftest.case import OESelftestTestCase
18from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer
19from oeqa.utils.commands import get_bb_vars, runqemu, get_test_layer
20from oeqa.core.decorator import OETestTag
21
22oldmetapath = None
23
24def setUpModule():
25    import bb.utils
26
27    global templayerdir
28    templayerdir = tempfile.mkdtemp(prefix='devtoolqa')
29    corecopydir = os.path.join(templayerdir, 'core-copy')
30    bblayers_conf = os.path.join(os.environ['BUILDDIR'], 'conf', 'bblayers.conf')
31    edited_layers = []
32    # make sure user doesn't have a local workspace
33    result = runCmd('bitbake-layers show-layers')
34    assert "workspacelayer" not in result.output, "Devtool test suite cannot be run with a local workspace directory"
35
36    # We need to take a copy of the meta layer so we can modify it and not
37    # have any races against other tests that might be running in parallel
38    # however things like COREBASE mean that you can't just copy meta, you
39    # need the whole repository.
40    def bblayers_edit_cb(layerpath, canonical_layerpath):
41        global oldmetapath
42        if not canonical_layerpath.endswith('/'):
43            # This helps us match exactly when we're using this path later
44            canonical_layerpath += '/'
45        if not edited_layers and canonical_layerpath.endswith('/meta/'):
46            canonical_layerpath = os.path.realpath(canonical_layerpath) + '/'
47            edited_layers.append(layerpath)
48            oldmetapath = os.path.realpath(layerpath)
49
50            # when downloading poky from tar.gz some tests will be skipped (BUG 12389)
51            try:
52                runCmd('git rev-parse --is-inside-work-tree', cwd=canonical_layerpath)
53            except:
54                raise unittest.SkipTest("devtool tests require folder to be a git repo")
55
56            result = runCmd('git rev-parse --show-toplevel', cwd=canonical_layerpath)
57            oldreporoot = result.output.rstrip()
58            newmetapath = os.path.join(corecopydir, os.path.relpath(oldmetapath, oldreporoot))
59            runCmd('git clone file://%s %s' % (oldreporoot, corecopydir), cwd=templayerdir)
60            # Now we need to copy any modified files
61            # You might ask "why not just copy the entire tree instead of
62            # cloning and doing this?" - well, the problem with that is
63            # TMPDIR or an equally large subdirectory might exist
64            # under COREBASE and we don't want to copy that, so we have
65            # to be selective.
66            result = runCmd('git status --porcelain', cwd=oldreporoot)
67            for line in result.output.splitlines():
68                if line.startswith(' M ') or line.startswith('?? '):
69                    relpth = line.split()[1]
70                    pth = os.path.join(oldreporoot, relpth)
71                    if pth.startswith(canonical_layerpath):
72                        if relpth.endswith('/'):
73                            destdir = os.path.join(corecopydir, relpth)
74                            # avoid race condition by not copying .pyc files YPBZ#13421,13803
75                            shutil.copytree(pth, destdir, ignore=shutil.ignore_patterns('*.pyc', '__pycache__'))
76                        else:
77                            destdir = os.path.join(corecopydir, os.path.dirname(relpth))
78                            bb.utils.mkdirhier(destdir)
79                            shutil.copy2(pth, destdir)
80            return newmetapath
81        else:
82            return layerpath
83    bb.utils.edit_bblayers_conf(bblayers_conf, None, None, bblayers_edit_cb)
84
85def tearDownModule():
86    if oldmetapath:
87        edited_layers = []
88        def bblayers_edit_cb(layerpath, canonical_layerpath):
89            if not edited_layers and canonical_layerpath.endswith('/meta'):
90                edited_layers.append(layerpath)
91                return oldmetapath
92            else:
93                return layerpath
94        bblayers_conf = os.path.join(os.environ['BUILDDIR'], 'conf', 'bblayers.conf')
95        bb.utils.edit_bblayers_conf(bblayers_conf, None, None, bblayers_edit_cb)
96    shutil.rmtree(templayerdir)
97
98class DevtoolTestCase(OESelftestTestCase):
99
100    def setUp(self):
101        """Test case setup function"""
102        super(DevtoolTestCase, self).setUp()
103        self.workspacedir = os.path.join(self.builddir, 'workspace')
104        self.assertTrue(not os.path.exists(self.workspacedir),
105                        'This test cannot be run with a workspace directory '
106                        'under the build directory')
107
108    def _check_src_repo(self, repo_dir):
109        """Check srctree git repository"""
110        self.assertTrue(os.path.isdir(os.path.join(repo_dir, '.git')),
111                        'git repository for external source tree not found')
112        result = runCmd('git status --porcelain', cwd=repo_dir)
113        self.assertEqual(result.output.strip(), "",
114                         'Created git repo is not clean')
115        result = runCmd('git symbolic-ref HEAD', cwd=repo_dir)
116        self.assertEqual(result.output.strip(), "refs/heads/devtool",
117                         'Wrong branch in git repo')
118
119    def _check_repo_status(self, repo_dir, expected_status):
120        """Check the worktree status of a repository"""
121        result = runCmd('git status . --porcelain',
122                        cwd=repo_dir)
123        for line in result.output.splitlines():
124            for ind, (f_status, fn_re) in enumerate(expected_status):
125                if re.match(fn_re, line[3:]):
126                    if f_status != line[:2]:
127                        self.fail('Unexpected status in line: %s' % line)
128                    expected_status.pop(ind)
129                    break
130            else:
131                self.fail('Unexpected modified file in line: %s' % line)
132        if expected_status:
133            self.fail('Missing file changes: %s' % expected_status)
134
135    def _test_recipe_contents(self, recipefile, checkvars, checkinherits):
136        with open(recipefile, 'r') as f:
137            invar = None
138            invalue = None
139            inherits = set()
140            for line in f:
141                var = None
142                if invar:
143                    value = line.strip().strip('"')
144                    if value.endswith('\\'):
145                        invalue += ' ' + value[:-1].strip()
146                        continue
147                    else:
148                        invalue += ' ' + value.strip()
149                        var = invar
150                        value = invalue
151                        invar = None
152                elif '=' in line:
153                    splitline = line.split('=', 1)
154                    var = splitline[0].rstrip()
155                    value = splitline[1].strip().strip('"')
156                    if value.endswith('\\'):
157                        invalue = value[:-1].strip()
158                        invar = var
159                        continue
160                elif line.startswith('inherit '):
161                    inherits.update(line.split()[1:])
162
163                if var and var in checkvars:
164                    needvalue = checkvars.pop(var)
165                    if needvalue is None:
166                        self.fail('Variable %s should not appear in recipe, but value is being set to "%s"' % (var, value))
167                    if isinstance(needvalue, set):
168                        if var == 'LICENSE':
169                            value = set(value.split(' & '))
170                        else:
171                            value = set(value.split())
172                    self.assertEqual(value, needvalue, 'values for %s do not match' % var)
173
174
175        missingvars = {}
176        for var, value in checkvars.items():
177            if value is not None:
178                missingvars[var] = value
179        self.assertEqual(missingvars, {}, 'Some expected variables not found in recipe: %s' % checkvars)
180
181        for inherit in checkinherits:
182            self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit)
183
184    def _check_bbappend(self, testrecipe, recipefile, appenddir):
185        result = runCmd('bitbake-layers show-appends', cwd=self.builddir)
186        resultlines = result.output.splitlines()
187        inrecipe = False
188        bbappends = []
189        bbappendfile = None
190        for line in resultlines:
191            if inrecipe:
192                if line.startswith(' '):
193                    bbappends.append(line.strip())
194                else:
195                    break
196            elif line == '%s:' % os.path.basename(recipefile):
197                inrecipe = True
198        self.assertLessEqual(len(bbappends), 2, '%s recipe is being bbappended by another layer - bbappends found:\n  %s' % (testrecipe, '\n  '.join(bbappends)))
199        for bbappend in bbappends:
200            if bbappend.startswith(appenddir):
201                bbappendfile = bbappend
202                break
203        else:
204            self.fail('bbappend for recipe %s does not seem to be created in test layer' % testrecipe)
205        return bbappendfile
206
207    def _create_temp_layer(self, templayerdir, addlayer, templayername, priority=999, recipepathspec='recipes-*/*'):
208        create_temp_layer(templayerdir, templayername, priority, recipepathspec)
209        if addlayer:
210            self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir)
211            result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir)
212
213    def _process_ls_output(self, output):
214        """
215        Convert ls -l output to a format we can reasonably compare from one context
216        to another (e.g. from host to target)
217        """
218        filelist = []
219        for line in output.splitlines():
220            splitline = line.split()
221            if len(splitline) < 8:
222                self.fail('_process_ls_output: invalid output line: %s' % line)
223            # Remove trailing . on perms
224            splitline[0] = splitline[0].rstrip('.')
225            # Remove leading . on paths
226            splitline[-1] = splitline[-1].lstrip('.')
227            # Drop fields we don't want to compare
228            del splitline[7]
229            del splitline[6]
230            del splitline[5]
231            del splitline[4]
232            del splitline[1]
233            filelist.append(' '.join(splitline))
234        return filelist
235
236    def _check_diff(self, diffoutput, addlines, removelines):
237        """Check output from 'git diff' matches expectation"""
238        remaining_addlines = addlines[:]
239        remaining_removelines = removelines[:]
240        for line in diffoutput.splitlines():
241            if line.startswith('+++') or line.startswith('---'):
242                continue
243            elif line.startswith('+'):
244                matched = False
245                for item in addlines:
246                    if re.match(item, line[1:].strip()):
247                        matched = True
248                        remaining_addlines.remove(item)
249                        break
250                self.assertTrue(matched, 'Unexpected diff add line: %s' % line)
251            elif line.startswith('-'):
252                matched = False
253                for item in removelines:
254                    if re.match(item, line[1:].strip()):
255                        matched = True
256                        remaining_removelines.remove(item)
257                        break
258                self.assertTrue(matched, 'Unexpected diff remove line: %s' % line)
259        if remaining_addlines:
260            self.fail('Expected added lines not found: %s' % remaining_addlines)
261        if remaining_removelines:
262            self.fail('Expected removed lines not found: %s' % remaining_removelines)
263
264    def _check_runqemu_prerequisites(self):
265        """Check runqemu is available
266
267        Whilst some tests would seemingly be better placed as a runtime test,
268        unfortunately the runtime tests run under bitbake and you can't run
269        devtool within bitbake (since devtool needs to run bitbake itself).
270        Additionally we are testing build-time functionality as well, so
271        really this has to be done as an oe-selftest test.
272        """
273        machine = get_bb_var('MACHINE')
274        if not machine.startswith('qemu'):
275            self.skipTest('This test only works with qemu machines')
276        if not os.path.exists('/etc/runqemu-nosudo'):
277            self.skipTest('You must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')
278        result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ip tuntap show', ignore_status=True)
279        if result.status != 0:
280            result = runCmd('PATH="$PATH:/sbin:/usr/sbin" ifconfig -a', ignore_status=True)
281            if result.status != 0:
282                self.skipTest('Failed to determine if tap devices exist with ifconfig or ip: %s' % result.output)
283        for line in result.output.splitlines():
284            if line.startswith('tap'):
285                break
286        else:
287            self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test')
288
289    def _test_devtool_add_git_url(self, git_url, version, pn, resulting_src_uri):
290        self.track_for_cleanup(self.workspacedir)
291        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
292        result = runCmd('devtool add --version %s %s %s' % (version, pn, git_url))
293        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created')
294        # Check the recipe name is correct
295        recipefile = get_bb_var('FILE', pn)
296        self.assertIn('%s_git.bb' % pn, recipefile, 'Recipe file incorrectly named')
297        self.assertIn(recipefile, result.output)
298        # Test devtool status
299        result = runCmd('devtool status')
300        self.assertIn(pn, result.output)
301        self.assertIn(recipefile, result.output)
302        checkvars = {}
303        checkvars['SRC_URI'] = resulting_src_uri
304        self._test_recipe_contents(recipefile, checkvars, [])
305
306class DevtoolBase(DevtoolTestCase):
307
308    @classmethod
309    def setUpClass(cls):
310        super(DevtoolBase, cls).setUpClass()
311        bb_vars = get_bb_vars(['TOPDIR', 'SSTATE_DIR'])
312        cls.original_sstate = bb_vars['SSTATE_DIR']
313        cls.devtool_sstate = os.path.join(bb_vars['TOPDIR'], 'sstate_devtool')
314        cls.sstate_conf  = 'SSTATE_DIR = "%s"\n' % cls.devtool_sstate
315        cls.sstate_conf += ('SSTATE_MIRRORS += "file://.* file:///%s/PATH"\n'
316                            % cls.original_sstate)
317        cls.sstate_conf += ('BB_HASHSERVE_UPSTREAM = "hashserv.yocto.io:8687"\n')
318
319    @classmethod
320    def tearDownClass(cls):
321        cls.logger.debug('Deleting devtool sstate cache on %s' % cls.devtool_sstate)
322        runCmd('rm -rf %s' % cls.devtool_sstate)
323        super(DevtoolBase, cls).tearDownClass()
324
325    def setUp(self):
326        """Test case setup function"""
327        super(DevtoolBase, self).setUp()
328        self.append_config(self.sstate_conf)
329
330
331class DevtoolTests(DevtoolBase):
332
333    def test_create_workspace(self):
334        # Check preconditions
335        result = runCmd('bitbake-layers show-layers')
336        self.assertTrue('\nworkspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf')
337        # remove conf/devtool.conf to avoid it corrupting tests
338        devtoolconf = os.path.join(self.builddir, 'conf', 'devtool.conf')
339        self.track_for_cleanup(devtoolconf)
340        # Try creating a workspace layer with a specific path
341        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
342        self.track_for_cleanup(tempdir)
343        result = runCmd('devtool create-workspace %s' % tempdir)
344        self.assertTrue(os.path.isfile(os.path.join(tempdir, 'conf', 'layer.conf')), msg = "No workspace created. devtool output: %s " % result.output)
345        result = runCmd('bitbake-layers show-layers')
346        self.assertIn(tempdir, result.output)
347        # Try creating a workspace layer with the default path
348        self.track_for_cleanup(self.workspacedir)
349        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
350        result = runCmd('devtool create-workspace')
351        self.assertTrue(os.path.isfile(os.path.join(self.workspacedir, 'conf', 'layer.conf')), msg = "No workspace created. devtool output: %s " % result.output)
352        result = runCmd('bitbake-layers show-layers')
353        self.assertNotIn(tempdir, result.output)
354        self.assertIn(self.workspacedir, result.output)
355
356class DevtoolAddTests(DevtoolBase):
357
358    def test_devtool_add(self):
359        # Fetch source
360        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
361        self.track_for_cleanup(tempdir)
362        pn = 'pv'
363        pv = '1.5.3'
364        url = 'http://downloads.yoctoproject.org/mirror/sources/pv-1.5.3.tar.bz2'
365        result = runCmd('wget %s' % url, cwd=tempdir)
366        result = runCmd('tar xfv %s' % os.path.basename(url), cwd=tempdir)
367        srcdir = os.path.join(tempdir, '%s-%s' % (pn, pv))
368        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')), 'Unable to find configure script in source directory')
369        # Test devtool add
370        self.track_for_cleanup(self.workspacedir)
371        self.add_command_to_tearDown('bitbake -c cleansstate %s' % pn)
372        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
373        result = runCmd('devtool add %s %s' % (pn, srcdir))
374        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created')
375        # Test devtool status
376        result = runCmd('devtool status')
377        recipepath = '%s/recipes/%s/%s_%s.bb' % (self.workspacedir, pn, pn, pv)
378        self.assertIn(recipepath, result.output)
379        self.assertIn(srcdir, result.output)
380        # Test devtool find-recipe
381        result = runCmd('devtool -q find-recipe %s' % pn)
382        self.assertEqual(recipepath, result.output.strip())
383        # Test devtool edit-recipe
384        result = runCmd('VISUAL="echo 123" devtool -q edit-recipe %s' % pn)
385        self.assertEqual('123 %s' % recipepath, result.output.strip())
386        # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then)
387        bitbake('%s -c cleansstate' % pn)
388        # Test devtool build
389        result = runCmd('devtool build %s' % pn)
390        bb_vars = get_bb_vars(['D', 'bindir'], pn)
391        installdir = bb_vars['D']
392        self.assertTrue(installdir, 'Could not query installdir variable')
393        bindir = bb_vars['bindir']
394        self.assertTrue(bindir, 'Could not query bindir variable')
395        if bindir[0] == '/':
396            bindir = bindir[1:]
397        self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D')
398
399    def test_devtool_add_binary(self):
400        # Create a binary package containing a known test file
401        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
402        self.track_for_cleanup(tempdir)
403        pn = 'tst-bin'
404        pv = '1.0'
405        test_file_dir     = "var/lib/%s/" % pn
406        test_file_name    = "test_file"
407        test_file_content = "TEST CONTENT"
408        test_file_package_root = os.path.join(tempdir, pn)
409        test_file_dir_full = os.path.join(test_file_package_root, test_file_dir)
410        bb.utils.mkdirhier(test_file_dir_full)
411        with open(os.path.join(test_file_dir_full, test_file_name), "w") as f:
412           f.write(test_file_content)
413        bin_package_path = os.path.join(tempdir, "%s.tar.gz" % pn)
414        runCmd("tar czf %s -C %s ." % (bin_package_path, test_file_package_root))
415
416        # Test devtool add -b on the binary package
417        self.track_for_cleanup(self.workspacedir)
418        self.add_command_to_tearDown('bitbake -c cleansstate %s' % pn)
419        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
420        result = runCmd('devtool add  -b %s %s' % (pn, bin_package_path))
421        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created')
422
423        # Build the resulting recipe
424        result = runCmd('devtool build %s' % pn)
425        installdir = get_bb_var('D', pn)
426        self.assertTrue(installdir, 'Could not query installdir variable')
427
428        # Check that a known file from the binary package has indeed been installed
429        self.assertTrue(os.path.isfile(os.path.join(installdir, test_file_dir, test_file_name)), '%s not found in D' % test_file_name)
430
431    def test_devtool_add_git_local(self):
432        # We need dbus built so that DEPENDS recognition works
433        bitbake('dbus')
434        # Fetch source from a remote URL, but do it outside of devtool
435        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
436        self.track_for_cleanup(tempdir)
437        pn = 'dbus-wait'
438        srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517'
439        # We choose an https:// git URL here to check rewriting the URL works
440        url = 'https://git.yoctoproject.org/git/dbus-wait'
441        # Force fetching to "noname" subdir so we verify we're picking up the name from autoconf
442        # instead of the directory name
443        result = runCmd('git clone %s noname' % url, cwd=tempdir)
444        srcdir = os.path.join(tempdir, 'noname')
445        result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir)
446        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory')
447        # Test devtool add
448        self.track_for_cleanup(self.workspacedir)
449        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
450        # Don't specify a name since we should be able to auto-detect it
451        result = runCmd('devtool add %s' % srcdir)
452        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created')
453        # Check the recipe name is correct
454        recipefile = get_bb_var('FILE', pn)
455        self.assertIn('%s_git.bb' % pn, recipefile, 'Recipe file incorrectly named')
456        self.assertIn(recipefile, result.output)
457        # Test devtool status
458        result = runCmd('devtool status')
459        self.assertIn(pn, result.output)
460        self.assertIn(srcdir, result.output)
461        self.assertIn(recipefile, result.output)
462        checkvars = {}
463        checkvars['LICENSE'] = 'GPL-2.0-only'
464        checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263'
465        checkvars['S'] = '${WORKDIR}/git'
466        checkvars['PV'] = '0.1+git'
467        checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/dbus-wait;protocol=https;branch=master'
468        checkvars['SRCREV'] = srcrev
469        checkvars['DEPENDS'] = set(['dbus'])
470        self._test_recipe_contents(recipefile, checkvars, [])
471
472    def test_devtool_add_git_style1(self):
473        version = 'v3.1.0'
474        pn = 'mbedtls'
475        # this will trigger reformat_git_uri with branch parameter in url
476        git_url = "'git://git@github.com/ARMmbed/mbedtls.git;branch=mbedtls-2.28;protocol=https'"
477        resulting_src_uri = "git://git@github.com/ARMmbed/mbedtls.git;branch=mbedtls-2.28;protocol=https"
478        self._test_devtool_add_git_url(git_url, version, pn, resulting_src_uri)
479
480    def test_devtool_add_git_style2(self):
481        version = 'v3.1.0'
482        pn = 'mbedtls'
483        # this will trigger reformat_git_uri with branch parameter in url
484        git_url = "'git://git@github.com/ARMmbed/mbedtls.git;protocol=https'"
485        resulting_src_uri = "gitsm://git@github.com/ARMmbed/mbedtls.git;protocol=https;branch=master"
486        self._test_devtool_add_git_url(git_url, version, pn, resulting_src_uri)
487
488    def test_devtool_add_library(self):
489        # Fetch source
490        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
491        self.track_for_cleanup(tempdir)
492        version = '1.1'
493        url = 'https://www.intra2net.com/en/developer/libftdi/download/libftdi1-%s.tar.bz2' % version
494        result = runCmd('wget %s' % url, cwd=tempdir)
495        result = runCmd('tar xfv libftdi1-%s.tar.bz2' % version, cwd=tempdir)
496        srcdir = os.path.join(tempdir, 'libftdi1-%s' % version)
497        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'CMakeLists.txt')), 'Unable to find CMakeLists.txt in source directory')
498        # Test devtool add (and use -V so we test that too)
499        self.track_for_cleanup(self.workspacedir)
500        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
501        result = runCmd('devtool add libftdi %s -V %s' % (srcdir, version))
502        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created')
503        # Test devtool status
504        result = runCmd('devtool status')
505        self.assertIn('libftdi', result.output)
506        self.assertIn(srcdir, result.output)
507        # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then)
508        bitbake('libftdi -c cleansstate')
509        # libftdi's python/CMakeLists.txt is a bit broken, so let's just disable it
510        # There's also the matter of it installing cmake files to a path we don't
511        # normally cover, which triggers the installed-vs-shipped QA test we have
512        # within do_package
513        recipefile = '%s/recipes/libftdi/libftdi_%s.bb' % (self.workspacedir, version)
514        result = runCmd('recipetool setvar %s EXTRA_OECMAKE -- \'-DPYTHON_BINDINGS=OFF -DLIBFTDI_CMAKE_CONFIG_DIR=${datadir}/cmake/Modules\'' % recipefile)
515        with open(recipefile, 'a') as f:
516            f.write('\nFILES:${PN}-dev += "${datadir}/cmake/Modules"\n')
517            # We don't have the ability to pick up this dependency automatically yet...
518            f.write('\nDEPENDS += "libusb1"\n')
519            f.write('\nTESTLIBOUTPUT = "${COMPONENTS_DIR}/${TUNE_PKGARCH}/${PN}/${libdir}"\n')
520        # Test devtool build
521        result = runCmd('devtool build libftdi')
522        bb_vars = get_bb_vars(['TESTLIBOUTPUT', 'STAMP'], 'libftdi')
523        staging_libdir = bb_vars['TESTLIBOUTPUT']
524        self.assertTrue(staging_libdir, 'Could not query TESTLIBOUTPUT variable')
525        self.assertTrue(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), "libftdi binary not found in STAGING_LIBDIR. Output of devtool build libftdi %s" % result.output)
526        # Test devtool reset
527        stampprefix = bb_vars['STAMP']
528        result = runCmd('devtool reset libftdi')
529        result = runCmd('devtool status')
530        self.assertNotIn('libftdi', result.output)
531        self.assertTrue(stampprefix, 'Unable to get STAMP value for recipe libftdi')
532        matches = glob.glob(stampprefix + '*')
533        self.assertFalse(matches, 'Stamp files exist for recipe libftdi that should have been cleaned')
534        self.assertFalse(os.path.isfile(os.path.join(staging_libdir, 'libftdi1.so.2.1.0')), 'libftdi binary still found in STAGING_LIBDIR after cleaning')
535
536    def test_devtool_add_fetch(self):
537        # Fetch source
538        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
539        self.track_for_cleanup(tempdir)
540        testver = '0.23'
541        url = 'https://files.pythonhosted.org/packages/c0/41/bae1254e0396c0cc8cf1751cb7d9afc90a602353695af5952530482c963f/MarkupSafe-%s.tar.gz' % testver
542        testrecipe = 'python-markupsafe'
543        srcdir = os.path.join(tempdir, testrecipe)
544        # Test devtool add
545        self.track_for_cleanup(self.workspacedir)
546        self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)
547        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
548        result = runCmd('devtool add --no-pypi %s %s -f %s' % (testrecipe, srcdir, url))
549        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. %s' % result.output)
550        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')
551        self.assertTrue(os.path.isdir(os.path.join(srcdir, '.git')), 'git repository for external source tree was not created')
552        # Test devtool status
553        result = runCmd('devtool status')
554        self.assertIn(testrecipe, result.output)
555        self.assertIn(srcdir, result.output)
556        # Check recipe
557        recipefile = get_bb_var('FILE', testrecipe)
558        self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named')
559        checkvars = {}
560        checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}'
561        checkvars['SRC_URI'] = url.replace(testver, '${PV}')
562        self._test_recipe_contents(recipefile, checkvars, [])
563        # Try with version specified
564        result = runCmd('devtool reset -n %s' % testrecipe)
565        shutil.rmtree(srcdir)
566        fakever = '1.9'
567        result = runCmd('devtool add --no-pypi %s %s -f %s -V %s' % (testrecipe, srcdir, url, fakever))
568        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory')
569        # Test devtool status
570        result = runCmd('devtool status')
571        self.assertIn(testrecipe, result.output)
572        self.assertIn(srcdir, result.output)
573        # Check recipe
574        recipefile = get_bb_var('FILE', testrecipe)
575        self.assertIn('%s_%s.bb' % (testrecipe, fakever), recipefile, 'Recipe file incorrectly named')
576        checkvars = {}
577        checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver
578        checkvars['SRC_URI'] = url
579        self._test_recipe_contents(recipefile, checkvars, [])
580
581    def test_devtool_add_fetch_git(self):
582        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
583        self.track_for_cleanup(tempdir)
584        url = 'gitsm://git.yoctoproject.org/mraa'
585        url_branch = '%s;branch=master' % url
586        checkrev = 'ae127b19a50aa54255e4330ccfdd9a5d058e581d'
587        testrecipe = 'mraa'
588        srcdir = os.path.join(tempdir, testrecipe)
589        # Test devtool add
590        self.track_for_cleanup(self.workspacedir)
591        self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe)
592        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
593        result = runCmd('devtool add %s %s -a -f %s' % (testrecipe, srcdir, url))
594        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created: %s' % result.output)
595        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'imraa', 'imraa.c')), 'Unable to find imraa/imraa.c in source directory')
596        # Test devtool status
597        result = runCmd('devtool status')
598        self.assertIn(testrecipe, result.output)
599        self.assertIn(srcdir, result.output)
600        # Check recipe
601        recipefile = get_bb_var('FILE', testrecipe)
602        self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')
603        checkvars = {}
604        checkvars['S'] = '${WORKDIR}/git'
605        checkvars['PV'] = '1.0+git'
606        checkvars['SRC_URI'] = url_branch
607        checkvars['SRCREV'] = '${AUTOREV}'
608        self._test_recipe_contents(recipefile, checkvars, [])
609        # Try with revision and version specified
610        result = runCmd('devtool reset -n %s' % testrecipe)
611        shutil.rmtree(srcdir)
612        url_rev = '%s;rev=%s' % (url, checkrev)
613        result = runCmd('devtool add %s %s -f "%s" -V 1.5' % (testrecipe, srcdir, url_rev))
614        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'imraa', 'imraa.c')), 'Unable to find imraa/imraa.c in source directory')
615        # Test devtool status
616        result = runCmd('devtool status')
617        self.assertIn(testrecipe, result.output)
618        self.assertIn(srcdir, result.output)
619        # Check recipe
620        recipefile = get_bb_var('FILE', testrecipe)
621        self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named')
622        checkvars = {}
623        checkvars['S'] = '${WORKDIR}/git'
624        checkvars['PV'] = '1.5+git'
625        checkvars['SRC_URI'] = url_branch
626        checkvars['SRCREV'] = checkrev
627        self._test_recipe_contents(recipefile, checkvars, [])
628
629    def test_devtool_add_fetch_simple(self):
630        # Fetch source from a remote URL, auto-detecting name
631        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
632        self.track_for_cleanup(tempdir)
633        testver = '1.6.0'
634        url = 'http://www.ivarch.com/programs/sources/pv-%s.tar.bz2' % testver
635        testrecipe = 'pv'
636        srcdir = os.path.join(self.workspacedir, 'sources', testrecipe)
637        # Test devtool add
638        self.track_for_cleanup(self.workspacedir)
639        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
640        result = runCmd('devtool add %s' % url)
641        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. %s' % result.output)
642        self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')), 'Unable to find configure script in source directory')
643        self.assertTrue(os.path.isdir(os.path.join(srcdir, '.git')), 'git repository for external source tree was not created')
644        # Test devtool status
645        result = runCmd('devtool status')
646        self.assertIn(testrecipe, result.output)
647        self.assertIn(srcdir, result.output)
648        # Check recipedevtool add
649        recipefile = get_bb_var('FILE', testrecipe)
650        self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named')
651        checkvars = {}
652        checkvars['S'] = None
653        checkvars['SRC_URI'] = url.replace(testver, '${PV}')
654        self._test_recipe_contents(recipefile, checkvars, [])
655
656    def test_devtool_add_npm(self):
657        collections = get_bb_var('BBFILE_COLLECTIONS').split()
658        if "openembedded-layer" not in collections:
659            self.skipTest("Test needs meta-oe for nodejs")
660
661        pn = 'savoirfairelinux-node-server-example'
662        pv = '1.0.0'
663        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=' + pv
664        # Test devtool add
665        self.track_for_cleanup(self.workspacedir)
666        self.add_command_to_tearDown('bitbake -c cleansstate %s' % pn)
667        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
668        result = runCmd('devtool add \'%s\'' % url)
669        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created')
670        self.assertExists(os.path.join(self.workspacedir, 'recipes', pn, '%s_%s.bb' % (pn, pv)), 'Recipe not created')
671        self.assertExists(os.path.join(self.workspacedir, 'recipes', pn, pn, 'npm-shrinkwrap.json'), 'Shrinkwrap not created')
672        # Test devtool status
673        result = runCmd('devtool status')
674        self.assertIn(pn, result.output)
675        # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then)
676        bitbake('%s -c cleansstate' % pn)
677        # Test devtool build
678        result = runCmd('devtool build %s' % pn)
679
680    def test_devtool_add_python_egg_requires(self):
681        # Fetch source
682        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
683        self.track_for_cleanup(tempdir)
684        testver = '0.14.0'
685        url = 'https://files.pythonhosted.org/packages/e9/9e/25d59f5043cf763833b2581c8027fa92342c4cf8ee523b498ecdf460c16d/uvicorn-%s.tar.gz' % testver
686        testrecipe = 'python3-uvicorn'
687        srcdir = os.path.join(tempdir, testrecipe)
688        # Test devtool add
689        self.track_for_cleanup(self.workspacedir)
690        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
691        result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url))
692
693class DevtoolModifyTests(DevtoolBase):
694
695    def test_devtool_modify(self):
696        import oe.path
697
698        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
699        self.track_for_cleanup(tempdir)
700        self.track_for_cleanup(self.workspacedir)
701        self.add_command_to_tearDown('bitbake -c clean mdadm')
702        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
703        result = runCmd('devtool modify mdadm -x %s' % tempdir)
704        self.assertExists(os.path.join(tempdir, 'Makefile'), 'Extracted source could not be found')
705        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created')
706        matches = glob.glob(os.path.join(self.workspacedir, 'appends', 'mdadm_*.bbappend'))
707        self.assertTrue(matches, 'bbappend not created %s' % result.output)
708
709        # Test devtool status
710        result = runCmd('devtool status')
711        self.assertIn('mdadm', result.output)
712        self.assertIn(tempdir, result.output)
713        self._check_src_repo(tempdir)
714
715        bitbake('mdadm -C unpack')
716
717        def check_line(checkfile, expected, message, present=True):
718            # Check for $expected, on a line on its own, in checkfile.
719            with open(checkfile, 'r') as f:
720                if present:
721                    self.assertIn(expected + '\n', f, message)
722                else:
723                    self.assertNotIn(expected + '\n', f, message)
724
725        modfile = os.path.join(tempdir, 'mdadm.8.in')
726        bb_vars = get_bb_vars(['PKGD', 'mandir'], 'mdadm')
727        pkgd = bb_vars['PKGD']
728        self.assertTrue(pkgd, 'Could not query PKGD variable')
729        mandir = bb_vars['mandir']
730        self.assertTrue(mandir, 'Could not query mandir variable')
731        manfile = oe.path.join(pkgd, mandir, 'man8', 'mdadm.8')
732
733        check_line(modfile, 'Linux Software RAID', 'Could not find initial string')
734        check_line(modfile, 'antique pin sardine', 'Unexpectedly found replacement string', present=False)
735
736        result = runCmd("sed -i 's!^Linux Software RAID$!antique pin sardine!' %s" % modfile)
737        check_line(modfile, 'antique pin sardine', 'mdadm.8.in file not modified (sed failed)')
738
739        bitbake('mdadm -c package')
740        check_line(manfile, 'antique pin sardine', 'man file not modified. man searched file path: %s' % manfile)
741
742        result = runCmd('git checkout -- %s' % modfile, cwd=tempdir)
743        check_line(modfile, 'Linux Software RAID', 'man .in file not restored (git failed)')
744
745        bitbake('mdadm -c package')
746        check_line(manfile, 'Linux Software RAID', 'man file not updated. man searched file path: %s' % manfile)
747
748        result = runCmd('devtool reset mdadm')
749        result = runCmd('devtool status')
750        self.assertNotIn('mdadm', result.output)
751
752    def test_devtool_buildclean(self):
753        def assertFile(path, *paths):
754            f = os.path.join(path, *paths)
755            self.assertExists(f)
756        def assertNoFile(path, *paths):
757            f = os.path.join(path, *paths)
758            self.assertNotExists(f)
759
760        # Clean up anything in the workdir/sysroot/sstate cache
761        bitbake('mdadm m4 -c cleansstate')
762        # Try modifying a recipe
763        tempdir_mdadm = tempfile.mkdtemp(prefix='devtoolqa')
764        tempdir_m4 = tempfile.mkdtemp(prefix='devtoolqa')
765        builddir_m4 = tempfile.mkdtemp(prefix='devtoolqa')
766        self.track_for_cleanup(tempdir_mdadm)
767        self.track_for_cleanup(tempdir_m4)
768        self.track_for_cleanup(builddir_m4)
769        self.track_for_cleanup(self.workspacedir)
770        self.add_command_to_tearDown('bitbake -c clean mdadm m4')
771        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
772        self.write_recipeinc('m4', 'EXTERNALSRC_BUILD = "%s"\ndo_clean() {\n\t:\n}\n' % builddir_m4)
773        try:
774            runCmd('devtool modify mdadm -x %s' % tempdir_mdadm)
775            runCmd('devtool modify m4 -x %s' % tempdir_m4)
776            assertNoFile(tempdir_mdadm, 'mdadm')
777            assertNoFile(builddir_m4, 'src/m4')
778            result = bitbake('m4 -e')
779            result = bitbake('mdadm m4 -c compile')
780            self.assertEqual(result.status, 0)
781            assertFile(tempdir_mdadm, 'mdadm')
782            assertFile(builddir_m4, 'src/m4')
783            # Check that buildclean task exists and does call make clean
784            bitbake('mdadm m4 -c buildclean')
785            assertNoFile(tempdir_mdadm, 'mdadm')
786            assertNoFile(builddir_m4, 'src/m4')
787            runCmd('echo "#Trigger rebuild" >> %s/Makefile' % tempdir_mdadm)
788            bitbake('mdadm m4 -c compile')
789            assertFile(tempdir_mdadm, 'mdadm')
790            assertFile(builddir_m4, 'src/m4')
791            bitbake('mdadm m4 -c clean')
792            # Check that buildclean task is run before clean for B == S
793            assertNoFile(tempdir_mdadm, 'mdadm')
794            # Check that buildclean task is not run before clean for B != S
795            assertFile(builddir_m4, 'src/m4')
796        finally:
797            self.delete_recipeinc('m4')
798
799    def test_devtool_modify_invalid(self):
800        # Try modifying some recipes
801        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
802        self.track_for_cleanup(tempdir)
803        self.track_for_cleanup(self.workspacedir)
804        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
805
806        testrecipes = 'perf kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk'.split()
807        # Find actual name of gcc-source since it now includes the version - crude, but good enough for this purpose
808        result = runCmd('bitbake-layers show-recipes gcc-source*')
809        for line in result.output.splitlines():
810            # just match those lines that contain a real target
811            m = re.match('(?P<recipe>^[a-zA-Z0-9.-]+)(?P<colon>:$)', line)
812            if m:
813                testrecipes.append(m.group('recipe'))
814        for testrecipe in testrecipes:
815            # Check it's a valid recipe
816            bitbake('%s -e' % testrecipe)
817            # devtool extract should fail
818            result = runCmd('devtool extract %s %s' % (testrecipe, os.path.join(tempdir, testrecipe)), ignore_status=True)
819            self.assertNotEqual(result.status, 0, 'devtool extract on %s should have failed. devtool output: %s' % (testrecipe, result.output))
820            self.assertNotIn('Fetching ', result.output, 'devtool extract on %s should have errored out before trying to fetch' % testrecipe)
821            self.assertIn('ERROR: ', result.output, 'devtool extract on %s should have given an ERROR' % testrecipe)
822            # devtool modify should fail
823            result = runCmd('devtool modify %s -x %s' % (testrecipe, os.path.join(tempdir, testrecipe)), ignore_status=True)
824            self.assertNotEqual(result.status, 0, 'devtool modify on %s should have failed. devtool output: %s' %  (testrecipe, result.output))
825            self.assertIn('ERROR: ', result.output, 'devtool modify on %s should have given an ERROR' % testrecipe)
826
827    def test_devtool_modify_native(self):
828        # Check preconditions
829        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
830        # Try modifying some recipes
831        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
832        self.track_for_cleanup(tempdir)
833        self.track_for_cleanup(self.workspacedir)
834        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
835
836        bbclassextended = False
837        inheritnative = False
838        testrecipes = 'cdrtools-native mtools-native apt-native desktop-file-utils-native'.split()
839        for testrecipe in testrecipes:
840            checkextend = 'native' in (get_bb_var('BBCLASSEXTEND', testrecipe) or '').split()
841            if not bbclassextended:
842                bbclassextended = checkextend
843            if not inheritnative:
844                inheritnative = not checkextend
845            result = runCmd('devtool modify %s -x %s' % (testrecipe, os.path.join(tempdir, testrecipe)))
846            self.assertNotIn('ERROR: ', result.output, 'ERROR in devtool modify output: %s' % result.output)
847            result = runCmd('devtool build %s' % testrecipe)
848            self.assertNotIn('ERROR: ', result.output, 'ERROR in devtool build output: %s' % result.output)
849            result = runCmd('devtool reset %s' % testrecipe)
850            self.assertNotIn('ERROR: ', result.output, 'ERROR in devtool reset output: %s' % result.output)
851
852        self.assertTrue(bbclassextended, 'None of these recipes are BBCLASSEXTENDed to native - need to adjust testrecipes list: %s' % ', '.join(testrecipes))
853        self.assertTrue(inheritnative, 'None of these recipes do "inherit native" - need to adjust testrecipes list: %s' % ', '.join(testrecipes))
854
855    def test_devtool_modify_localfiles_only(self):
856        # Check preconditions
857        testrecipe = 'base-files'
858        src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split()
859        foundlocalonly = False
860        correct_symlink = False
861        for item in src_uri:
862            if item.startswith('file://'):
863                if '.patch' not in item:
864                    foundlocalonly = True
865            else:
866                foundlocalonly = False
867                break
868        self.assertTrue(foundlocalonly, 'This test expects the %s recipe to fetch local files only and it seems that it no longer does' % testrecipe)
869        # Clean up anything in the workdir/sysroot/sstate cache
870        bitbake('%s -c cleansstate' % testrecipe)
871        # Try modifying a recipe
872        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
873        self.track_for_cleanup(tempdir)
874        self.track_for_cleanup(self.workspacedir)
875        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
876        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
877        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
878        srcfile = os.path.join(tempdir, 'oe-local-files/share/dot.bashrc')
879        srclink = os.path.join(tempdir, 'share/dot.bashrc')
880        self.assertExists(srcfile, 'Extracted source could not be found')
881        if os.path.islink(srclink) and os.path.exists(srclink) and os.path.samefile(srcfile, srclink):
882            correct_symlink = True
883        self.assertTrue(correct_symlink, 'Source symlink to oe-local-files is broken')
884
885        matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % testrecipe))
886        self.assertTrue(matches, 'bbappend not created')
887        # Test devtool status
888        result = runCmd('devtool status')
889        self.assertIn(testrecipe, result.output)
890        self.assertIn(tempdir, result.output)
891        # Try building
892        bitbake(testrecipe)
893
894    def test_devtool_modify_git(self):
895        # Check preconditions
896        testrecipe = 'psplash'
897        src_uri = get_bb_var('SRC_URI', testrecipe)
898        self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)
899        # Clean up anything in the workdir/sysroot/sstate cache
900        bitbake('%s -c cleansstate' % testrecipe)
901        # Try modifying a recipe
902        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
903        self.track_for_cleanup(tempdir)
904        self.track_for_cleanup(self.workspacedir)
905        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
906        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
907        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
908        self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found')
909        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output)
910        matches = glob.glob(os.path.join(self.workspacedir, 'appends', 'psplash_*.bbappend'))
911        self.assertTrue(matches, 'bbappend not created')
912        # Test devtool status
913        result = runCmd('devtool status')
914        self.assertIn(testrecipe, result.output)
915        self.assertIn(tempdir, result.output)
916        # Check git repo
917        self._check_src_repo(tempdir)
918        # Try building
919        bitbake(testrecipe)
920
921    def test_devtool_modify_git_no_extract(self):
922        # Check preconditions
923        testrecipe = 'psplash'
924        src_uri = get_bb_var('SRC_URI', testrecipe)
925        self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)
926        # Clean up anything in the workdir/sysroot/sstate cache
927        bitbake('%s -c cleansstate' % testrecipe)
928        # Try modifying a recipe
929        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
930        self.track_for_cleanup(tempdir)
931        self.track_for_cleanup(self.workspacedir)
932        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
933        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
934        result = runCmd('git clone https://git.yoctoproject.org/psplash %s && devtool modify -n %s %s' % (tempdir, testrecipe, tempdir))
935        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output)
936        matches = glob.glob(os.path.join(self.workspacedir, 'appends', 'psplash_*.bbappend'))
937        self.assertTrue(matches, 'bbappend not created')
938        # Test devtool status
939        result = runCmd('devtool status')
940        self.assertIn(testrecipe, result.output)
941        self.assertIn(tempdir, result.output)
942
943    def test_devtool_modify_git_crates_subpath(self):
944        # This tests two things in devtool context:
945        #   - that we support local git dependencies for cargo based recipe
946        #   - that we support patches in SRC_URI when git url contains subpath parameter
947
948        # Check preconditions:
949        #    recipe inherits cargo
950        #    git:// uri with a subpath as the main package
951        #    some crate:// in SRC_URI
952        #    others git:// in SRC_URI
953        #    cointains a patch
954        testrecipe = 'hello-rs'
955        bb_vars = get_bb_vars(['SRC_URI', 'FILE', 'WORKDIR', 'CARGO_HOME'], testrecipe)
956        recipefile = bb_vars['FILE']
957        workdir = bb_vars['WORKDIR']
958        cargo_home = bb_vars['CARGO_HOME']
959        src_uri = bb_vars['SRC_URI'].split()
960        self.assertTrue(src_uri[0].startswith('git://'),
961                        'This test expects the %s recipe to have a git repo has its main uri' % testrecipe)
962        self.assertIn(';subpath=', src_uri[0],
963                      'This test expects the %s recipe to have a git uri with subpath' % testrecipe)
964        self.assertTrue(any([uri.startswith('crate://') for uri in src_uri]),
965                        'This test expects the %s recipe to have some crates in its src uris' % testrecipe)
966        self.assertGreaterEqual(sum(map(lambda x:x.startswith('git://'), src_uri)), 2,
967                           'This test expects the %s recipe to have several git:// uris' % testrecipe)
968        self.assertTrue(any([uri.startswith('file://') and '.patch' in uri for uri in src_uri]),
969                        'This test expects the %s recipe to have a patch in its src uris' % testrecipe)
970
971        self._test_recipe_contents(recipefile, {}, ['ptest-cargo'])
972
973        # Clean up anything in the workdir/sysroot/sstate cache
974        bitbake('%s -c cleansstate' % testrecipe)
975        # Try modifying a recipe
976        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
977        self.track_for_cleanup(tempdir)
978        self.track_for_cleanup(self.workspacedir)
979        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
980        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
981        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
982        self.assertExists(os.path.join(tempdir, 'Cargo.toml'), 'Extracted source could not be found')
983        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output)
984        matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % testrecipe))
985        self.assertTrue(matches, 'bbappend not created')
986        # Test devtool status
987        result = runCmd('devtool status')
988        self.assertIn(testrecipe, result.output)
989        self.assertIn(tempdir, result.output)
990        # Check git repo
991        self._check_src_repo(tempdir)
992        # Check that the patch is correctly applied.
993        # The last commit message in the tree must contain the following note:
994        # Notes (devtool):
995        #     original patch: <patchname>
996        # ..
997        patchname = None
998        for uri in src_uri:
999            if uri.startswith('file://') and '.patch' in uri:
1000                patchname = uri.replace("file://", "").partition('.patch')[0] + '.patch'
1001        self.assertIsNotNone(patchname)
1002        result = runCmd('git -C %s log -1' % tempdir)
1003        self.assertIn("Notes (devtool):\n    original patch: %s" % patchname, result.output)
1004
1005        # Configure the recipe to check that the git dependencies are correctly patched in cargo config
1006        bitbake('-c configure %s' % testrecipe)
1007
1008        cargo_config_path = os.path.join(cargo_home, 'config')
1009        with open(cargo_config_path, "r") as f:
1010            cargo_config_contents = [line.strip('\n') for line in f.readlines()]
1011
1012        # Get back git dependencies of the recipe (ignoring the main one)
1013        # and check that they are all correctly patched to be fetched locally
1014        git_deps = [uri for uri in src_uri if uri.startswith("git://")][1:]
1015        for git_dep in git_deps:
1016            raw_url, _, raw_parms = git_dep.partition(";")
1017            parms = {}
1018            for parm in raw_parms.split(";"):
1019                name_parm, _, value_parm = parm.partition('=')
1020                parms[name_parm]=value_parm
1021            self.assertIn('protocol', parms, 'git dependencies uri should contain the "protocol" parameter')
1022            self.assertIn('name', parms, 'git dependencies uri should contain the "name" parameter')
1023            self.assertIn('destsuffix', parms, 'git dependencies uri should contain the "destsuffix" parameter')
1024            self.assertIn('type', parms, 'git dependencies uri should contain the "type" parameter')
1025            self.assertEqual(parms['type'], 'git-dependency', 'git dependencies uri should have "type=git-dependency"')
1026            raw_url = raw_url.replace("git://", '%s://' % parms['protocol'])
1027            patch_line = '[patch."%s"]' % raw_url
1028            path_patched = os.path.join(workdir, parms['destsuffix'])
1029            path_override_line = '%s = { path = "%s" }' % (parms['name'], path_patched)
1030            # Would have been better to use tomllib to read this file :/
1031            self.assertIn(patch_line, cargo_config_contents)
1032            self.assertIn(path_override_line, cargo_config_contents)
1033
1034        # Try to package the recipe
1035        bitbake('-c package_qa %s' % testrecipe)
1036
1037    def test_devtool_modify_localfiles(self):
1038        # Check preconditions
1039        testrecipe = 'lighttpd'
1040        src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split()
1041        foundlocal = False
1042        for item in src_uri:
1043            if item.startswith('file://') and '.patch' not in item:
1044                foundlocal = True
1045                break
1046        self.assertTrue(foundlocal, 'This test expects the %s recipe to fetch local files and it seems that it no longer does' % testrecipe)
1047        # Clean up anything in the workdir/sysroot/sstate cache
1048        bitbake('%s -c cleansstate' % testrecipe)
1049        # Try modifying a recipe
1050        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1051        self.track_for_cleanup(tempdir)
1052        self.track_for_cleanup(self.workspacedir)
1053        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
1054        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1055        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
1056        self.assertExists(os.path.join(tempdir, 'configure.ac'), 'Extracted source could not be found')
1057        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created')
1058        matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % testrecipe))
1059        self.assertTrue(matches, 'bbappend not created')
1060        # Test devtool status
1061        result = runCmd('devtool status')
1062        self.assertIn(testrecipe, result.output)
1063        self.assertIn(tempdir, result.output)
1064        # Try building
1065        bitbake(testrecipe)
1066
1067    def test_devtool_modify_virtual(self):
1068        # Try modifying a virtual recipe
1069        virtrecipe = 'virtual/make'
1070        realrecipe = 'make'
1071        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1072        self.track_for_cleanup(tempdir)
1073        self.track_for_cleanup(self.workspacedir)
1074        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1075        result = runCmd('devtool modify %s -x %s' % (virtrecipe, tempdir))
1076        self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found')
1077        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created')
1078        matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % realrecipe))
1079        self.assertTrue(matches, 'bbappend not created %s' % result.output)
1080        # Test devtool status
1081        result = runCmd('devtool status')
1082        self.assertNotIn(virtrecipe, result.output)
1083        self.assertIn(realrecipe, result.output)
1084        # Check git repo
1085        self._check_src_repo(tempdir)
1086        # This is probably sufficient
1087
1088    def test_devtool_modify_overrides(self):
1089        # Try modifying a recipe with patches in overrides
1090        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1091        self.track_for_cleanup(tempdir)
1092        self.track_for_cleanup(self.workspacedir)
1093        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1094        result = runCmd('devtool modify devtool-patch-overrides -x %s' % (tempdir))
1095
1096        self._check_src_repo(tempdir)
1097        source = os.path.join(tempdir, "source")
1098        def check(branch, expected):
1099            runCmd('git -C %s checkout %s' % (tempdir, branch))
1100            with open(source, "rt") as f:
1101                content = f.read()
1102            self.assertEqual(content, expected)
1103        if self.td["MACHINE"] == "qemux86":
1104            check('devtool', 'This is a test for qemux86\n')
1105        elif self.td["MACHINE"] == "qemuarm":
1106            check('devtool', 'This is a test for qemuarm\n')
1107        else:
1108            check('devtool', 'This is a test for something\n')
1109        check('devtool-no-overrides', 'This is a test for something\n')
1110        check('devtool-override-qemuarm', 'This is a test for qemuarm\n')
1111        check('devtool-override-qemux86', 'This is a test for qemux86\n')
1112
1113    def test_devtool_modify_multiple_sources(self):
1114        # This test check that recipes fetching several sources can be used with devtool modify/build
1115        # Check preconditions
1116        testrecipe = 'bzip2'
1117        src_uri = get_bb_var('SRC_URI', testrecipe)
1118        src1 = 'https://' in src_uri
1119        src2 = 'git://' in src_uri
1120        self.assertTrue(src1 and src2, 'This test expects the %s recipe to fetch both a git source and a tarball and it seems that it no longer does' % testrecipe)
1121        # Clean up anything in the workdir/sysroot/sstate cache
1122        bitbake('%s -c cleansstate' % testrecipe)
1123        # Try modifying a recipe
1124        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1125        self.track_for_cleanup(tempdir)
1126        self.track_for_cleanup(self.workspacedir)
1127        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
1128        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1129        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
1130        self.assertEqual(result.status, 0, "Could not modify recipe %s. Output: %s" % (testrecipe, result.output))
1131        # Test devtool status
1132        result = runCmd('devtool status')
1133        self.assertIn(testrecipe, result.output)
1134        self.assertIn(tempdir, result.output)
1135        # Try building
1136        result = bitbake(testrecipe)
1137        self.assertEqual(result.status, 0, "Bitbake failed, exit code %s, output %s" % (result.status, result.output))
1138
1139class DevtoolUpdateTests(DevtoolBase):
1140
1141    def test_devtool_update_recipe(self):
1142        # Check preconditions
1143        testrecipe = 'minicom'
1144        bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe)
1145        recipefile = bb_vars['FILE']
1146        src_uri = bb_vars['SRC_URI']
1147        self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe)
1148        self._check_repo_status(os.path.dirname(recipefile), [])
1149        # First, modify a recipe
1150        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1151        self.track_for_cleanup(tempdir)
1152        self.track_for_cleanup(self.workspacedir)
1153        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1154        # (don't bother with cleaning the recipe on teardown, we won't be building it)
1155        # We don't use -x here so that we test the behaviour of devtool modify without it
1156        result = runCmd('devtool modify %s %s' % (testrecipe, tempdir))
1157        # Check git repo
1158        self._check_src_repo(tempdir)
1159        # Add a couple of commits
1160        # FIXME: this only tests adding, need to also test update and remove
1161        result = runCmd('echo "Additional line" >> README', cwd=tempdir)
1162        result = runCmd('git commit -a -m "Change the README"', cwd=tempdir)
1163        result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir)
1164        result = runCmd('git add devtool-new-file', cwd=tempdir)
1165        result = runCmd('git commit -m "Add a new file"', cwd=tempdir)
1166        self.add_command_to_tearDown('cd %s; rm %s/*.patch; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
1167        result = runCmd('devtool update-recipe %s' % testrecipe)
1168        result = runCmd('git add minicom', cwd=os.path.dirname(recipefile))
1169        expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)),
1170                           ('A ', '.*/0001-Change-the-README.patch$'),
1171                           ('A ', '.*/0002-Add-a-new-file.patch$')]
1172        self._check_repo_status(os.path.dirname(recipefile), expected_status)
1173
1174    def test_devtool_update_recipe_git(self):
1175        # Check preconditions
1176        testrecipe = 'mtd-utils-selftest'
1177        bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe)
1178        recipefile = bb_vars['FILE']
1179        src_uri = bb_vars['SRC_URI']
1180        self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)
1181        patches = []
1182        for entry in src_uri.split():
1183            if entry.startswith('file://') and entry.endswith('.patch'):
1184                patches.append(entry[7:].split(';')[0])
1185        self.assertGreater(len(patches), 0, 'The %s recipe does not appear to contain any patches, so this test will not be effective' % testrecipe)
1186        self._check_repo_status(os.path.dirname(recipefile), [])
1187        # First, modify a recipe
1188        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1189        self.track_for_cleanup(tempdir)
1190        self.track_for_cleanup(self.workspacedir)
1191        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1192        # (don't bother with cleaning the recipe on teardown, we won't be building it)
1193        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
1194        # Check git repo
1195        self._check_src_repo(tempdir)
1196        # Add a couple of commits
1197        # FIXME: this only tests adding, need to also test update and remove
1198        result = runCmd('echo "# Additional line" >> Makefile.am', cwd=tempdir)
1199        result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempdir)
1200        result = runCmd('echo "A new file" > devtool-new-file', cwd=tempdir)
1201        result = runCmd('git add devtool-new-file', cwd=tempdir)
1202        result = runCmd('git commit -m "Add a new file"', cwd=tempdir)
1203        self.add_command_to_tearDown('cd %s; rm -rf %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
1204        result = runCmd('devtool update-recipe -m srcrev %s' % testrecipe)
1205        expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile))] + \
1206                          [(' D', '.*/%s$' % patch) for patch in patches]
1207        self._check_repo_status(os.path.dirname(recipefile), expected_status)
1208
1209        result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile))
1210        addlines = ['SRCREV = ".*"', 'SRC_URI = "git://git.infradead.org/mtd-utils.git;branch=master"']
1211        srcurilines = src_uri.split()
1212        srcurilines[0] = 'SRC_URI = "' + srcurilines[0]
1213        srcurilines.append('"')
1214        removelines = ['SRCREV = ".*"'] + srcurilines
1215        self._check_diff(result.output, addlines, removelines)
1216        # Now try with auto mode
1217        runCmd('cd %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, os.path.basename(recipefile)))
1218        result = runCmd('devtool update-recipe %s' % testrecipe)
1219        result = runCmd('git rev-parse --show-toplevel', cwd=os.path.dirname(recipefile))
1220        topleveldir = result.output.strip()
1221        relpatchpath = os.path.join(os.path.relpath(os.path.dirname(recipefile), topleveldir), testrecipe)
1222        expected_status = [(' M', os.path.relpath(recipefile, topleveldir)),
1223                           ('??', '%s/0001-Change-the-Makefile.patch' % relpatchpath),
1224                           ('??', '%s/0002-Add-a-new-file.patch' % relpatchpath)]
1225        self._check_repo_status(os.path.dirname(recipefile), expected_status)
1226
1227    def test_devtool_update_recipe_append(self):
1228        # Check preconditions
1229        testrecipe = 'mdadm'
1230        bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe)
1231        recipefile = bb_vars['FILE']
1232        src_uri = bb_vars['SRC_URI']
1233        self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe)
1234        self._check_repo_status(os.path.dirname(recipefile), [])
1235        # First, modify a recipe
1236        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1237        tempsrcdir = os.path.join(tempdir, 'source')
1238        templayerdir = os.path.join(tempdir, 'layer')
1239        self.track_for_cleanup(tempdir)
1240        self.track_for_cleanup(self.workspacedir)
1241        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1242        # (don't bother with cleaning the recipe on teardown, we won't be building it)
1243        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir))
1244        # Check git repo
1245        self._check_src_repo(tempsrcdir)
1246        # Add a commit
1247        result = runCmd("sed 's!\\(#define VERSION\\W*\"[^\"]*\\)\"!\\1-custom\"!' -i ReadMe.c", cwd=tempsrcdir)
1248        result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir)
1249        self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (os.path.dirname(recipefile), testrecipe))
1250        # Create a temporary layer and add it to bblayers.conf
1251        self._create_temp_layer(templayerdir, True, 'selftestupdaterecipe')
1252        # Create the bbappend
1253        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))
1254        self.assertNotIn('WARNING:', result.output)
1255        # Check recipe is still clean
1256        self._check_repo_status(os.path.dirname(recipefile), [])
1257        # Check bbappend was created
1258        splitpath = os.path.dirname(recipefile).split(os.sep)
1259        appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1])
1260        bbappendfile = self._check_bbappend(testrecipe, recipefile, appenddir)
1261        patchfile = os.path.join(appenddir, testrecipe, '0001-Add-our-custom-version.patch')
1262        self.assertExists(patchfile, 'Patch file not created')
1263
1264        # Check bbappend contents
1265        expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n',
1266                         '\n',
1267                         'SRC_URI += "file://0001-Add-our-custom-version.patch"\n',
1268                         '\n']
1269        with open(bbappendfile, 'r') as f:
1270            self.assertEqual(expectedlines, f.readlines())
1271
1272        # Check we can run it again and bbappend isn't modified
1273        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))
1274        with open(bbappendfile, 'r') as f:
1275            self.assertEqual(expectedlines, f.readlines())
1276        # Drop new commit and check patch gets deleted
1277        result = runCmd('git reset HEAD^', cwd=tempsrcdir)
1278        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))
1279        self.assertNotExists(patchfile, 'Patch file not deleted')
1280        expectedlines2 = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n',
1281                         '\n']
1282        with open(bbappendfile, 'r') as f:
1283            self.assertEqual(expectedlines2, f.readlines())
1284        # Put commit back and check we can run it if layer isn't in bblayers.conf
1285        os.remove(bbappendfile)
1286        result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir)
1287        result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir)
1288        result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir))
1289        self.assertIn('WARNING: Specified layer is not currently enabled in bblayers.conf', result.output)
1290        self.assertExists(patchfile, 'Patch file not created (with disabled layer)')
1291        with open(bbappendfile, 'r') as f:
1292            self.assertEqual(expectedlines, f.readlines())
1293        # Deleting isn't expected to work under these circumstances
1294
1295    def test_devtool_update_recipe_append_git(self):
1296        # Check preconditions
1297        testrecipe = 'mtd-utils-selftest'
1298        bb_vars = get_bb_vars(['FILE', 'SRC_URI', 'LAYERSERIES_CORENAMES'], testrecipe)
1299        recipefile = bb_vars['FILE']
1300        src_uri = bb_vars['SRC_URI']
1301        corenames = bb_vars['LAYERSERIES_CORENAMES']
1302        self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe)
1303        for entry in src_uri.split():
1304            if entry.startswith('git://'):
1305                git_uri = entry
1306                break
1307        self._check_repo_status(os.path.dirname(recipefile), [])
1308        # First, modify a recipe
1309        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1310        tempsrcdir = os.path.join(tempdir, 'source')
1311        templayerdir = os.path.join(tempdir, 'layer')
1312        self.track_for_cleanup(tempdir)
1313        self.track_for_cleanup(self.workspacedir)
1314        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1315        # (don't bother with cleaning the recipe on teardown, we won't be building it)
1316        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir))
1317        # Check git repo
1318        self._check_src_repo(tempsrcdir)
1319        # Add a commit
1320        result = runCmd('echo "# Additional line" >> Makefile.am', cwd=tempsrcdir)
1321        result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir)
1322        self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (os.path.dirname(recipefile), testrecipe))
1323        # Create a temporary layer
1324        os.makedirs(os.path.join(templayerdir, 'conf'))
1325        with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f:
1326            f.write('BBPATH .= ":${LAYERDIR}"\n')
1327            f.write('BBFILES += "${LAYERDIR}/recipes-*/*/*.bbappend"\n')
1328            f.write('BBFILE_COLLECTIONS += "oeselftesttemplayer"\n')
1329            f.write('BBFILE_PATTERN_oeselftesttemplayer = "^${LAYERDIR}/"\n')
1330            f.write('BBFILE_PRIORITY_oeselftesttemplayer = "999"\n')
1331            f.write('BBFILE_PATTERN_IGNORE_EMPTY_oeselftesttemplayer = "1"\n')
1332            f.write('LAYERSERIES_COMPAT_oeselftesttemplayer = "%s"\n' % corenames)
1333        self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir)
1334        result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir)
1335        # Create the bbappend
1336        result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir))
1337        self.assertNotIn('WARNING:', result.output)
1338        # Check recipe is still clean
1339        self._check_repo_status(os.path.dirname(recipefile), [])
1340        # Check bbappend was created
1341        splitpath = os.path.dirname(recipefile).split(os.sep)
1342        appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1])
1343        bbappendfile = self._check_bbappend(testrecipe, recipefile, appenddir)
1344        self.assertNotExists(os.path.join(appenddir, testrecipe), 'Patch directory should not be created')
1345
1346        # Check bbappend contents
1347        result = runCmd('git rev-parse HEAD', cwd=tempsrcdir)
1348        expectedlines = set(['SRCREV = "%s"\n' % result.output,
1349                             '\n',
1350                             'SRC_URI = "%s"\n' % git_uri,
1351                             '\n'])
1352        with open(bbappendfile, 'r') as f:
1353            self.assertEqual(expectedlines, set(f.readlines()))
1354
1355        # Check we can run it again and bbappend isn't modified
1356        result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir))
1357        with open(bbappendfile, 'r') as f:
1358            self.assertEqual(expectedlines, set(f.readlines()))
1359        # Drop new commit and check SRCREV changes
1360        result = runCmd('git reset HEAD^', cwd=tempsrcdir)
1361        result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir))
1362        self.assertNotExists(os.path.join(appenddir, testrecipe), 'Patch directory should not be created')
1363        result = runCmd('git rev-parse HEAD', cwd=tempsrcdir)
1364        expectedlines = set(['SRCREV = "%s"\n' % result.output,
1365                             '\n',
1366                             'SRC_URI = "%s"\n' % git_uri,
1367                             '\n'])
1368        with open(bbappendfile, 'r') as f:
1369            self.assertEqual(expectedlines, set(f.readlines()))
1370        # Put commit back and check we can run it if layer isn't in bblayers.conf
1371        os.remove(bbappendfile)
1372        result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir)
1373        result = runCmd('bitbake-layers remove-layer %s' % templayerdir, cwd=self.builddir)
1374        result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir))
1375        self.assertIn('WARNING: Specified layer is not currently enabled in bblayers.conf', result.output)
1376        self.assertNotExists(os.path.join(appenddir, testrecipe), 'Patch directory should not be created')
1377        result = runCmd('git rev-parse HEAD', cwd=tempsrcdir)
1378        expectedlines = set(['SRCREV = "%s"\n' % result.output,
1379                             '\n',
1380                             'SRC_URI = "%s"\n' % git_uri,
1381                             '\n'])
1382        with open(bbappendfile, 'r') as f:
1383            self.assertEqual(expectedlines, set(f.readlines()))
1384        # Deleting isn't expected to work under these circumstances
1385
1386    def test_devtool_update_recipe_local_files(self):
1387        """Check that local source files are copied over instead of patched"""
1388        testrecipe = 'makedevs'
1389        recipefile = get_bb_var('FILE', testrecipe)
1390        # Setup srctree for modifying the recipe
1391        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1392        self.track_for_cleanup(tempdir)
1393        self.track_for_cleanup(self.workspacedir)
1394        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1395        # (don't bother with cleaning the recipe on teardown, we won't be
1396        # building it)
1397        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
1398        # Check git repo
1399        self._check_src_repo(tempdir)
1400        # Try building just to ensure we haven't broken that
1401        bitbake("%s" % testrecipe)
1402        # Edit / commit local source
1403        runCmd('echo "/* Foobar */" >> oe-local-files/makedevs.c', cwd=tempdir)
1404        runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir)
1405        runCmd('echo "Bar" > new-file', cwd=tempdir)
1406        runCmd('git add new-file', cwd=tempdir)
1407        runCmd('git commit -m "Add new file"', cwd=tempdir)
1408        self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' %
1409                                     os.path.dirname(recipefile))
1410        runCmd('devtool update-recipe %s' % testrecipe)
1411        expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)),
1412                           (' M', '.*/makedevs/makedevs.c$'),
1413                           ('??', '.*/makedevs/new-local$'),
1414                           ('??', '.*/makedevs/0001-Add-new-file.patch$')]
1415        self._check_repo_status(os.path.dirname(recipefile), expected_status)
1416
1417    def test_devtool_update_recipe_local_files_2(self):
1418        """Check local source files support when oe-local-files is in Git"""
1419        testrecipe = 'devtool-test-local'
1420        recipefile = get_bb_var('FILE', testrecipe)
1421        recipedir = os.path.dirname(recipefile)
1422        result = runCmd('git status --porcelain .', cwd=recipedir)
1423        if result.output.strip():
1424            self.fail('Recipe directory for %s contains uncommitted changes' % testrecipe)
1425        # Setup srctree for modifying the recipe
1426        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1427        self.track_for_cleanup(tempdir)
1428        self.track_for_cleanup(self.workspacedir)
1429        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1430        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
1431        # Check git repo
1432        self._check_src_repo(tempdir)
1433        # Add oe-local-files to Git
1434        runCmd('rm oe-local-files/.gitignore', cwd=tempdir)
1435        runCmd('git add oe-local-files', cwd=tempdir)
1436        runCmd('git commit -m "Add local sources"', cwd=tempdir)
1437        # Edit / commit local sources
1438        runCmd('echo "# Foobar" >> oe-local-files/file1', cwd=tempdir)
1439        runCmd('git commit -am "Edit existing file"', cwd=tempdir)
1440        runCmd('git rm oe-local-files/file2', cwd=tempdir)
1441        runCmd('git commit -m"Remove file"', cwd=tempdir)
1442        runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir)
1443        runCmd('git add oe-local-files/new-local', cwd=tempdir)
1444        runCmd('git commit -m "Add new local file"', cwd=tempdir)
1445        runCmd('echo "Gar" > new-file', cwd=tempdir)
1446        runCmd('git add new-file', cwd=tempdir)
1447        runCmd('git commit -m "Add new file"', cwd=tempdir)
1448        self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' %
1449                                     os.path.dirname(recipefile))
1450        # Checkout unmodified file to working copy -> devtool should still pick
1451        # the modified version from HEAD
1452        runCmd('git checkout HEAD^ -- oe-local-files/file1', cwd=tempdir)
1453        runCmd('devtool update-recipe %s' % testrecipe)
1454        expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)),
1455                           (' M', '.*/file1$'),
1456                           (' D', '.*/file2$'),
1457                           ('??', '.*/new-local$'),
1458                           ('??', '.*/0001-Add-new-file.patch$')]
1459        self._check_repo_status(os.path.dirname(recipefile), expected_status)
1460
1461    def test_devtool_update_recipe_with_gitignore(self):
1462        # First, modify the recipe
1463        testrecipe = 'devtool-test-ignored'
1464        bb_vars = get_bb_vars(['FILE'], testrecipe)
1465        recipefile = bb_vars['FILE']
1466        patchfile = os.path.join(os.path.dirname(recipefile), testrecipe, testrecipe + '.patch')
1467        newpatchfile = os.path.join(os.path.dirname(recipefile), testrecipe, testrecipe + '.patch.expected')
1468        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1469        self.track_for_cleanup(tempdir)
1470        self.track_for_cleanup(self.workspacedir)
1471        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1472        # (don't bother with cleaning the recipe on teardown, we won't be building it)
1473        result = runCmd('devtool modify %s' % testrecipe)
1474        self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
1475        result = runCmd('devtool finish --force-patch-refresh %s meta-selftest' % testrecipe)
1476        # Check recipe got changed as expected
1477        with open(newpatchfile, 'r') as f:
1478            desiredlines = f.readlines()
1479        with open(patchfile, 'r') as f:
1480            newlines = f.readlines()
1481        # Ignore the initial lines, because oe-selftest creates own meta-selftest repo
1482        # which changes the metadata subject which is added into the patch, but keep
1483        # .patch.expected as it is in case someone runs devtool finish --force-patch-refresh
1484        # devtool-test-ignored manually, then it should generate exactly the same .patch file
1485        self.assertEqual(desiredlines[5:], newlines[5:])
1486
1487    def test_devtool_update_recipe_long_filename(self):
1488        # First, modify the recipe
1489        testrecipe = 'devtool-test-long-filename'
1490        bb_vars = get_bb_vars(['FILE'], testrecipe)
1491        recipefile = bb_vars['FILE']
1492        patchfilename = '0001-I-ll-patch-you-only-if-devtool-lets-me-to-do-it-corr.patch'
1493        patchfile = os.path.join(os.path.dirname(recipefile), testrecipe, patchfilename)
1494        newpatchfile = os.path.join(os.path.dirname(recipefile), testrecipe, patchfilename + '.expected')
1495        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1496        self.track_for_cleanup(tempdir)
1497        self.track_for_cleanup(self.workspacedir)
1498        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1499        # (don't bother with cleaning the recipe on teardown, we won't be building it)
1500        result = runCmd('devtool modify %s' % testrecipe)
1501        self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
1502        result = runCmd('devtool finish --force-patch-refresh %s meta-selftest' % testrecipe)
1503        # Check recipe got changed as expected
1504        with open(newpatchfile, 'r') as f:
1505            desiredlines = f.readlines()
1506        with open(patchfile, 'r') as f:
1507            newlines = f.readlines()
1508        # Ignore the initial lines, because oe-selftest creates own meta-selftest repo
1509        # which changes the metadata subject which is added into the patch, but keep
1510        # .patch.expected as it is in case someone runs devtool finish --force-patch-refresh
1511        # devtool-test-ignored manually, then it should generate exactly the same .patch file
1512        self.assertEqual(desiredlines[5:], newlines[5:])
1513
1514    def test_devtool_update_recipe_local_files_3(self):
1515        # First, modify the recipe
1516        testrecipe = 'devtool-test-localonly'
1517        bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe)
1518        recipefile = bb_vars['FILE']
1519        src_uri = bb_vars['SRC_URI']
1520        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1521        self.track_for_cleanup(tempdir)
1522        self.track_for_cleanup(self.workspacedir)
1523        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1524        # (don't bother with cleaning the recipe on teardown, we won't be building it)
1525        result = runCmd('devtool modify %s' % testrecipe)
1526        # Modify one file
1527        runCmd('echo "Another line" >> file2', cwd=os.path.join(self.workspacedir, 'sources', testrecipe, 'oe-local-files'))
1528        self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
1529        result = runCmd('devtool update-recipe %s' % testrecipe)
1530        expected_status = [(' M', '.*/%s/file2$' % testrecipe)]
1531        self._check_repo_status(os.path.dirname(recipefile), expected_status)
1532
1533    def test_devtool_update_recipe_local_patch_gz(self):
1534        # First, modify the recipe
1535        testrecipe = 'devtool-test-patch-gz'
1536        if get_bb_var('DISTRO') == 'poky-tiny':
1537            self.skipTest("The DISTRO 'poky-tiny' does not provide the dependencies needed by %s" % testrecipe)
1538        bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe)
1539        recipefile = bb_vars['FILE']
1540        src_uri = bb_vars['SRC_URI']
1541        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1542        self.track_for_cleanup(tempdir)
1543        self.track_for_cleanup(self.workspacedir)
1544        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1545        # (don't bother with cleaning the recipe on teardown, we won't be building it)
1546        result = runCmd('devtool modify %s' % testrecipe)
1547        # Modify one file
1548        srctree = os.path.join(self.workspacedir, 'sources', testrecipe)
1549        runCmd('echo "Another line" >> README', cwd=srctree)
1550        runCmd('git commit -a --amend --no-edit --no-verify', cwd=srctree)
1551        self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
1552        result = runCmd('devtool update-recipe %s' % testrecipe)
1553        expected_status = [(' M', '.*/%s/readme.patch.gz$' % testrecipe)]
1554        self._check_repo_status(os.path.dirname(recipefile), expected_status)
1555        patch_gz = os.path.join(os.path.dirname(recipefile), testrecipe, 'readme.patch.gz')
1556        result = runCmd('file %s' % patch_gz)
1557        if 'gzip compressed data' not in result.output:
1558            self.fail('New patch file is not gzipped - file reports:\n%s' % result.output)
1559
1560    def test_devtool_update_recipe_local_files_subdir(self):
1561        # Try devtool update-recipe on a recipe that has a file with subdir= set in
1562        # SRC_URI such that it overwrites a file that was in an archive that
1563        # was also in SRC_URI
1564        # First, modify the recipe
1565        testrecipe = 'devtool-test-subdir'
1566        bb_vars = get_bb_vars(['FILE', 'SRC_URI'], testrecipe)
1567        recipefile = bb_vars['FILE']
1568        src_uri = bb_vars['SRC_URI']
1569        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1570        self.track_for_cleanup(tempdir)
1571        self.track_for_cleanup(self.workspacedir)
1572        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1573        # (don't bother with cleaning the recipe on teardown, we won't be building it)
1574        result = runCmd('devtool modify %s' % testrecipe)
1575        testfile = os.path.join(self.workspacedir, 'sources', testrecipe, 'testfile')
1576        self.assertExists(testfile, 'Extracted source could not be found')
1577        with open(testfile, 'r') as f:
1578            contents = f.read().rstrip()
1579        self.assertEqual(contents, 'Modified version', 'File has apparently not been overwritten as it should have been')
1580        # Test devtool update-recipe without modifying any files
1581        self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile)))
1582        result = runCmd('devtool update-recipe %s' % testrecipe)
1583        expected_status = []
1584        self._check_repo_status(os.path.dirname(recipefile), expected_status)
1585
1586    def test_devtool_finish_modify_git_subdir(self):
1587        # Check preconditions
1588        testrecipe = 'dos2unix'
1589        self.append_config('ERROR_QA:remove:pn-dos2unix = "patch-status"\n')
1590        bb_vars = get_bb_vars(['SRC_URI', 'S', 'WORKDIR', 'FILE'], testrecipe)
1591        self.assertIn('git://', bb_vars['SRC_URI'], 'This test expects the %s recipe to be a git recipe' % testrecipe)
1592        workdir_git = '%s/git/' % bb_vars['WORKDIR']
1593        if not bb_vars['S'].startswith(workdir_git):
1594            self.fail('This test expects the %s recipe to be building from a subdirectory of the git repo' % testrecipe)
1595        subdir = bb_vars['S'].split(workdir_git, 1)[1]
1596        # Clean up anything in the workdir/sysroot/sstate cache
1597        bitbake('%s -c cleansstate' % testrecipe)
1598        # Try modifying a recipe
1599        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1600        self.track_for_cleanup(tempdir)
1601        self.track_for_cleanup(self.workspacedir)
1602        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
1603        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1604        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
1605        testsrcfile = os.path.join(tempdir, subdir, 'dos2unix.c')
1606        self.assertExists(testsrcfile, 'Extracted source could not be found')
1607        self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output)
1608        self.assertNotExists(os.path.join(tempdir, subdir, '.git'), 'Subdirectory has been initialised as a git repo')
1609        # Check git repo
1610        self._check_src_repo(tempdir)
1611        # Modify file
1612        runCmd("sed -i '1s:^:/* Add a comment */\\n:' %s" % testsrcfile)
1613        result = runCmd('git commit -a -m "Add a comment"', cwd=tempdir)
1614        # Now try updating original recipe
1615        recipefile = bb_vars['FILE']
1616        recipedir = os.path.dirname(recipefile)
1617        self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (recipedir, testrecipe))
1618        result = runCmd('devtool update-recipe %s' % testrecipe)
1619        expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)),
1620                           ('??', '.*/%s/%s/$' % (testrecipe, testrecipe))]
1621        self._check_repo_status(os.path.dirname(recipefile), expected_status)
1622        result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile))
1623        removelines = ['SRC_URI = "git://.*"']
1624        addlines = [
1625            'SRC_URI = "git://.* \\\\',
1626            'file://0001-Add-a-comment.patch;patchdir=.. \\\\',
1627            '"'
1628        ]
1629        self._check_diff(result.output, addlines, removelines)
1630        # Put things back so we can run devtool finish on a different layer
1631        runCmd('cd %s; rm -f %s/*.patch; git checkout .' % (recipedir, testrecipe))
1632        # Run devtool finish
1633        res = re.search('recipes-.*', recipedir)
1634        self.assertTrue(res, 'Unable to find recipe subdirectory')
1635        recipesubdir = res[0]
1636        self.add_command_to_tearDown('rm -rf %s' % os.path.join(self.testlayer_path, recipesubdir))
1637        result = runCmd('devtool finish %s meta-selftest' % testrecipe)
1638        # Check bbappend file contents
1639        appendfn = os.path.join(self.testlayer_path, recipesubdir, '%s_%%.bbappend' % testrecipe)
1640        with open(appendfn, 'r') as f:
1641            appendlines = f.readlines()
1642        expected_appendlines = [
1643            'FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n',
1644            '\n',
1645            'SRC_URI += "file://0001-Add-a-comment.patch;patchdir=.."\n',
1646            '\n'
1647        ]
1648        self.assertEqual(appendlines, expected_appendlines)
1649        self.assertExists(os.path.join(os.path.dirname(appendfn), testrecipe, '0001-Add-a-comment.patch'))
1650        # Try building
1651        bitbake('%s -c patch' % testrecipe)
1652
1653    def test_devtool_git_submodules(self):
1654        # This tests if we can add a patch in a git submodule and extract it properly using devtool finish
1655        # Check preconditions
1656        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
1657        self.track_for_cleanup(self.workspacedir)
1658        recipe = 'vulkan-samples'
1659        src_uri = get_bb_var('SRC_URI', recipe)
1660        self.assertIn('gitsm://', src_uri, 'This test expects the %s recipe to be a git recipe with submodules' % recipe)
1661        oldrecipefile = get_bb_var('FILE', recipe)
1662        recipedir = os.path.dirname(oldrecipefile)
1663        result = runCmd('git status --porcelain .', cwd=recipedir)
1664        if result.output.strip():
1665            self.fail('Recipe directory for %s contains uncommitted changes' % recipe)
1666        self.assertIn('/meta/', recipedir)
1667        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1668        self.track_for_cleanup(tempdir)
1669        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1670        result = runCmd('devtool modify %s %s' % (recipe, tempdir))
1671        self.assertExists(os.path.join(tempdir, 'CMakeLists.txt'), 'Extracted source could not be found')
1672        # Test devtool status
1673        result = runCmd('devtool status')
1674        self.assertIn(recipe, result.output)
1675        self.assertIn(tempdir, result.output)
1676        # Modify a source file in a submodule, (grab the first one)
1677        result = runCmd('git submodule --quiet foreach \'echo $sm_path\'', cwd=tempdir)
1678        submodule = result.output.splitlines()[0]
1679        submodule_path = os.path.join(tempdir, submodule)
1680        runCmd('echo "#This is a first comment" >> testfile', cwd=submodule_path)
1681        result = runCmd('git status --porcelain . ', cwd=submodule_path)
1682        self.assertIn("testfile", result.output)
1683        runCmd('git add testfile; git commit -m "Adding a new file"', cwd=submodule_path)
1684
1685        # Try finish to the original layer
1686        self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir))
1687        runCmd('devtool finish -f %s meta' % recipe)
1688        result = runCmd('devtool status')
1689        self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t')
1690        self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish')
1691        expected_status = [(' M', '.*/%s$' % os.path.basename(oldrecipefile)),
1692                           ('??', '.*/.*-Adding-a-new-file.patch$')]
1693        self._check_repo_status(recipedir, expected_status)
1694        # Make sure the patch is added to the recipe with the correct "patchdir" option
1695        result = runCmd('git diff .', cwd=recipedir)
1696        addlines = [
1697           'file://0001-Adding-a-new-file.patch;patchdir=%s \\\\' % submodule
1698        ]
1699        self._check_diff(result.output, addlines, [])
1700
1701class DevtoolExtractTests(DevtoolBase):
1702
1703    def test_devtool_extract(self):
1704        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1705        # Try devtool extract
1706        self.track_for_cleanup(tempdir)
1707        self.track_for_cleanup(self.workspacedir)
1708        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1709        result = runCmd('devtool extract matchbox-terminal %s' % tempdir)
1710        self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found')
1711        self._check_src_repo(tempdir)
1712
1713    def test_devtool_extract_virtual(self):
1714        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1715        # Try devtool extract
1716        self.track_for_cleanup(tempdir)
1717        self.track_for_cleanup(self.workspacedir)
1718        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1719        result = runCmd('devtool extract virtual/make %s' % tempdir)
1720        self.assertExists(os.path.join(tempdir, 'Makefile.am'), 'Extracted source could not be found')
1721        self._check_src_repo(tempdir)
1722
1723    def test_devtool_reset_all(self):
1724        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1725        self.track_for_cleanup(tempdir)
1726        self.track_for_cleanup(self.workspacedir)
1727        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1728        testrecipe1 = 'mdadm'
1729        testrecipe2 = 'cronie'
1730        result = runCmd('devtool modify -x %s %s' % (testrecipe1, os.path.join(tempdir, testrecipe1)))
1731        result = runCmd('devtool modify -x %s %s' % (testrecipe2, os.path.join(tempdir, testrecipe2)))
1732        result = runCmd('devtool build %s' % testrecipe1)
1733        result = runCmd('devtool build %s' % testrecipe2)
1734        stampprefix1 = get_bb_var('STAMP', testrecipe1)
1735        self.assertTrue(stampprefix1, 'Unable to get STAMP value for recipe %s' % testrecipe1)
1736        stampprefix2 = get_bb_var('STAMP', testrecipe2)
1737        self.assertTrue(stampprefix2, 'Unable to get STAMP value for recipe %s' % testrecipe2)
1738        result = runCmd('devtool reset -a')
1739        self.assertIn(testrecipe1, result.output)
1740        self.assertIn(testrecipe2, result.output)
1741        result = runCmd('devtool status')
1742        self.assertNotIn(testrecipe1, result.output)
1743        self.assertNotIn(testrecipe2, result.output)
1744        matches1 = glob.glob(stampprefix1 + '*')
1745        self.assertFalse(matches1, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe1)
1746        matches2 = glob.glob(stampprefix2 + '*')
1747        self.assertFalse(matches2, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe2)
1748
1749    @OETestTag("runqemu")
1750    def test_devtool_deploy_target(self):
1751        self._check_runqemu_prerequisites()
1752        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
1753        # Definitions
1754        testrecipe = 'mdadm'
1755        testfile = '/sbin/mdadm'
1756        testimage = 'oe-selftest-image'
1757        testcommand = '/sbin/mdadm --help'
1758        # Build an image to run
1759        bitbake("%s qemu-native qemu-helper-native" % testimage)
1760        deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE')
1761        self.add_command_to_tearDown('bitbake -c clean %s' % testimage)
1762        self.add_command_to_tearDown('rm -f %s/%s*' % (deploy_dir_image, testimage))
1763        # Clean recipe so the first deploy will fail
1764        bitbake("%s -c clean" % testrecipe)
1765        # Try devtool modify
1766        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1767        self.track_for_cleanup(tempdir)
1768        self.track_for_cleanup(self.workspacedir)
1769        self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe)
1770        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1771        result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir))
1772        # Test that deploy-target at this point fails (properly)
1773        result = runCmd('devtool deploy-target -n %s root@localhost' % testrecipe, ignore_status=True)
1774        self.assertNotEqual(result.output, 0, 'devtool deploy-target should have failed, output: %s' % result.output)
1775        self.assertNotIn(result.output, 'Traceback', 'devtool deploy-target should have failed with a proper error not a traceback, output: %s' % result.output)
1776        result = runCmd('devtool build %s' % testrecipe)
1777        # First try a dry-run of deploy-target
1778        result = runCmd('devtool deploy-target -n %s root@localhost' % testrecipe)
1779        self.assertIn('  %s' % testfile, result.output)
1780        # Boot the image
1781        with runqemu(testimage) as qemu:
1782            # Now really test deploy-target
1783            result = runCmd('devtool deploy-target -c %s root@%s' % (testrecipe, qemu.ip))
1784            # Run a test command to see if it was installed properly
1785            sshargs = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
1786            result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, testcommand))
1787            # Check if it deployed all of the files with the right ownership/perms
1788            # First look on the host - need to do this under pseudo to get the correct ownership/perms
1789            bb_vars = get_bb_vars(['D', 'FAKEROOTENV', 'FAKEROOTCMD'], testrecipe)
1790            installdir = bb_vars['D']
1791            fakerootenv = bb_vars['FAKEROOTENV']
1792            fakerootcmd = bb_vars['FAKEROOTCMD']
1793            result = runCmd('%s %s find . -type f -exec ls -l {} \\;' % (fakerootenv, fakerootcmd), cwd=installdir)
1794            filelist1 = self._process_ls_output(result.output)
1795
1796            # Now look on the target
1797            tempdir2 = tempfile.mkdtemp(prefix='devtoolqa')
1798            self.track_for_cleanup(tempdir2)
1799            tmpfilelist = os.path.join(tempdir2, 'files.txt')
1800            with open(tmpfilelist, 'w') as f:
1801                for line in filelist1:
1802                    splitline = line.split()
1803                    f.write(splitline[-1] + '\n')
1804            result = runCmd('cat %s | ssh -q %s root@%s \'xargs ls -l\'' % (tmpfilelist, sshargs, qemu.ip))
1805            filelist2 = self._process_ls_output(result.output)
1806            filelist1.sort(key=lambda item: item.split()[-1])
1807            filelist2.sort(key=lambda item: item.split()[-1])
1808            self.assertEqual(filelist1, filelist2)
1809            # Test undeploy-target
1810            result = runCmd('devtool undeploy-target -c %s root@%s' % (testrecipe, qemu.ip))
1811            result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, testcommand), ignore_status=True)
1812            self.assertNotEqual(result, 0, 'undeploy-target did not remove command as it should have')
1813
1814    def test_devtool_build_image(self):
1815        """Test devtool build-image plugin"""
1816        # Check preconditions
1817        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
1818        image = 'core-image-minimal'
1819        self.track_for_cleanup(self.workspacedir)
1820        self.add_command_to_tearDown('bitbake -c clean %s' % image)
1821        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1822        bitbake('%s -c clean' % image)
1823        # Add target and native recipes to workspace
1824        recipes = ['mdadm', 'parted-native']
1825        for recipe in recipes:
1826            tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1827            self.track_for_cleanup(tempdir)
1828            self.add_command_to_tearDown('bitbake -c clean %s' % recipe)
1829            runCmd('devtool modify %s -x %s' % (recipe, tempdir))
1830        # Try to build image
1831        result = runCmd('devtool build-image %s' % image)
1832        self.assertNotEqual(result, 0, 'devtool build-image failed')
1833        # Check if image contains expected packages
1834        deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE')
1835        image_link_name = get_bb_var('IMAGE_LINK_NAME', image)
1836        reqpkgs = [item for item in recipes if not item.endswith('-native')]
1837        with open(os.path.join(deploy_dir_image, image_link_name + '.manifest'), 'r') as f:
1838            for line in f:
1839                splitval = line.split()
1840                if splitval:
1841                    pkg = splitval[0]
1842                    if pkg in reqpkgs:
1843                        reqpkgs.remove(pkg)
1844        if reqpkgs:
1845            self.fail('The following packages were not present in the image as expected: %s' % ', '.join(reqpkgs))
1846
1847class DevtoolUpgradeTests(DevtoolBase):
1848
1849    def setUp(self):
1850        super().setUp()
1851        try:
1852            runCmd("git config --global user.name")
1853            runCmd("git config --global user.email")
1854        except:
1855            self.skip("Git user.name and user.email must be set")
1856
1857    def test_devtool_upgrade(self):
1858        # Check preconditions
1859        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
1860        self.track_for_cleanup(self.workspacedir)
1861        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1862        # Check parameters
1863        result = runCmd('devtool upgrade -h')
1864        for param in 'recipename srctree --version -V --branch -b --keep-temp --no-patch'.split():
1865            self.assertIn(param, result.output)
1866        # For the moment, we are using a real recipe.
1867        recipe = 'devtool-upgrade-test1'
1868        version = '1.6.0'
1869        oldrecipefile = get_bb_var('FILE', recipe)
1870        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1871        self.track_for_cleanup(tempdir)
1872        # Check that recipe is not already under devtool control
1873        result = runCmd('devtool status')
1874        self.assertNotIn(recipe, result.output)
1875        # Check upgrade. Code does not check if new PV is older or newer that current PV, so, it may be that
1876        # we are downgrading instead of upgrading.
1877        result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, version))
1878        # Check if srctree at least is populated
1879        self.assertTrue(len(os.listdir(tempdir)) > 0, 'srctree (%s) should be populated with new (%s) source code' % (tempdir, version))
1880        # Check new recipe subdirectory is present
1881        self.assertExists(os.path.join(self.workspacedir, 'recipes', recipe, '%s-%s' % (recipe, version)), 'Recipe folder should exist')
1882        # Check new recipe file is present
1883        newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, '%s_%s.bb' % (recipe, version))
1884        self.assertExists(newrecipefile, 'Recipe file should exist after upgrade')
1885        # Check devtool status and make sure recipe is present
1886        result = runCmd('devtool status')
1887        self.assertIn(recipe, result.output)
1888        self.assertIn(tempdir, result.output)
1889        # Check recipe got changed as expected
1890        with open(oldrecipefile + '.upgraded', 'r') as f:
1891            desiredlines = f.readlines()
1892        with open(newrecipefile, 'r') as f:
1893            newlines = f.readlines()
1894        self.assertEqual(desiredlines, newlines)
1895        # Check devtool reset recipe
1896        result = runCmd('devtool reset %s -n' % recipe)
1897        result = runCmd('devtool status')
1898        self.assertNotIn(recipe, result.output)
1899        self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after resetting')
1900
1901    def test_devtool_upgrade_git(self):
1902        # Check preconditions
1903        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
1904        self.track_for_cleanup(self.workspacedir)
1905        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1906        recipe = 'devtool-upgrade-test2'
1907        commit = '6cc6077a36fe2648a5f993fe7c16c9632f946517'
1908        oldrecipefile = get_bb_var('FILE', recipe)
1909        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1910        self.track_for_cleanup(tempdir)
1911        # Check that recipe is not already under devtool control
1912        result = runCmd('devtool status')
1913        self.assertNotIn(recipe, result.output)
1914        # Check upgrade
1915        result = runCmd('devtool upgrade %s %s -S %s' % (recipe, tempdir, commit))
1916        # Check if srctree at least is populated
1917        self.assertTrue(len(os.listdir(tempdir)) > 0, 'srctree (%s) should be populated with new (%s) source code' % (tempdir, commit))
1918        # Check new recipe file is present
1919        newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, os.path.basename(oldrecipefile))
1920        self.assertExists(newrecipefile, 'Recipe file should exist after upgrade')
1921        # Check devtool status and make sure recipe is present
1922        result = runCmd('devtool status')
1923        self.assertIn(recipe, result.output)
1924        self.assertIn(tempdir, result.output)
1925        # Check recipe got changed as expected
1926        with open(oldrecipefile + '.upgraded', 'r') as f:
1927            desiredlines = f.readlines()
1928        with open(newrecipefile, 'r') as f:
1929            newlines = f.readlines()
1930        self.assertEqual(desiredlines, newlines)
1931        # Check devtool reset recipe
1932        result = runCmd('devtool reset %s -n' % recipe)
1933        result = runCmd('devtool status')
1934        self.assertNotIn(recipe, result.output)
1935        self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after resetting')
1936
1937    def test_devtool_upgrade_drop_md5sum(self):
1938        # Check preconditions
1939        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
1940        self.track_for_cleanup(self.workspacedir)
1941        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1942        # For the moment, we are using a real recipe.
1943        recipe = 'devtool-upgrade-test3'
1944        version = '1.6.0'
1945        oldrecipefile = get_bb_var('FILE', recipe)
1946        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1947        self.track_for_cleanup(tempdir)
1948        # Check upgrade. Code does not check if new PV is older or newer that current PV, so, it may be that
1949        # we are downgrading instead of upgrading.
1950        result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, version))
1951        # Check new recipe file is present
1952        newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, '%s_%s.bb' % (recipe, version))
1953        self.assertExists(newrecipefile, 'Recipe file should exist after upgrade')
1954        # Check recipe got changed as expected
1955        with open(oldrecipefile + '.upgraded', 'r') as f:
1956            desiredlines = f.readlines()
1957        with open(newrecipefile, 'r') as f:
1958            newlines = f.readlines()
1959        self.assertEqual(desiredlines, newlines)
1960
1961    def test_devtool_upgrade_all_checksums(self):
1962        # Check preconditions
1963        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
1964        self.track_for_cleanup(self.workspacedir)
1965        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1966        # For the moment, we are using a real recipe.
1967        recipe = 'devtool-upgrade-test4'
1968        version = '1.6.0'
1969        oldrecipefile = get_bb_var('FILE', recipe)
1970        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
1971        self.track_for_cleanup(tempdir)
1972        # Check upgrade. Code does not check if new PV is older or newer that current PV, so, it may be that
1973        # we are downgrading instead of upgrading.
1974        result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, version))
1975        # Check new recipe file is present
1976        newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, '%s_%s.bb' % (recipe, version))
1977        self.assertExists(newrecipefile, 'Recipe file should exist after upgrade')
1978        # Check recipe got changed as expected
1979        with open(oldrecipefile + '.upgraded', 'r') as f:
1980            desiredlines = f.readlines()
1981        with open(newrecipefile, 'r') as f:
1982            newlines = f.readlines()
1983        self.assertEqual(desiredlines, newlines)
1984
1985    def test_devtool_layer_plugins(self):
1986        """Test that devtool can use plugins from other layers.
1987
1988        This test executes the selftest-reverse command from meta-selftest."""
1989
1990        self.track_for_cleanup(self.workspacedir)
1991        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
1992
1993        s = "Microsoft Made No Profit From Anyone's Zunes Yo"
1994        result = runCmd("devtool --quiet selftest-reverse \"%s\"" % s)
1995        self.assertEqual(result.output, s[::-1])
1996
1997    def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths):
1998        dstdir = basedstdir
1999        self.assertExists(dstdir)
2000        for p in paths:
2001            dstdir = os.path.join(dstdir, p)
2002            if not os.path.exists(dstdir):
2003                try:
2004                    os.makedirs(dstdir)
2005                except PermissionError:
2006                    return False
2007                except OSError as e:
2008                    if e.errno == errno.EROFS:
2009                        return False
2010                    else:
2011                        raise e
2012                if p == "lib":
2013                    # Can race with other tests
2014                    self.add_command_to_tearDown('rmdir --ignore-fail-on-non-empty %s' % dstdir)
2015                else:
2016                    self.track_for_cleanup(dstdir)
2017        dstfile = os.path.join(dstdir, os.path.basename(srcfile))
2018        if srcfile != dstfile:
2019            try:
2020                shutil.copy(srcfile, dstfile)
2021            except PermissionError:
2022                return False
2023            self.track_for_cleanup(dstfile)
2024        return True
2025
2026    def test_devtool_load_plugin(self):
2027        """Test that devtool loads only the first found plugin in BBPATH."""
2028
2029        self.track_for_cleanup(self.workspacedir)
2030        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
2031
2032        devtool = runCmd("which devtool")
2033        fromname = runCmd("devtool --quiet pluginfile")
2034        srcfile = fromname.output
2035        bbpath = get_bb_var('BBPATH')
2036        searchpath = bbpath.split(':') + [os.path.dirname(devtool.output)]
2037        plugincontent = []
2038        with open(srcfile) as fh:
2039            plugincontent = fh.readlines()
2040        try:
2041            self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found')
2042            searchpath = [
2043                path for path in searchpath
2044                if self._copy_file_with_cleanup(srcfile, path, 'lib', 'devtool')
2045            ]
2046            result = runCmd("devtool --quiet count")
2047            self.assertEqual(result.output, '1')
2048            result = runCmd("devtool --quiet multiloaded")
2049            self.assertEqual(result.output, "no")
2050            for path in searchpath:
2051                result = runCmd("devtool --quiet bbdir")
2052                self.assertEqual(os.path.realpath(result.output), os.path.realpath(path))
2053                os.unlink(os.path.join(result.output, 'lib', 'devtool', 'bbpath.py'))
2054        finally:
2055            with open(srcfile, 'w') as fh:
2056                fh.writelines(plugincontent)
2057
2058    def _setup_test_devtool_finish_upgrade(self):
2059        # Check preconditions
2060        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
2061        self.track_for_cleanup(self.workspacedir)
2062        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
2063        # Use a "real" recipe from meta-selftest
2064        recipe = 'devtool-upgrade-test1'
2065        oldversion = '1.5.3'
2066        newversion = '1.6.0'
2067        oldrecipefile = get_bb_var('FILE', recipe)
2068        recipedir = os.path.dirname(oldrecipefile)
2069        result = runCmd('git status --porcelain .', cwd=recipedir)
2070        if result.output.strip():
2071            self.fail('Recipe directory for %s contains uncommitted changes' % recipe)
2072        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
2073        self.track_for_cleanup(tempdir)
2074        # Check that recipe is not already under devtool control
2075        result = runCmd('devtool status')
2076        self.assertNotIn(recipe, result.output)
2077        # Do the upgrade
2078        result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, newversion))
2079        # Check devtool status and make sure recipe is present
2080        result = runCmd('devtool status')
2081        self.assertIn(recipe, result.output)
2082        self.assertIn(tempdir, result.output)
2083        # Make a change to the source
2084        result = runCmd('sed -i \'/^#include "pv.h"/a \\/* Here is a new comment *\\/\' src/pv/number.c', cwd=tempdir)
2085        result = runCmd('git status --porcelain', cwd=tempdir)
2086        self.assertIn('M src/pv/number.c', result.output)
2087        result = runCmd('git commit src/pv/number.c -m "Add a comment to the code"', cwd=tempdir)
2088        # Check if patch is there
2089        recipedir = os.path.dirname(oldrecipefile)
2090        olddir = os.path.join(recipedir, recipe + '-' + oldversion)
2091        patchfn = '0001-Add-a-note-line-to-the-quick-reference.patch'
2092        backportedpatchfn = 'backported.patch'
2093        self.assertExists(os.path.join(olddir, patchfn), 'Original patch file does not exist')
2094        self.assertExists(os.path.join(olddir, backportedpatchfn), 'Backported patch file does not exist')
2095        return recipe, oldrecipefile, recipedir, olddir, newversion, patchfn, backportedpatchfn
2096
2097    def test_devtool_finish_upgrade_origlayer(self):
2098        recipe, oldrecipefile, recipedir, olddir, newversion, patchfn, backportedpatchfn = self._setup_test_devtool_finish_upgrade()
2099        # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things)
2100        self.assertIn('/meta-selftest/', recipedir)
2101        # Try finish to the original layer
2102        self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir))
2103        result = runCmd('devtool finish %s meta-selftest' % recipe)
2104        result = runCmd('devtool status')
2105        self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t')
2106        self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish')
2107        self.assertNotExists(oldrecipefile, 'Old recipe file should have been deleted but wasn\'t')
2108        self.assertNotExists(os.path.join(olddir, patchfn), 'Old patch file should have been deleted but wasn\'t')
2109        self.assertNotExists(os.path.join(olddir, backportedpatchfn), 'Old backported patch file should have been deleted but wasn\'t')
2110        newrecipefile = os.path.join(recipedir, '%s_%s.bb' % (recipe, newversion))
2111        newdir = os.path.join(recipedir, recipe + '-' + newversion)
2112        self.assertExists(newrecipefile, 'New recipe file should have been copied into existing layer but wasn\'t')
2113        self.assertExists(os.path.join(newdir, patchfn), 'Patch file should have been copied into new directory but wasn\'t')
2114        self.assertNotExists(os.path.join(newdir, backportedpatchfn), 'Backported patch file should not have been copied into new directory but was')
2115        self.assertExists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch'), 'New patch file should have been created but wasn\'t')
2116        with open(newrecipefile, 'r') as f:
2117            newcontent = f.read()
2118        self.assertNotIn(backportedpatchfn, newcontent, "Backported patch should have been removed from the recipe but wasn't")
2119        self.assertIn(patchfn, newcontent, "Old patch should have not been removed from the recipe but was")
2120        self.assertIn("0002-Add-a-comment-to-the-code.patch", newcontent, "New patch should have been added to the recipe but wasn't")
2121        self.assertIn("http://www.ivarch.com/programs/sources/pv-${PV}.tar.gz", newcontent, "New recipe no longer has upstream source in SRC_URI")
2122
2123
2124    def test_devtool_finish_upgrade_otherlayer(self):
2125        recipe, oldrecipefile, recipedir, olddir, newversion, patchfn, backportedpatchfn = self._setup_test_devtool_finish_upgrade()
2126        # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things)
2127        self.assertIn('/meta-selftest/', recipedir)
2128        # Try finish to a different layer - should create a bbappend
2129        # This cleanup isn't strictly necessary but do it anyway just in case it goes wrong and writes to here
2130        self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir))
2131        oe_core_dir = os.path.join(get_bb_var('COREBASE'), 'meta')
2132        newrecipedir = os.path.join(oe_core_dir, 'recipes-test', 'devtool')
2133        newrecipefile = os.path.join(newrecipedir, '%s_%s.bb' % (recipe, newversion))
2134        self.track_for_cleanup(newrecipedir)
2135        result = runCmd('devtool finish %s oe-core' % recipe)
2136        result = runCmd('devtool status')
2137        self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t')
2138        self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish')
2139        self.assertExists(oldrecipefile, 'Old recipe file should not have been deleted')
2140        self.assertExists(os.path.join(olddir, patchfn), 'Old patch file should not have been deleted')
2141        self.assertExists(os.path.join(olddir, backportedpatchfn), 'Old backported patch file should not have been deleted')
2142        newdir = os.path.join(newrecipedir, recipe + '-' + newversion)
2143        self.assertExists(newrecipefile, 'New recipe file should have been copied into existing layer but wasn\'t')
2144        self.assertExists(os.path.join(newdir, patchfn), 'Patch file should have been copied into new directory but wasn\'t')
2145        self.assertNotExists(os.path.join(newdir, backportedpatchfn), 'Backported patch file should not have been copied into new directory but was')
2146        self.assertExists(os.path.join(newdir, '0002-Add-a-comment-to-the-code.patch'), 'New patch file should have been created but wasn\'t')
2147        with open(newrecipefile, 'r') as f:
2148            newcontent = f.read()
2149        self.assertNotIn(backportedpatchfn, newcontent, "Backported patch should have been removed from the recipe but wasn't")
2150        self.assertIn(patchfn, newcontent, "Old patch should have not been removed from the recipe but was")
2151        self.assertIn("0002-Add-a-comment-to-the-code.patch", newcontent, "New patch should have been added to the recipe but wasn't")
2152        self.assertIn("http://www.ivarch.com/programs/sources/pv-${PV}.tar.gz", newcontent, "New recipe no longer has upstream source in SRC_URI")
2153
2154    def _setup_test_devtool_finish_modify(self):
2155        # Check preconditions
2156        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
2157        # Try modifying a recipe
2158        self.track_for_cleanup(self.workspacedir)
2159        recipe = 'mdadm'
2160        oldrecipefile = get_bb_var('FILE', recipe)
2161        recipedir = os.path.dirname(oldrecipefile)
2162        result = runCmd('git status --porcelain .', cwd=recipedir)
2163        if result.output.strip():
2164            self.fail('Recipe directory for %s contains uncommitted changes' % recipe)
2165        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
2166        self.track_for_cleanup(tempdir)
2167        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
2168        result = runCmd('devtool modify %s %s' % (recipe, tempdir))
2169        self.assertExists(os.path.join(tempdir, 'Makefile'), 'Extracted source could not be found')
2170        # Test devtool status
2171        result = runCmd('devtool status')
2172        self.assertIn(recipe, result.output)
2173        self.assertIn(tempdir, result.output)
2174        # Make a change to the source
2175        result = runCmd('sed -i \'/^#include "mdadm.h"/a \\/* Here is a new comment *\\/\' maps.c', cwd=tempdir)
2176        result = runCmd('git status --porcelain', cwd=tempdir)
2177        self.assertIn('M maps.c', result.output)
2178        result = runCmd('git commit maps.c -m "Add a comment to the code"', cwd=tempdir)
2179        for entry in os.listdir(recipedir):
2180            filesdir = os.path.join(recipedir, entry)
2181            if os.path.isdir(filesdir):
2182                break
2183        else:
2184            self.fail('Unable to find recipe files directory for %s' % recipe)
2185        return recipe, oldrecipefile, recipedir, filesdir
2186
2187    def test_devtool_finish_modify_origlayer(self):
2188        recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify()
2189        # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things)
2190        self.assertIn('/meta/', recipedir)
2191        # Try finish to the original layer
2192        self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir))
2193        result = runCmd('devtool finish %s meta' % recipe)
2194        result = runCmd('devtool status')
2195        self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t')
2196        self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish')
2197        expected_status = [(' M', '.*/%s$' % os.path.basename(oldrecipefile)),
2198                           ('??', '.*/.*-Add-a-comment-to-the-code.patch$')]
2199        self._check_repo_status(recipedir, expected_status)
2200
2201    def test_devtool_finish_modify_otherlayer(self):
2202        recipe, oldrecipefile, recipedir, filesdir = self._setup_test_devtool_finish_modify()
2203        # Ensure the recipe is where we think it should be (so that cleanup doesn't trash things)
2204        self.assertIn('/meta/', recipedir)
2205        relpth = os.path.relpath(recipedir, os.path.join(get_bb_var('COREBASE'), 'meta'))
2206        appenddir = os.path.join(get_test_layer(), relpth)
2207        self.track_for_cleanup(appenddir)
2208        # Try finish to the original layer
2209        self.add_command_to_tearDown('rm -rf %s ; cd %s ; git checkout %s' % (recipedir, os.path.dirname(recipedir), recipedir))
2210        result = runCmd('devtool finish %s meta-selftest' % recipe)
2211        result = runCmd('devtool status')
2212        self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t')
2213        self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish')
2214        result = runCmd('git status --porcelain .', cwd=recipedir)
2215        if result.output.strip():
2216            self.fail('Recipe directory for %s contains the following unexpected changes after finish:\n%s' % (recipe, result.output.strip()))
2217        recipefn = os.path.splitext(os.path.basename(oldrecipefile))[0]
2218        recipefn = recipefn.split('_')[0] + '_%'
2219        appendfile = os.path.join(appenddir, recipefn + '.bbappend')
2220        self.assertExists(appendfile, 'bbappend %s should have been created but wasn\'t' % appendfile)
2221        newdir = os.path.join(appenddir, recipe)
2222        files = os.listdir(newdir)
2223        foundpatch = None
2224        for fn in files:
2225            if fnmatch.fnmatch(fn, '*-Add-a-comment-to-the-code.patch'):
2226                foundpatch = fn
2227        if not foundpatch:
2228            self.fail('No patch file created next to bbappend')
2229        files.remove(foundpatch)
2230        if files:
2231            self.fail('Unexpected file(s) copied next to bbappend: %s' % ', '.join(files))
2232
2233    def test_devtool_finish_update_patch(self):
2234        # This test uses a modified version of the sysdig recipe from meta-oe.
2235        # - The patches have been renamed.
2236        # - The dependencies are commented out since the recipe is not being
2237        #   built.
2238        #
2239        # The sysdig recipe is interesting in that it fetches two different Git
2240        # repositories, and there are patches for both. This leads to that
2241        # devtool will create ignore commits as it uses Git submodules to keep
2242        # track of the second repository.
2243        #
2244        # This test will verify that the ignored commits actually are ignored
2245        # when a commit in between is modified. It will also verify that the
2246        # updated patch keeps its original name.
2247
2248        # Check preconditions
2249        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
2250        # Try modifying a recipe
2251        self.track_for_cleanup(self.workspacedir)
2252        recipe = 'sysdig-selftest'
2253        recipefile = get_bb_var('FILE', recipe)
2254        recipedir = os.path.dirname(recipefile)
2255        result = runCmd('git status --porcelain .', cwd=recipedir)
2256        if result.output.strip():
2257            self.fail('Recipe directory for %s contains uncommitted changes' % recipe)
2258        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
2259        self.track_for_cleanup(tempdir)
2260        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
2261        result = runCmd('devtool modify %s %s' % (recipe, tempdir))
2262        self.add_command_to_tearDown('cd %s; rm %s/*; git checkout %s %s' % (recipedir, recipe, recipe, os.path.basename(recipefile)))
2263        self.assertExists(os.path.join(tempdir, 'CMakeLists.txt'), 'Extracted source could not be found')
2264        # Make a change to one of the existing commits
2265        result = runCmd('echo "# A comment " >> CMakeLists.txt', cwd=tempdir)
2266        result = runCmd('git status --porcelain', cwd=tempdir)
2267        self.assertIn('M CMakeLists.txt', result.output)
2268        result = runCmd('git commit --fixup HEAD^ CMakeLists.txt', cwd=tempdir)
2269        result = runCmd('git show -s --format=%s', cwd=tempdir)
2270        self.assertIn('fixup! cmake: Pass PROBE_NAME via CFLAGS', result.output)
2271        result = runCmd('GIT_SEQUENCE_EDITOR=true git rebase -i --autosquash devtool-base', cwd=tempdir)
2272        result = runCmd('devtool finish %s meta-selftest' % recipe)
2273        result = runCmd('devtool status')
2274        self.assertNotIn(recipe, result.output, 'Recipe should have been reset by finish but wasn\'t')
2275        self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipe), 'Recipe directory should not exist after finish')
2276        expected_status = [(' M', '.*/0099-cmake-Pass-PROBE_NAME-via-CFLAGS.patch$')]
2277        self._check_repo_status(recipedir, expected_status)
2278
2279    def test_devtool_rename(self):
2280        # Check preconditions
2281        self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory')
2282        self.track_for_cleanup(self.workspacedir)
2283        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
2284
2285        # First run devtool add
2286        # We already have this recipe in OE-Core, but that doesn't matter
2287        recipename = 'i2c-tools'
2288        recipever = '3.1.2'
2289        recipefile = os.path.join(self.workspacedir, 'recipes', recipename, '%s_%s.bb' % (recipename, recipever))
2290        url = 'http://downloads.yoctoproject.org/mirror/sources/i2c-tools-%s.tar.bz2' % recipever
2291        def add_recipe():
2292            result = runCmd('devtool add %s' % url)
2293            self.assertExists(recipefile, 'Expected recipe file not created')
2294            self.assertExists(os.path.join(self.workspacedir, 'sources', recipename), 'Source directory not created')
2295            checkvars = {}
2296            checkvars['S'] = None
2297            checkvars['SRC_URI'] = url.replace(recipever, '${PV}')
2298            self._test_recipe_contents(recipefile, checkvars, [])
2299        add_recipe()
2300        # Now rename it - change both name and version
2301        newrecipename = 'mynewrecipe'
2302        newrecipever = '456'
2303        newrecipefile = os.path.join(self.workspacedir, 'recipes', newrecipename, '%s_%s.bb' % (newrecipename, newrecipever))
2304        result = runCmd('devtool rename %s %s -V %s' % (recipename, newrecipename, newrecipever))
2305        self.assertExists(newrecipefile, 'Recipe file not renamed')
2306        self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipename), 'Old recipe directory still exists')
2307        newsrctree = os.path.join(self.workspacedir, 'sources', newrecipename)
2308        self.assertExists(newsrctree, 'Source directory not renamed')
2309        checkvars = {}
2310        checkvars['S'] = '${WORKDIR}/%s-%s' % (recipename, recipever)
2311        checkvars['SRC_URI'] = url
2312        self._test_recipe_contents(newrecipefile, checkvars, [])
2313        # Try again - change just name this time
2314        result = runCmd('devtool reset -n %s' % newrecipename)
2315        add_recipe()
2316        newrecipefile = os.path.join(self.workspacedir, 'recipes', newrecipename, '%s_%s.bb' % (newrecipename, recipever))
2317        result = runCmd('devtool rename %s %s' % (recipename, newrecipename))
2318        self.assertExists(newrecipefile, 'Recipe file not renamed')
2319        self.assertNotExists(os.path.join(self.workspacedir, 'recipes', recipename), 'Old recipe directory still exists')
2320        self.assertExists(os.path.join(self.workspacedir, 'sources', newrecipename), 'Source directory not renamed')
2321        checkvars = {}
2322        checkvars['S'] = '${WORKDIR}/%s-${PV}' % recipename
2323        checkvars['SRC_URI'] = url.replace(recipever, '${PV}')
2324        self._test_recipe_contents(newrecipefile, checkvars, [])
2325        # Try again - change just version this time
2326        result = runCmd('devtool reset -n %s' % newrecipename)
2327        add_recipe()
2328        newrecipefile = os.path.join(self.workspacedir, 'recipes', recipename, '%s_%s.bb' % (recipename, newrecipever))
2329        result = runCmd('devtool rename %s -V %s' % (recipename, newrecipever))
2330        self.assertExists(newrecipefile, 'Recipe file not renamed')
2331        self.assertExists(os.path.join(self.workspacedir, 'sources', recipename), 'Source directory no longer exists')
2332        checkvars = {}
2333        checkvars['S'] = '${WORKDIR}/${BPN}-%s' % recipever
2334        checkvars['SRC_URI'] = url
2335        self._test_recipe_contents(newrecipefile, checkvars, [])
2336
2337    def test_devtool_virtual_kernel_modify(self):
2338        """
2339        Summary:        The purpose of this test case is to verify that
2340                        devtool modify works correctly when building
2341                        the kernel.
2342        Dependencies:   NA
2343        Steps:          1. Build kernel with bitbake.
2344                        2. Save the config file generated.
2345                        3. Clean the environment.
2346                        4. Use `devtool modify virtual/kernel` to validate following:
2347                           4.1 The source is checked out correctly.
2348                           4.2 The resulting configuration is the same as
2349                               what was get on step 2.
2350                           4.3 The Kernel can be build correctly.
2351                           4.4 Changes made on the source are reflected on the
2352                               subsequent builds.
2353                           4.5 Changes on the configuration are reflected on the
2354                               subsequent builds
2355         Expected:       devtool modify is able to checkout the source of the kernel
2356                         and modification to the source and configurations are reflected
2357                         when building the kernel.
2358        """
2359        kernel_provider = self.td['PREFERRED_PROVIDER_virtual/kernel']
2360
2361        # Clean up the environment
2362        bitbake('%s -c clean' % kernel_provider)
2363        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
2364        tempdir_cfg = tempfile.mkdtemp(prefix='config_qa')
2365        self.track_for_cleanup(tempdir)
2366        self.track_for_cleanup(tempdir_cfg)
2367        self.track_for_cleanup(self.workspacedir)
2368        self.add_command_to_tearDown('bitbake -c clean %s' % kernel_provider)
2369        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
2370        #Step 1
2371        #Here is just generated the config file instead of all the kernel to optimize the
2372        #time of executing this test case.
2373        bitbake('%s -c configure' % kernel_provider)
2374        bbconfig = os.path.join(get_bb_var('B', kernel_provider),'.config')
2375        #Step 2
2376        runCmd('cp %s %s' % (bbconfig, tempdir_cfg))
2377        self.assertExists(os.path.join(tempdir_cfg, '.config'), 'Could not copy .config file from kernel')
2378
2379        tmpconfig = os.path.join(tempdir_cfg, '.config')
2380        #Step 3
2381        bitbake('%s -c clean' % kernel_provider)
2382        #Step 4.1
2383        runCmd('devtool modify virtual/kernel -x %s' % tempdir)
2384        self.assertExists(os.path.join(tempdir, 'Makefile'), 'Extracted source could not be found')
2385        #Step 4.2
2386        configfile = os.path.join(tempdir,'.config')
2387        runCmd('diff %s %s' % (tmpconfig, configfile))
2388
2389        #Step 4.3
2390        #NOTE: virtual/kernel is mapped to kernel_provider
2391        runCmd('devtool build %s' % kernel_provider)
2392        kernelfile = os.path.join(get_bb_var('KBUILD_OUTPUT', kernel_provider), 'vmlinux')
2393        self.assertExists(kernelfile, 'Kernel was not build correctly')
2394
2395        #Modify the kernel source
2396        modfile = os.path.join(tempdir, 'init/version.c')
2397        # Moved to uts.h in 6.1 onwards
2398        modfile2 = os.path.join(tempdir, 'include/linux/uts.h')
2399        runCmd("sed -i 's/Linux/LiNuX/g' %s %s" % (modfile, modfile2))
2400
2401        #Modify the configuration
2402        codeconfigfile = os.path.join(tempdir, '.config.new')
2403        modconfopt = "CONFIG_SG_POOL=n"
2404        runCmd("sed -i 's/CONFIG_SG_POOL=y/%s/' %s" % (modconfopt, codeconfigfile))
2405
2406        #Build again kernel with devtool
2407        runCmd('devtool build %s' % kernel_provider)
2408
2409        #Step 4.4
2410        runCmd("grep '%s' %s" % ('LiNuX', kernelfile))
2411
2412        #Step 4.5
2413        runCmd("grep %s %s" % (modconfopt, codeconfigfile))
2414
2415
2416class DevtoolIdeSdkTests(DevtoolBase):
2417    def _write_bb_config(self, recipe_names):
2418        """Helper to write the bitbake local.conf file"""
2419        conf_lines = [
2420            'IMAGE_CLASSES += "image-combined-dbg"',
2421            'IMAGE_GEN_DEBUGFS = "1"',
2422            'IMAGE_INSTALL:append = " gdbserver %s"' % ' '.join(
2423                [r + '-ptest' for r in recipe_names])
2424        ]
2425        self.write_config("\n".join(conf_lines))
2426
2427    def _check_workspace(self):
2428        """Check if a workspace directory is available and setup the cleanup"""
2429        self.assertTrue(not os.path.exists(self.workspacedir),
2430                        'This test cannot be run with a workspace directory under the build directory')
2431        self.track_for_cleanup(self.workspacedir)
2432        self.add_command_to_tearDown('bitbake-layers remove-layer */workspace')
2433
2434    def _workspace_scripts_dir(self, recipe_name):
2435        return os.path.realpath(os.path.join(self.builddir, 'workspace', 'ide-sdk', recipe_name, 'scripts'))
2436
2437    def _sources_scripts_dir(self, src_dir):
2438        return os.path.realpath(os.path.join(src_dir, 'oe-scripts'))
2439
2440    def _workspace_gdbinit_dir(self, recipe_name):
2441        return os.path.realpath(os.path.join(self.builddir, 'workspace', 'ide-sdk', recipe_name, 'scripts', 'gdbinit'))
2442
2443    def _sources_gdbinit_dir(self, src_dir):
2444        return os.path.realpath(os.path.join(src_dir, 'oe-gdbinit'))
2445
2446    def _devtool_ide_sdk_recipe(self, recipe_name, build_file, testimage):
2447        """Setup a recipe for working with devtool ide-sdk
2448
2449        Basically devtool modify -x followed by some tests
2450        """
2451        tempdir = tempfile.mkdtemp(prefix='devtoolqa')
2452        self.track_for_cleanup(tempdir)
2453        self.add_command_to_tearDown('bitbake -c clean %s' % recipe_name)
2454
2455        result = runCmd('devtool modify %s -x %s' % (recipe_name, tempdir))
2456        self.assertExists(os.path.join(tempdir, build_file),
2457                          'Extracted source could not be found')
2458        self.assertExists(os.path.join(self.workspacedir, 'conf',
2459                                       'layer.conf'), 'Workspace directory not created')
2460        matches = glob.glob(os.path.join(self.workspacedir,
2461                            'appends', recipe_name + '.bbappend'))
2462        self.assertTrue(matches, 'bbappend not created %s' % result.output)
2463
2464        # Test devtool status
2465        result = runCmd('devtool status')
2466        self.assertIn(recipe_name, result.output)
2467        self.assertIn(tempdir, result.output)
2468        self._check_src_repo(tempdir)
2469
2470        # Usually devtool ide-sdk would initiate the build of the SDK.
2471        # But there is a circular dependency with starting Qemu and passing the IP of runqemu to devtool ide-sdk.
2472        if testimage:
2473            bitbake("%s qemu-native qemu-helper-native" % testimage)
2474            deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE')
2475            self.add_command_to_tearDown('bitbake -c clean %s' % testimage)
2476            self.add_command_to_tearDown(
2477                'rm -f %s/%s*' % (deploy_dir_image, testimage))
2478
2479        return tempdir
2480
2481    def _get_recipe_ids(self, recipe_name):
2482        """IDs needed to write recipe specific config entries into IDE config files"""
2483        package_arch = get_bb_var('PACKAGE_ARCH', recipe_name)
2484        recipe_id = recipe_name + "-" + package_arch
2485        recipe_id_pretty = recipe_name + ": " + package_arch
2486        return (recipe_id, recipe_id_pretty)
2487
2488    def _verify_install_script_code(self, tempdir, recipe_name):
2489        """Verify the scripts referred by the tasks.json file are fine.
2490
2491        This function does not depend on Qemu. Therefore it verifies the scripts
2492        exists and the delete step works as expected. But it does not try to
2493        deploy to Qemu.
2494        """
2495        recipe_id, recipe_id_pretty = self._get_recipe_ids(recipe_name)
2496        with open(os.path.join(tempdir, '.vscode', 'tasks.json')) as tasks_j:
2497            tasks_d = json.load(tasks_j)
2498        tasks = tasks_d["tasks"]
2499        task_install = next(
2500            (task for task in tasks if task["label"] == "install && deploy-target %s" % recipe_id_pretty), None)
2501        self.assertIsNot(task_install, None)
2502        # execute only the bb_run_do_install script since the deploy would require e.g. Qemu running.
2503        i_and_d_script = "install_and_deploy_" + recipe_id
2504        i_and_d_script_path = os.path.join(
2505            self._workspace_scripts_dir(recipe_name), i_and_d_script)
2506        self.assertExists(i_and_d_script_path)
2507        del_script = "delete_package_dirs_" + recipe_id
2508        del_script_path = os.path.join(
2509            self._workspace_scripts_dir(recipe_name), del_script)
2510        self.assertExists(del_script_path)
2511        runCmd(del_script_path, cwd=tempdir)
2512
2513    def _devtool_ide_sdk_qemu(self, tempdir, qemu, recipe_name, example_exe):
2514        """Verify deployment and execution in Qemu system work for one recipe.
2515
2516        This function checks the entire SDK workflow: changing the code, recompiling
2517        it and deploying it back to Qemu, and checking that the changes have been
2518        incorporated into the provided binaries. It also runs the tests of the recipe.
2519        """
2520        recipe_id, _ = self._get_recipe_ids(recipe_name)
2521        i_and_d_script = "install_and_deploy_" + recipe_id
2522        install_deploy_cmd = os.path.join(
2523            self._workspace_scripts_dir(recipe_name), i_and_d_script)
2524        self.assertExists(install_deploy_cmd,
2525                          '%s script not found' % install_deploy_cmd)
2526        runCmd(install_deploy_cmd)
2527
2528        MAGIC_STRING_ORIG = "Magic: 123456789"
2529        MAGIC_STRING_NEW = "Magic: 987654321"
2530        ptest_cmd = "ptest-runner " + recipe_name
2531
2532        # validate that SSH is working
2533        status, _ = qemu.run("uname")
2534        self.assertEqual(
2535            status, 0, msg="Failed to connect to the SSH server on Qemu")
2536
2537        # Verify the unmodified example prints the magic string
2538        status, output = qemu.run(example_exe)
2539        self.assertEqual(status, 0, msg="%s failed: %s" %
2540                         (example_exe, output))
2541        self.assertIn(MAGIC_STRING_ORIG, output)
2542
2543        # Verify the unmodified ptests work
2544        status, output = qemu.run(ptest_cmd)
2545        self.assertEqual(status, 0, msg="%s failed: %s" % (ptest_cmd, output))
2546        self.assertIn("PASS: cpp-example-lib", output)
2547
2548        # Verify remote debugging works
2549        self._gdb_cross_debugging(
2550            qemu, recipe_name, example_exe, MAGIC_STRING_ORIG)
2551
2552        # Replace the Magic String in the code, compile and deploy to Qemu
2553        cpp_example_lib_hpp = os.path.join(tempdir, 'cpp-example-lib.hpp')
2554        with open(cpp_example_lib_hpp, 'r') as file:
2555            cpp_code = file.read()
2556            cpp_code = cpp_code.replace(MAGIC_STRING_ORIG, MAGIC_STRING_NEW)
2557        with open(cpp_example_lib_hpp, 'w') as file:
2558            file.write(cpp_code)
2559        runCmd(install_deploy_cmd, cwd=tempdir)
2560
2561        # Verify the modified example prints the modified magic string
2562        status, output = qemu.run(example_exe)
2563        self.assertEqual(status, 0, msg="%s failed: %s" %
2564                         (example_exe, output))
2565        self.assertNotIn(MAGIC_STRING_ORIG, output)
2566        self.assertIn(MAGIC_STRING_NEW, output)
2567
2568        # Verify the modified example ptests work
2569        status, output = qemu.run(ptest_cmd)
2570        self.assertEqual(status, 0, msg="%s failed: %s" % (ptest_cmd, output))
2571        self.assertIn("PASS: cpp-example-lib", output)
2572
2573        # Verify remote debugging works wit the modified magic string
2574        self._gdb_cross_debugging(
2575            qemu, recipe_name, example_exe, MAGIC_STRING_NEW)
2576
2577    def _gdb_cross(self):
2578        """Verify gdb-cross is provided by devtool ide-sdk"""
2579        target_arch = self.td["TARGET_ARCH"]
2580        target_sys = self.td["TARGET_SYS"]
2581        gdb_recipe = "gdb-cross-" + target_arch
2582        gdb_binary = target_sys + "-gdb"
2583
2584        native_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", gdb_recipe)
2585        r = runCmd("%s --version" % gdb_binary,
2586                   native_sysroot=native_sysroot, target_sys=target_sys)
2587        self.assertEqual(r.status, 0)
2588        self.assertIn("GNU gdb", r.output)
2589
2590    def _gdb_cross_debugging(self, qemu, recipe_name, example_exe, magic_string):
2591        """Verify gdb-cross is working
2592
2593        Test remote debugging:
2594        break main
2595        run
2596        continue
2597        break CppExample::print_json()
2598        continue
2599        print CppExample::test_string.compare("cpp-example-lib Magic: 123456789")
2600        $1 = 0
2601        print CppExample::test_string.compare("cpp-example-lib Magic: 123456789aaa")
2602        $2 = -3
2603        list cpp-example-lib.hpp:13,13
2604        13	    inline static const std::string test_string = "cpp-example-lib Magic: 123456789";
2605        continue
2606        """
2607        sshargs = '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
2608        gdbserver_script = os.path.join(self._workspace_scripts_dir(
2609            recipe_name), 'gdbserver_1234_usr-bin-' + example_exe + '_m')
2610        gdb_script = os.path.join(self._workspace_scripts_dir(
2611            recipe_name), 'gdb_1234_usr-bin-' + example_exe)
2612
2613        # Start a gdbserver
2614        r = runCmd(gdbserver_script)
2615        self.assertEqual(r.status, 0)
2616
2617        # Check there is a gdbserver running
2618        r = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, 'ps'))
2619        self.assertEqual(r.status, 0)
2620        self.assertIn("gdbserver ", r.output)
2621
2622        # Check the pid file is correct
2623        test_cmd = "cat /proc/$(cat /tmp/gdbserver_1234_usr-bin-" + \
2624            example_exe + "/pid)/cmdline"
2625        r = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, test_cmd))
2626        self.assertEqual(r.status, 0)
2627        self.assertIn("gdbserver", r.output)
2628
2629        # Test remote debugging works
2630        gdb_batch_cmd = " --batch -ex 'break main' -ex 'run'"
2631        gdb_batch_cmd += " -ex 'break CppExample::print_json()' -ex 'continue'"
2632        gdb_batch_cmd += " -ex 'print CppExample::test_string.compare(\"cpp-example-lib %s\")'" % magic_string
2633        gdb_batch_cmd += " -ex 'print CppExample::test_string.compare(\"cpp-example-lib %saaa\")'" % magic_string
2634        gdb_batch_cmd += " -ex 'list cpp-example-lib.hpp:13,13'"
2635        gdb_batch_cmd += " -ex 'continue'"
2636        r = runCmd(gdb_script + gdb_batch_cmd)
2637        self.logger.debug("%s %s returned: %s", gdb_script,
2638                          gdb_batch_cmd, r.output)
2639        self.assertEqual(r.status, 0)
2640        self.assertIn("Breakpoint 1, main", r.output)
2641        self.assertIn("$1 = 0", r.output)  # test.string.compare equal
2642        self.assertIn("$2 = -3", r.output)  # test.string.compare longer
2643        self.assertIn(
2644            'inline static const std::string test_string = "cpp-example-lib %s";' % magic_string, r.output)
2645        self.assertIn("exited normally", r.output)
2646
2647        # Stop the gdbserver
2648        r = runCmd(gdbserver_script + ' stop')
2649        self.assertEqual(r.status, 0)
2650
2651        # Check there is no gdbserver running
2652        r = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, 'ps'))
2653        self.assertEqual(r.status, 0)
2654        self.assertNotIn("gdbserver ", r.output)
2655
2656    def _verify_cmake_preset(self, tempdir):
2657        """Verify the generated cmake preset works as expected
2658
2659        Check if compiling works
2660        Check if unit tests can be executed in qemu (not qemu-system)
2661        """
2662        with open(os.path.join(tempdir, 'CMakeUserPresets.json')) as cmake_preset_j:
2663            cmake_preset_d = json.load(cmake_preset_j)
2664        config_presets = cmake_preset_d["configurePresets"]
2665        self.assertEqual(len(config_presets), 1)
2666        cmake_exe = config_presets[0]["cmakeExecutable"]
2667        preset_name = config_presets[0]["name"]
2668
2669        # Verify the wrapper for cmake native is available
2670        self.assertExists(cmake_exe)
2671
2672        # Verify the cmake preset generated by devtool ide-sdk is available
2673        result = runCmd('%s --list-presets' % cmake_exe, cwd=tempdir)
2674        self.assertIn(preset_name, result.output)
2675
2676        # Verify cmake re-uses the o files compiled by bitbake
2677        result = runCmd('%s --build --preset %s' %
2678                        (cmake_exe, preset_name), cwd=tempdir)
2679        self.assertIn("ninja: no work to do.", result.output)
2680
2681        # Verify the unit tests work (in Qemu user mode)
2682        result = runCmd('%s --build --preset %s --target test' %
2683                        (cmake_exe, preset_name), cwd=tempdir)
2684        self.assertIn("100% tests passed", result.output)
2685
2686        # Verify re-building and testing works again
2687        result = runCmd('%s --build --preset %s --target clean' %
2688                        (cmake_exe, preset_name), cwd=tempdir)
2689        self.assertIn("Cleaning", result.output)
2690        result = runCmd('%s --build --preset %s' %
2691                        (cmake_exe, preset_name), cwd=tempdir)
2692        self.assertIn("Building", result.output)
2693        self.assertIn("Linking", result.output)
2694        result = runCmd('%s --build --preset %s --target test' %
2695                        (cmake_exe, preset_name), cwd=tempdir)
2696        self.assertIn("Running tests...", result.output)
2697        self.assertIn("100% tests passed", result.output)
2698
2699    @OETestTag("runqemu")
2700    def test_devtool_ide_sdk_none_qemu(self):
2701        """Start qemu-system and run tests for multiple recipes. ide=none is used."""
2702        recipe_names = ["cmake-example", "meson-example"]
2703        testimage = "oe-selftest-image"
2704
2705        self._check_workspace()
2706        self._write_bb_config(recipe_names)
2707        self._check_runqemu_prerequisites()
2708
2709        # Verify deployment to Qemu (system mode) works
2710        bitbake(testimage)
2711        with runqemu(testimage, runqemuparams="nographic") as qemu:
2712            # cmake-example recipe
2713            recipe_name = "cmake-example"
2714            example_exe = "cmake-example"
2715            build_file = "CMakeLists.txt"
2716            tempdir = self._devtool_ide_sdk_recipe(
2717                recipe_name, build_file, testimage)
2718            bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@%s -c --ide=none' % (
2719                recipe_name, testimage, qemu.ip)
2720            runCmd(bitbake_sdk_cmd)
2721            self._gdb_cross()
2722            self._verify_cmake_preset(tempdir)
2723            self._devtool_ide_sdk_qemu(tempdir, qemu, recipe_name, example_exe)
2724            # Verify the oe-scripts sym-link is valid
2725            self.assertEqual(self._workspace_scripts_dir(
2726                recipe_name), self._sources_scripts_dir(tempdir))
2727
2728            # meson-example recipe
2729            recipe_name = "meson-example"
2730            example_exe = "mesonex"
2731            build_file = "meson.build"
2732            tempdir = self._devtool_ide_sdk_recipe(
2733                recipe_name, build_file, testimage)
2734            bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@%s -c --ide=none' % (
2735                recipe_name, testimage, qemu.ip)
2736            runCmd(bitbake_sdk_cmd)
2737            self._gdb_cross()
2738            self._devtool_ide_sdk_qemu(tempdir, qemu, recipe_name, example_exe)
2739            # Verify the oe-scripts sym-link is valid
2740            self.assertEqual(self._workspace_scripts_dir(
2741                recipe_name), self._sources_scripts_dir(tempdir))
2742
2743    def test_devtool_ide_sdk_code_cmake(self):
2744        """Verify a cmake recipe works with ide=code mode"""
2745        recipe_name = "cmake-example"
2746        build_file = "CMakeLists.txt"
2747        testimage = "oe-selftest-image"
2748
2749        self._check_workspace()
2750        self._write_bb_config([recipe_name])
2751        tempdir = self._devtool_ide_sdk_recipe(
2752            recipe_name, build_file, testimage)
2753        bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@192.168.17.17 -c --ide=code' % (
2754            recipe_name, testimage)
2755        runCmd(bitbake_sdk_cmd)
2756        self._verify_cmake_preset(tempdir)
2757        self._verify_install_script_code(tempdir,  recipe_name)
2758        self._gdb_cross()
2759
2760    def test_devtool_ide_sdk_code_meson(self):
2761        """Verify a meson recipe works with ide=code mode"""
2762        recipe_name = "meson-example"
2763        build_file = "meson.build"
2764        testimage = "oe-selftest-image"
2765
2766        self._check_workspace()
2767        self._write_bb_config([recipe_name])
2768        tempdir = self._devtool_ide_sdk_recipe(
2769            recipe_name, build_file, testimage)
2770        bitbake_sdk_cmd = 'devtool ide-sdk %s %s -t root@192.168.17.17 -c --ide=code' % (
2771            recipe_name, testimage)
2772        runCmd(bitbake_sdk_cmd)
2773
2774        with open(os.path.join(tempdir, '.vscode', 'settings.json')) as settings_j:
2775            settings_d = json.load(settings_j)
2776        meson_exe = settings_d["mesonbuild.mesonPath"]
2777        meson_build_folder = settings_d["mesonbuild.buildFolder"]
2778
2779        # Verify the wrapper for meson native is available
2780        self.assertExists(meson_exe)
2781
2782        # Verify meson re-uses the o files compiled by bitbake
2783        result = runCmd('%s compile -C  %s' %
2784                        (meson_exe, meson_build_folder), cwd=tempdir)
2785        self.assertIn("ninja: no work to do.", result.output)
2786
2787        # Verify the unit tests work (in Qemu)
2788        runCmd('%s test -C %s' % (meson_exe, meson_build_folder), cwd=tempdir)
2789
2790        # Verify re-building and testing works again
2791        result = runCmd('%s compile -C  %s --clean' %
2792                        (meson_exe, meson_build_folder), cwd=tempdir)
2793        self.assertIn("Cleaning...", result.output)
2794        result = runCmd('%s compile -C  %s' %
2795                        (meson_exe, meson_build_folder), cwd=tempdir)
2796        self.assertIn("Linking target", result.output)
2797        runCmd('%s test -C %s' % (meson_exe, meson_build_folder), cwd=tempdir)
2798
2799        self._verify_install_script_code(tempdir,  recipe_name)
2800        self._gdb_cross()
2801
2802    def test_devtool_ide_sdk_shared_sysroots(self):
2803        """Verify the shared sysroot SDK"""
2804
2805        # Handle the workspace (which is not needed by this test case)
2806        self._check_workspace()
2807
2808        result_init = runCmd(
2809            'devtool ide-sdk -m shared oe-selftest-image cmake-example meson-example --ide=code')
2810        bb_vars = get_bb_vars(
2811            ['REAL_MULTIMACH_TARGET_SYS', 'DEPLOY_DIR_IMAGE', 'COREBASE'], "meta-ide-support")
2812        environment_script = 'environment-setup-%s' % bb_vars['REAL_MULTIMACH_TARGET_SYS']
2813        deploydir = bb_vars['DEPLOY_DIR_IMAGE']
2814        environment_script_path = os.path.join(deploydir, environment_script)
2815        cpp_example_src = os.path.join(
2816            bb_vars['COREBASE'], 'meta-selftest', 'recipes-test', 'cpp', 'files')
2817
2818        # Verify the cross environment script is available
2819        self.assertExists(environment_script_path)
2820
2821        def runCmdEnv(cmd, cwd):
2822            cmd = '/bin/sh -c ". %s > /dev/null && %s"' % (
2823                environment_script_path, cmd)
2824            return runCmd(cmd, cwd)
2825
2826        # Verify building the C++ example works with CMake
2827        tempdir_cmake = tempfile.mkdtemp(prefix='devtoolqa')
2828        self.track_for_cleanup(tempdir_cmake)
2829
2830        result_cmake = runCmdEnv("which cmake", cwd=tempdir_cmake)
2831        cmake_native = os.path.normpath(result_cmake.output.strip())
2832        self.assertExists(cmake_native)
2833
2834        runCmdEnv('cmake %s' % cpp_example_src, cwd=tempdir_cmake)
2835        runCmdEnv('cmake --build %s' % tempdir_cmake, cwd=tempdir_cmake)
2836
2837        # Verify the printed note really referres to a cmake executable
2838        cmake_native_code = ""
2839        for line in result_init.output.splitlines():
2840            m = re.search(r'"cmake.cmakePath": "(.*)"', line)
2841            if m:
2842                cmake_native_code = m.group(1)
2843                break
2844        self.assertExists(cmake_native_code)
2845        self.assertEqual(cmake_native, cmake_native_code)
2846
2847        # Verify building the C++ example works with Meson
2848        tempdir_meson = tempfile.mkdtemp(prefix='devtoolqa')
2849        self.track_for_cleanup(tempdir_meson)
2850
2851        result_cmake = runCmdEnv("which meson", cwd=tempdir_meson)
2852        meson_native = os.path.normpath(result_cmake.output.strip())
2853        self.assertExists(meson_native)
2854
2855        runCmdEnv('meson setup %s' % tempdir_meson, cwd=cpp_example_src)
2856        runCmdEnv('meson compile', cwd=tempdir_meson)
2857
2858    def test_devtool_ide_sdk_plugins(self):
2859        """Test that devtool ide-sdk can use plugins from other layers."""
2860
2861        # We need a workspace layer and a modified recipe (but no image)
2862        modified_recipe_name = "meson-example"
2863        modified_build_file = "meson.build"
2864        testimage = "oe-selftest-image"
2865        shared_recipe_name = "cmake-example"
2866
2867        self._check_workspace()
2868        self._write_bb_config([modified_recipe_name])
2869        tempdir = self._devtool_ide_sdk_recipe(
2870            modified_recipe_name, modified_build_file, None)
2871
2872        IDE_RE = re.compile(r'.*--ide \{(.*)\}.*')
2873
2874        def get_ides_from_help(help_str):
2875            m = IDE_RE.search(help_str)
2876            return m.group(1).split(',')
2877
2878        # verify the default plugins are available but the foo plugin is not
2879        result = runCmd('devtool ide-sdk -h')
2880        found_ides = get_ides_from_help(result.output)
2881        self.assertIn('code', found_ides)
2882        self.assertIn('none', found_ides)
2883        self.assertNotIn('foo', found_ides)
2884
2885        shared_config_file = os.path.join(tempdir, 'shared-config.txt')
2886        shared_config_str = 'Dummy shared IDE config'
2887        modified_config_file = os.path.join(tempdir, 'modified-config.txt')
2888        modified_config_str = 'Dummy modified IDE config'
2889
2890        # Generate a foo plugin in the workspace layer
2891        plugin_dir = os.path.join(
2892            self.workspacedir, 'lib', 'devtool', 'ide_plugins')
2893        os.makedirs(plugin_dir)
2894        plugin_code = 'from devtool.ide_plugins import IdeBase\n\n'
2895        plugin_code += 'class IdeFoo(IdeBase):\n'
2896        plugin_code += '    def setup_shared_sysroots(self, shared_env):\n'
2897        plugin_code += '        with open("%s", "w") as config_file:\n' % shared_config_file
2898        plugin_code += '            config_file.write("%s")\n\n' % shared_config_str
2899        plugin_code += '    def setup_modified_recipe(self, args, image_recipe, modified_recipe):\n'
2900        plugin_code += '        with open("%s", "w") as config_file:\n' % modified_config_file
2901        plugin_code += '            config_file.write("%s")\n\n' % modified_config_str
2902        plugin_code += 'def register_ide_plugin(ide_plugins):\n'
2903        plugin_code += '    ide_plugins["foo"] = IdeFoo\n'
2904
2905        plugin_py = os.path.join(plugin_dir, 'ide_foo.py')
2906        with open(plugin_py, 'w') as plugin_file:
2907            plugin_file.write(plugin_code)
2908
2909        # Verify the foo plugin is available as well
2910        result = runCmd('devtool ide-sdk -h')
2911        found_ides = get_ides_from_help(result.output)
2912        self.assertIn('code', found_ides)
2913        self.assertIn('none', found_ides)
2914        self.assertIn('foo', found_ides)
2915
2916        # Verify the foo plugin generates a shared config
2917        result = runCmd(
2918            'devtool ide-sdk -m shared --skip-bitbake --ide foo %s' % shared_recipe_name)
2919        with open(shared_config_file) as shared_config:
2920            shared_config_new = shared_config.read()
2921        self.assertEqual(shared_config_str, shared_config_new)
2922
2923        # Verify the foo plugin generates a modified config
2924        result = runCmd('devtool ide-sdk --skip-bitbake --ide foo %s %s' %
2925                        (modified_recipe_name, testimage))
2926        with open(modified_config_file) as modified_config:
2927            modified_config_new = modified_config.read()
2928        self.assertEqual(modified_config_str, modified_config_new)
2929