1#
2# Copyright BitBake Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import layerindexlib
8
9import argparse
10import logging
11import os
12import subprocess
13
14from bblayers.action import ActionPlugin
15
16logger = logging.getLogger('bitbake-layers')
17
18
19def plugin_init(plugins):
20    return LayerIndexPlugin()
21
22
23class LayerIndexPlugin(ActionPlugin):
24    """Subcommands for interacting with the layer index.
25
26    This class inherits ActionPlugin to get do_add_layer.
27    """
28
29    def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer, branch, shallow=False):
30        layername = self.get_layer_name(url)
31        if os.path.splitext(layername)[1] == '.git':
32            layername = os.path.splitext(layername)[0]
33        repodir = os.path.join(fetchdir, layername)
34        layerdir = os.path.join(repodir, subdir)
35        if not os.path.exists(repodir):
36            if fetch_layer:
37                cmd = ['git', 'clone']
38                if shallow:
39                    cmd.extend(['--depth', '1'])
40                if branch:
41                    cmd.extend(['-b' , branch])
42                cmd.extend([url, repodir])
43                result = subprocess.call(cmd)
44                if result:
45                    logger.error("Failed to download %s (%s)" % (url, branch))
46                    return None, None, None
47                else:
48                    return subdir, layername, layerdir
49            else:
50                logger.plain("Repository %s needs to be fetched" % url)
51                return subdir, layername, layerdir
52        elif os.path.exists(repodir) and branch:
53            """
54            If the repo is already cloned, ensure it is on the correct branch,
55            switching branches if necessary and possible.
56            """
57            base_cmd = ['git', '--git-dir=%s/.git' % repodir, '--work-tree=%s' % repodir]
58            cmd = base_cmd + ['branch']
59            completed_proc = subprocess.run(cmd, text=True, capture_output=True)
60            if completed_proc.returncode:
61                logger.error("Unable to validate repo %s (%s)" % (repodir, stderr))
62                return None, None, None
63            else:
64                if branch != completed_proc.stdout[2:-1]:
65                    cmd = base_cmd + ['status', '--short']
66                    completed_proc = subprocess.run(cmd, text=True, capture_output=True)
67                    if completed_proc.stdout.count('\n') != 0:
68                        logger.warning("There are uncommitted changes in repo %s" % repodir)
69                    cmd = base_cmd + ['checkout', branch]
70                    completed_proc = subprocess.run(cmd, text=True, capture_output=True)
71                    if completed_proc.returncode:
72                        # Could be due to original shallow clone on a different branch for example
73                        logger.error("Unable to automatically switch %s to desired branch '%s' (%s)"
74                                     % (repodir, branch, completed_proc.stderr))
75                        return None, None, None
76            return subdir, layername, layerdir
77        elif os.path.exists(layerdir):
78            return subdir, layername, layerdir
79        else:
80            logger.error("%s is not in %s" % (url, subdir))
81        return None, None, None
82
83    def do_layerindex_fetch(self, args):
84        """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
85"""
86
87        def _construct_url(baseurls, branches):
88            urls = []
89            for baseurl in baseurls:
90                if baseurl[-1] != '/':
91                    baseurl += '/'
92
93                if not baseurl.startswith('cooker'):
94                    baseurl += "api/"
95
96                if branches:
97                    baseurl += ";branch=%s" % ','.join(branches)
98
99                urls.append(baseurl)
100
101            return urls
102
103
104        # Set the default...
105        if args.branch:
106            branches = [args.branch]
107        else:
108            branches = (self.tinfoil.config_data.getVar('LAYERSERIES_CORENAMES') or 'master').split()
109        logger.debug('Trying branches: %s' % branches)
110
111        ignore_layers = []
112        if args.ignore:
113            ignore_layers.extend(args.ignore.split(','))
114
115        # Load the cooker DB
116        cookerIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
117        cookerIndex.load_layerindex('cooker://', load='layerDependencies')
118
119        # Fast path, check if we already have what has been requested!
120        (dependencies, invalidnames) = cookerIndex.find_dependencies(names=args.layername, ignores=ignore_layers)
121        if not args.show_only and not invalidnames:
122            logger.plain("You already have the requested layer(s): %s" % args.layername)
123            return 0
124
125        # The information to show is already in the cookerIndex
126        if invalidnames:
127            # General URL to use to access the layer index
128            # While there is ONE right now, we're expect users could enter several
129            apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL').split()
130            if not apiurl:
131                logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
132                return 1
133
134            remoteIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
135
136            for remoteurl in _construct_url(apiurl, branches):
137                logger.plain("Loading %s..." % remoteurl)
138                remoteIndex.load_layerindex(remoteurl)
139
140            if remoteIndex.is_empty():
141                logger.error("Remote layer index %s is empty for branches %s" % (apiurl, branches))
142                return 1
143
144            lIndex = cookerIndex + remoteIndex
145
146            (dependencies, invalidnames) = lIndex.find_dependencies(names=args.layername, ignores=ignore_layers)
147
148            if invalidnames:
149                for invaluename in invalidnames:
150                    logger.error('Layer "%s" not found in layer index' % invaluename)
151                return 1
152
153        logger.plain("%s  %s  %s" % ("Layer".ljust(49), "Git repository (branch)".ljust(54), "Subdirectory"))
154        logger.plain('=' * 125)
155
156        for deplayerbranch in dependencies:
157            layerBranch = dependencies[deplayerbranch][0]
158
159            # TODO: Determine display behavior
160            # This is the local content, uncomment to hide local
161            # layers from the display.
162            #if layerBranch.index.config['TYPE'] == 'cooker':
163            #    continue
164
165            layerDeps = dependencies[deplayerbranch][1:]
166
167            requiredby = []
168            recommendedby = []
169            for dep in layerDeps:
170                if dep.required:
171                    requiredby.append(dep.layer.name)
172                else:
173                    recommendedby.append(dep.layer.name)
174
175            logger.plain('%s %s %s' % (("%s:%s:%s" %
176                                  (layerBranch.index.config['DESCRIPTION'],
177                                  layerBranch.branch.name,
178                                  layerBranch.layer.name)).ljust(50),
179                                  ("%s (%s)" % (layerBranch.layer.vcs_url,
180                                  layerBranch.actual_branch)).ljust(55),
181                                  layerBranch.vcs_subdir
182                                               ))
183            if requiredby:
184                logger.plain('  required by: %s' % ' '.join(requiredby))
185            if recommendedby:
186                logger.plain('  recommended by: %s' % ' '.join(recommendedby))
187
188        if dependencies:
189            if args.fetchdir:
190                fetchdir = args.fetchdir
191            else:
192                fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
193                if not fetchdir:
194                    logger.error("Cannot get BBLAYERS_FETCH_DIR")
195                    return 1
196
197            if not os.path.exists(fetchdir):
198                os.makedirs(fetchdir)
199
200            addlayers = []
201
202            for deplayerbranch in dependencies:
203                layerBranch = dependencies[deplayerbranch][0]
204
205                if layerBranch.index.config['TYPE'] == 'cooker':
206                    # Anything loaded via cooker is already local, skip it
207                    continue
208
209                subdir, name, layerdir = self.get_fetch_layer(fetchdir,
210                                                      layerBranch.layer.vcs_url,
211                                                      layerBranch.vcs_subdir,
212                                                      not args.show_only,
213                                                      layerBranch.actual_branch,
214                                                      args.shallow)
215                if not name:
216                    # Error already shown
217                    return 1
218                addlayers.append((subdir, name, layerdir))
219        if not args.show_only:
220            localargs = argparse.Namespace()
221            localargs.layerdir = []
222            localargs.force = args.force
223            for subdir, name, layerdir in addlayers:
224                if os.path.exists(layerdir):
225                    if subdir:
226                        logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (subdir, layerdir))
227                    else:
228                        logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (name, layerdir))
229                    localargs.layerdir.append(layerdir)
230                else:
231                    break
232
233            if localargs.layerdir:
234                self.do_add_layer(localargs)
235
236    def do_layerindex_show_depends(self, args):
237        """Find layer dependencies from layer index.
238"""
239        args.show_only = True
240        args.ignore = []
241        args.fetchdir = ""
242        args.shallow = True
243        self.do_layerindex_fetch(args)
244
245    def register_commands(self, sp):
246        parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch, parserecipes=False)
247        parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true')
248        parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch')
249        parser_layerindex_fetch.add_argument('-s', '--shallow', help='do only shallow clones (--depth=1)', action='store_true')
250        parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
251        parser_layerindex_fetch.add_argument('-f', '--fetchdir', help='directory to fetch the layer(s) into (will be created if it does not exist)')
252        parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
253
254        parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends, parserecipes=False)
255        parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch')
256        parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query')
257