1#!/usr/bin/env python3
2
3# Contributors Listed Below - COPYRIGHT 2018
4# [+] International Business Machines Corp.
5#
6#
7# Licensed under the Apache License, Version 2.0 (the "License");
8# you may not use this file except in compliance with the License.
9# You may obtain a copy of the License at
10#
11#     http://www.apache.org/licenses/LICENSE-2.0
12#
13# Unless required by applicable law or agreed to in writing, software
14# distributed under the License is distributed on an "AS IS" BASIS,
15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
16# implied. See the License for the specific language governing
17# permissions and limitations under the License.
18
19import argparse
20import os
21import sh
22import sys
23
24git = sh.git.bake('--no-pager')
25
26
27def log(msg, args):
28    if args.noisy:
29        sys.stderr.write('{}\n'.format(msg))
30
31
32def git_clone_or_reset(local_name, remote, args):
33    if not os.path.exists(local_name):
34        log('cloning into {}...'.format(local_name), args)
35        git.clone(remote, local_name)
36    else:
37        log('{} exists, updating...'.format(local_name), args)
38        git.fetch(_cwd=local_name)
39        git.reset('--hard', 'FETCH_HEAD', _cwd=local_name)
40
41
42def extract_project_from_uris(uris):
43    # remove SRC_URI = and quotes (does not handle escaped quotes)
44    uris = uris.split('"')[1]
45    for uri in uris.split():
46        if 'github.com/openbmc' not in uri:
47            continue
48
49        # remove fetcher arguments
50        uri = uri.split(';')[0]
51        # the project is the right-most path segment
52        return uri.split('/')[-1].replace('.git', '')
53
54    return None
55
56
57def extract_sha_from_recipe(recipe):
58    with open(recipe) as fp:
59        uris = ''
60        project = None
61        sha = None
62
63        for line in fp:
64            line = line.rstrip()
65            if 'SRCREV' in line:
66                sha = line.split('=')[-1].replace('"', '').strip()
67            elif not project and uris or '_URI' in line:
68                uris += line.split('\\')[0]
69                if '\\' not in line:
70                    # In uris we've gathered a complete (possibly multi-line)
71                    # assignment to a bitbake variable that ends with _URI.
72                    # Try to pull an OpenBMC project out of it.
73                    project = extract_project_from_uris(uris)
74                    if project is None:
75                        # We didn't find a project.  Unset uris and look for
76                        # another bitbake variable that ends with _URI.
77                        uris = ''
78
79            if project and sha:
80                return (project, sha)
81
82        raise RuntimeError('No SRCREV or URI found in {}'.format(recipe))
83
84
85def find_candidate_recipes(meta, args):
86    remote_fmt_args = (args.ssh_config_host, meta)
87    remote = 'ssh://{}/openbmc/{}'.format(*remote_fmt_args)
88    try:
89        git_clone_or_reset(meta, remote, args)
90    except sh.ErrorReturnCode as e:
91        log('{}'.format(e), args)
92        return []
93
94    match_suffixes = ('bb', 'bbclass', 'inc')
95    pathspecs = ('*.{}'.format(x) for x in match_suffixes)
96    grep_args = ('-l', '-e', '_URI', '--and', '-e', 'github.com/openbmc')
97    grep_args = (*grep_args, *pathspecs)
98    try:
99        return git.grep(*grep_args, _cwd=meta).stdout.decode('utf-8').split()
100    except sh.ErrorReturnCode_1:
101        pass
102    except sh.ErrorReturnCode as e:
103        log('{}'.format(e), args)
104
105    return []
106
107
108def find_and_process_bumps(meta, args):
109    candidate_recipes = find_candidate_recipes(meta, args)
110
111    for recipe in candidate_recipes:
112        full_recipe_path = os.path.join(meta, recipe)
113        recipe_basename = os.path.basename(full_recipe_path)
114        project_name, recipe_sha = extract_sha_from_recipe(full_recipe_path)
115
116        remote_fmt_args = (args.ssh_config_host, project_name)
117        remote = 'ssh://{}/openbmc/{}'.format(*remote_fmt_args)
118        ls_remote_args = [remote, 'refs/heads/{}'.format(args.branch)]
119        try:
120            project_sha = git('ls-remote', *ls_remote_args)
121            project_sha = project_sha.stdout.decode('utf-8').split()[0]
122        except sh.ErrorReturnCode as e:
123            log('{}'.format(e), args)
124            continue
125
126        if project_sha == recipe_sha:
127            message_args = (recipe_basename, recipe_sha[:10])
128            print('{} is up to date ({})'.format(*message_args))
129            continue
130
131        change_id = 'autobump {} {} {}'.format(recipe, recipe_sha, project_sha)
132        hash_object_args = ['-t', 'blob', '--stdin']
133        change_id = git(sh.echo(change_id), 'hash-object', *hash_object_args)
134        change_id = 'I{}'.format(change_id.strip())
135
136        query_args = ['query', 'change:{}'.format(change_id)]
137        gerrit_query_result = args.gerrit(*query_args)
138        gerrit_query_result = gerrit_query_result.stdout.decode('utf-8')
139
140        if (change_id in gerrit_query_result):
141            message_args = (recipe_basename, change_id)
142            print('{} {} already exists'.format(*message_args))
143            continue
144
145        message_args = (recipe_basename, recipe_sha[:10], project_sha[:10])
146        print('{} updating from {} to {}'.format(*message_args))
147
148        remote_args = (args.ssh_config_host, project_name)
149        remote = 'ssh://{}/openbmc/{}'.format(*remote_args)
150        git_clone_or_reset(project_name, remote, args)
151
152        try:
153            revlist = '{}..{}'.format(recipe_sha, project_sha)
154            shortlog = git.shortlog(revlist, _cwd=project_name)
155            shortlog = shortlog.stdout.decode('utf-8')
156        except sh.ErrorReturnCode as e:
157            log('{}'.format(e), args)
158            continue
159
160        reset_args = ['--hard', 'origin/{}'.format(args.branch)]
161        git.reset(*reset_args, _cwd=meta)
162
163        recipe_content = None
164        with open(full_recipe_path) as fd:
165            recipe_content = fd.read()
166
167        recipe_content = recipe_content.replace(recipe_sha, project_sha)
168        with open(full_recipe_path, 'w') as fd:
169            fd.write(recipe_content)
170
171        git.add(recipe, _cwd=meta)
172
173        commit_summary_args = (project_name, recipe_sha[:10], project_sha[:10])
174        commit_msg = '{}: srcrev bump {}..{}'.format(*commit_summary_args)
175        commit_msg += '\n\n{}'.format(shortlog)
176        commit_msg += '\n\nChange-Id: {}'.format(change_id)
177
178        git.commit(sh.echo(commit_msg), '-s', '-F', '-', _cwd=meta)
179
180        push_args = [
181            'origin',
182            'HEAD:refs/for/{}%topic=autobump'.format(args.branch)
183        ]
184        if not args.dry_run:
185            git.push(*push_args, _cwd=meta)
186
187
188def main():
189    app_description = '''OpenBMC bitbake recipe bumping tool.
190
191Find bitbake metadata files (recipes) that use the git fetcher
192and check the project repository for newer revisions.
193
194Generate commits that update bitbake metadata files with SRCREV.
195
196Push generated commits to the OpenBMC Gerrit instance for review.
197    '''
198    parser = argparse.ArgumentParser(
199        description=app_description,
200        formatter_class=argparse.RawDescriptionHelpFormatter)
201
202    parser.set_defaults(branch='master')
203    parser.add_argument(
204        '-d', '--dry-run', dest='dry_run', action='store_true',
205        help='perform a dry run only')
206    parser.add_argument(
207        '-m', '--meta-repository', dest='meta_repository', action='append',
208        help='meta repository to check for updates')
209    parser.add_argument(
210        '-v', '--verbose', dest='noisy', action='store_true',
211        help='enable verbose status messages')
212    parser.add_argument(
213        'ssh_config_host', metavar='SSH_CONFIG_HOST_ENTRY',
214        help='SSH config host entry for Gerrit connectivity')
215
216    args = parser.parse_args()
217    setattr(args, 'gerrit', sh.ssh.bake(args.ssh_config_host, 'gerrit'))
218
219    metas = getattr(args, 'meta_repository')
220    if metas is None:
221        metas = args.gerrit('ls-projects', '-m', 'meta-')
222        metas = metas.stdout.decode('utf-8').split()
223        metas = [os.path.split(x)[-1] for x in metas]
224
225    for meta in metas:
226        find_and_process_bumps(meta, args)
227
228
229if __name__ == '__main__':
230    sys.exit(0 if main() else 1)
231