1#!/usr/bin/env python3
2
3# Contributors Listed Below - COPYRIGHT 2018
4# [+] International Business Machines Corp.
5#
6#
7# Licensed under the Apache License, Version 2.0 (the "License");
8# you may not use this file except in compliance with the License.
9# You may obtain a copy of the License at
10#
11#     http://www.apache.org/licenses/LICENSE-2.0
12#
13# Unless required by applicable law or agreed to in writing, software
14# distributed under the License is distributed on an "AS IS" BASIS,
15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
16# implied. See the License for the specific language governing
17# permissions and limitations under the License.
18
19import argparse
20import os
21import sh
22import sys
23
24git = sh.git.bake('--no-pager')
25
26
27def log(msg, args):
28    if args.noisy:
29        sys.stderr.write('{}\n'.format(msg))
30
31
32def git_clone_or_reset(local_name, remote, args):
33    if not os.path.exists(local_name):
34        log('cloning into {}...'.format(local_name), args)
35        git.clone(remote, local_name)
36    else:
37        log('{} exists, updating...'.format(local_name), args)
38        git.fetch(_cwd=local_name)
39        git.reset('--hard', 'FETCH_HEAD', _cwd=local_name)
40
41
42def extract_project_from_uris(uris):
43    # remove SRC_URI = and quotes (does not handle escaped quotes)
44    uris = uris.split('"')[1]
45    for uri in uris.split():
46        if 'github.com/openbmc' not in uri:
47            continue
48
49        # remove fetcher arguments
50        uri = uri.split(';')[0]
51        # the project is the right-most path segment
52        return uri.split('/')[-1].replace('.git', '')
53
54    return None
55
56
57def extract_sha_from_recipe(recipe):
58    with open(recipe) as fp:
59        uris = ''
60        project = None
61        sha = None
62
63        for line in fp:
64            line = line.rstrip()
65            if 'SRCREV' in line:
66                sha = line.split('=')[-1].replace('"', '').strip()
67            elif not project and uris or '_URI' in line:
68                uris += line.split('\\')[0]
69                if '\\' not in line:
70                    # In uris we've gathered a complete (possibly multi-line)
71                    # assignment to a bitbake variable that ends with _URI.
72                    # Try to pull an OpenBMC project out of it.
73                    project = extract_project_from_uris(uris)
74                    if project is None:
75                        # We didn't find a project.  Unset uris and look for
76                        # another bitbake variable that ends with _URI.
77                        uris = ''
78
79            if project and sha:
80                return (project, sha)
81
82        raise RuntimeError('No SRCREV or URI found in {}'.format(recipe))
83
84
85def find_candidate_recipes(meta, args):
86    remote_fmt_args = (args.ssh_config_host, meta)
87    remote = 'ssh://{}/openbmc/{}'.format(*remote_fmt_args)
88    try:
89        git_clone_or_reset(meta, remote, args)
90    except sh.ErrorReturnCode as e:
91        log('{}'.format(e), args)
92        return []
93
94    grep_args = ['-l', '-e', '_URI', '--and', '-e', 'github.com/openbmc']
95    try:
96        return git.grep(*grep_args, _cwd=meta).stdout.decode('utf-8').split()
97    except sh.ErrorReturnCode_1:
98        pass
99    except sh.ErrorReturnCode as e:
100        log('{}'.format(e), args)
101
102    return []
103
104
105def find_and_process_bumps(meta, args):
106    candidate_recipes = find_candidate_recipes(meta, args)
107
108    for recipe in candidate_recipes:
109        full_recipe_path = os.path.join(meta, recipe)
110        recipe_basename = os.path.basename(full_recipe_path)
111        project_name, recipe_sha = extract_sha_from_recipe(full_recipe_path)
112
113        remote_fmt_args = (args.ssh_config_host, project_name)
114        remote = 'ssh://{}/openbmc/{}'.format(*remote_fmt_args)
115        ls_remote_args = [remote, 'refs/heads/{}'.format(args.branch)]
116        try:
117            project_sha = git('ls-remote', *ls_remote_args)
118            project_sha = project_sha.stdout.decode('utf-8').split()[0]
119        except sh.ErrorReturnCode as e:
120            log('{}'.format(e), args)
121            continue
122
123        if project_sha == recipe_sha:
124            message_args = (recipe_basename, recipe_sha[:10])
125            print('{} is up to date ({})'.format(*message_args))
126            continue
127
128        change_id = 'autobump {} {} {}'.format(recipe, recipe_sha, project_sha)
129        hash_object_args = ['-t', 'blob', '--stdin']
130        change_id = git(sh.echo(change_id), 'hash-object', *hash_object_args)
131        change_id = 'I{}'.format(change_id.strip())
132
133        query_args = ['query', 'change:{}'.format(change_id)]
134        gerrit_query_result = args.gerrit(*query_args)
135        gerrit_query_result = gerrit_query_result.stdout.decode('utf-8')
136
137        if (change_id in gerrit_query_result):
138            message_args = (recipe_basename, change_id)
139            print('{} {} already exists'.format(*message_args))
140            continue
141
142        message_args = (recipe_basename, recipe_sha[:10], project_sha[:10])
143        print('{} updating from {} to {}'.format(*message_args))
144
145        remote_args = (args.ssh_config_host, project_name)
146        remote = 'ssh://{}/openbmc/{}'.format(*remote_args)
147        git_clone_or_reset(project_name, remote, args)
148
149        try:
150            revlist = '{}..{}'.format(recipe_sha, project_sha)
151            shortlog = git.shortlog(revlist, _cwd=project_name)
152            shortlog = shortlog.stdout.decode('utf-8')
153        except sh.ErrorReturnCode as e:
154            log('{}'.format(e), args)
155            continue
156
157        reset_args = ['--hard', 'origin/{}'.format(args.branch)]
158        git.reset(*reset_args, _cwd=meta)
159
160        recipe_content = None
161        with open(full_recipe_path) as fd:
162            recipe_content = fd.read()
163
164        recipe_content = recipe_content.replace(recipe_sha, project_sha)
165        with open(full_recipe_path, 'w') as fd:
166            fd.write(recipe_content)
167
168        git.add(recipe, _cwd=meta)
169
170        commit_summary_args = (project_name, recipe_sha[:10], project_sha[:10])
171        commit_msg = '{}: srcrev bump {}..{}'.format(*commit_summary_args)
172        commit_msg += '\n\n{}'.format(shortlog)
173        commit_msg += '\n\nChange-Id: {}'.format(change_id)
174
175        git.commit(sh.echo(commit_msg), '-s', '-F', '-', _cwd=meta)
176
177        push_args = ['origin', 'HEAD:refs/for/{}/autobump'.format(args.branch)]
178        if not args.dry_run:
179            git.push(*push_args, _cwd=meta)
180
181
182def main():
183    app_description = '''OpenBMC bitbake recipe bumping tool.
184
185Find bitbake metadata files (recipes) that use the git fetcher
186and check the project repository for newer revisions.
187
188Generate commits that update bitbake metadata files with SRCREV.
189
190Push generated commits to the OpenBMC Gerrit instance for review.
191    '''
192    parser = argparse.ArgumentParser(
193        description=app_description,
194        formatter_class=argparse.RawDescriptionHelpFormatter)
195
196    parser.set_defaults(branch='master')
197    parser.add_argument(
198        '-d', '--dry-run', dest='dry_run', action='store_true',
199        help='perform a dry run only')
200    parser.add_argument(
201        '-m', '--meta-repository', dest='meta_repository', action='append',
202        help='meta repository to check for updates')
203    parser.add_argument(
204        '-v', '--verbose', dest='noisy', action='store_true',
205        help='enable verbose status messages')
206    parser.add_argument(
207        'ssh_config_host', metavar='SSH_CONFIG_HOST_ENTRY',
208        help='SSH config host entry for Gerrit connectivity')
209
210    args = parser.parse_args()
211    setattr(args, 'gerrit', sh.ssh.bake(args.ssh_config_host, 'gerrit'))
212
213    metas = getattr(args, 'meta_repository')
214    if metas is None:
215        metas = args.gerrit('ls-projects', '-m', 'meta-')
216        metas = metas.stdout.decode('utf-8').split()
217        metas = [os.path.split(x)[-1] for x in metas]
218
219    for meta in metas:
220        find_and_process_bumps(meta, args)
221
222
223if __name__ == '__main__':
224    sys.exit(0 if main() else 1)
225