1#!/usr/bin/env python3
2
3# Contributors Listed Below - COPYRIGHT 2018
4# [+] International Business Machines Corp.
5#
6#
7# Licensed under the Apache License, Version 2.0 (the "License");
8# you may not use this file except in compliance with the License.
9# You may obtain a copy of the License at
10#
11#     http://www.apache.org/licenses/LICENSE-2.0
12#
13# Unless required by applicable law or agreed to in writing, software
14# distributed under the License is distributed on an "AS IS" BASIS,
15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
16# implied. See the License for the specific language governing
17# permissions and limitations under the License.
18
19import argparse
20import os
21import sys
22
23import sh
24
25git = sh.git.bake("--no-pager")
26
27
28def log(msg, args):
29    if args.noisy:
30        sys.stderr.write("{}\n".format(msg))
31
32
33def git_clone_or_reset(local_name, remote, args):
34    if not os.path.exists(local_name):
35        log("cloning into {}...".format(local_name), args)
36        git.clone(remote, local_name)
37    else:
38        log("{} exists, updating...".format(local_name), args)
39        git.fetch(_cwd=local_name)
40        git.reset("--hard", "FETCH_HEAD", _cwd=local_name)
41
42
43def extract_project_from_uris(uris, args):
44    # remove SRC_URI = and quotes (does not handle escaped quotes)
45    uris = uris.split('"')[1]
46    for uri in uris.split():
47        if "github.com/openbmc" not in uri:
48            continue
49
50        segments = uri.split(";")
51        branch = args.branch
52
53        for s in segments[1:]:
54            if "branch=" not in s:
55                continue
56            if "nobranch=" in s:
57                continue
58            branch = s.split("=")[1]
59
60        # remove fetcher arguments
61        uri = segments[0]
62        # the project is the right-most path segment
63        return (uri.split("/")[-1].replace(".git", ""), branch)
64
65    return None
66
67
68def extract_sha_from_recipe(recipe, args):
69    with open(recipe) as fp:
70        uris = ""
71        project = None
72        branch = None
73        sha = None
74
75        for line in fp:
76            line = line.rstrip()
77            if "SRCREV" in line:
78                sha = line.split("=")[-1].replace('"', "").strip()
79            elif not project and uris or "_URI" in line:
80                uris += line.split("\\")[0]
81                if "\\" not in line:
82                    # In uris we've gathered a complete (possibly multi-line)
83                    # assignment to a bitbake variable that ends with _URI.
84                    # Try to pull an OpenBMC project out of it.
85                    uri = extract_project_from_uris(uris, args)
86                    if uri is None:
87                        # We didn't find a project.  Unset uris and look for
88                        # another bitbake variable that ends with _URI.
89                        uris = ""
90                    else:
91                        (project, branch) = uri
92
93            if project and branch and sha:
94                return (project, branch, sha)
95
96        raise RuntimeError("No SRCREV or URI found in {}".format(recipe))
97
98
99def find_candidate_recipes(meta, args):
100    remote_fmt_args = (args.ssh_config_host, meta)
101    remote = "ssh://{}/openbmc/{}".format(*remote_fmt_args)
102    try:
103        git_clone_or_reset(meta, remote, args)
104    except sh.ErrorReturnCode as e:
105        log("{}".format(e), args)
106        return []
107
108    match_suffixes = ("bb", "bbclass", "inc")
109    pathspecs = ("*.{}".format(x) for x in match_suffixes)
110    grep_args = (
111        "--no-color",
112        "-l",
113        "-e",
114        "_URI",
115        "--and",
116        "-e",
117        "github.com/openbmc",
118    )
119    grep_args = (*grep_args, *pathspecs)
120    try:
121        return git.grep(*grep_args, _cwd=meta).stdout.decode("utf-8").split()
122    except sh.ErrorReturnCode_1:
123        pass
124    except sh.ErrorReturnCode as e:
125        log("{}".format(e), args)
126
127    return []
128
129
130def find_and_process_bumps(meta, args):
131    candidate_recipes = find_candidate_recipes(meta, args)
132
133    for recipe in candidate_recipes:
134        full_recipe_path = os.path.join(meta, recipe)
135        recipe_basename = os.path.basename(full_recipe_path)
136        project_name, recipe_branch, recipe_sha = extract_sha_from_recipe(
137            full_recipe_path, args
138        )
139
140        remote_fmt_args = (args.ssh_config_host, project_name)
141        remote = "ssh://{}/openbmc/{}".format(*remote_fmt_args)
142        ls_remote_args = [remote, "refs/heads/{}".format(recipe_branch)]
143        try:
144            project_sha = git("ls-remote", *ls_remote_args)
145            project_sha = project_sha.stdout.decode("utf-8").split()[0]
146        except sh.ErrorReturnCode as e:
147            log("{}".format(e), args)
148            continue
149
150        if project_sha == recipe_sha:
151            message_args = (recipe_basename, recipe_sha[:10])
152            print("{} is up to date ({})".format(*message_args))
153            continue
154
155        change_id = "autobump {} {} {}".format(recipe, recipe_sha, project_sha)
156        hash_object_args = ["-t", "blob", "--stdin"]
157        change_id = git(sh.echo(change_id), "hash-object", *hash_object_args)
158        change_id = "I{}".format(change_id.strip())
159
160        query_args = ["query", "change:{}".format(change_id)]
161        gerrit_query_result = args.gerrit(*query_args)
162        gerrit_query_result = gerrit_query_result.stdout.decode("utf-8")
163
164        if change_id in gerrit_query_result:
165            message_args = (recipe_basename, change_id)
166            print("{} {} already exists".format(*message_args))
167            continue
168
169        message_args = (recipe_basename, recipe_sha[:10], project_sha[:10])
170        print("{} updating from {} to {}".format(*message_args))
171
172        remote_args = (args.ssh_config_host, project_name)
173        remote = "ssh://{}/openbmc/{}".format(*remote_args)
174        git_clone_or_reset(project_name, remote, args)
175
176        try:
177            revlist = "{}..{}".format(recipe_sha, project_sha)
178            shortlog = git.shortlog(revlist, _cwd=project_name)
179            shortlog = shortlog.stdout.decode("utf-8")
180        except sh.ErrorReturnCode as e:
181            log("{}".format(e), args)
182            continue
183
184        reset_args = ["--hard", "origin/{}".format(args.branch)]
185        git.reset(*reset_args, _cwd=meta)
186
187        recipe_content = None
188        with open(full_recipe_path) as fd:
189            recipe_content = fd.read()
190
191        recipe_content = recipe_content.replace(recipe_sha, project_sha)
192        with open(full_recipe_path, "w") as fd:
193            fd.write(recipe_content)
194
195        git.add(recipe, _cwd=meta)
196
197        commit_summary_args = (project_name, recipe_sha[:10], project_sha[:10])
198        commit_msg = "{}: srcrev bump {}..{}".format(*commit_summary_args)
199        commit_msg += "\n\n{}".format(shortlog)
200        commit_msg += "\n\nChange-Id: {}".format(change_id)
201
202        git.commit(sh.echo(commit_msg), "-s", "-F", "-", _cwd=meta)
203
204        push_args = [
205            "origin",
206            "HEAD:refs/for/{}%topic=autobump".format(args.branch),
207        ]
208        if not args.dry_run:
209            git.push(*push_args, _cwd=meta)
210
211
212def main():
213    app_description = """OpenBMC bitbake recipe bumping tool.
214
215Find bitbake metadata files (recipes) that use the git fetcher
216and check the project repository for newer revisions.
217
218Generate commits that update bitbake metadata files with SRCREV.
219
220Push generated commits to the OpenBMC Gerrit instance for review.
221    """
222    parser = argparse.ArgumentParser(
223        description=app_description,
224        formatter_class=argparse.RawDescriptionHelpFormatter,
225    )
226
227    parser.set_defaults(branch="master")
228    parser.add_argument(
229        "-d",
230        "--dry-run",
231        dest="dry_run",
232        action="store_true",
233        help="perform a dry run only",
234    )
235    parser.add_argument(
236        "-m",
237        "--meta-repository",
238        dest="meta_repository",
239        action="append",
240        help="meta repository to check for updates",
241    )
242    parser.add_argument(
243        "-v",
244        "--verbose",
245        dest="noisy",
246        action="store_true",
247        help="enable verbose status messages",
248    )
249    parser.add_argument(
250        "ssh_config_host",
251        metavar="SSH_CONFIG_HOST_ENTRY",
252        help="SSH config host entry for Gerrit connectivity",
253    )
254
255    args = parser.parse_args()
256    setattr(args, "gerrit", sh.ssh.bake(args.ssh_config_host, "gerrit"))
257
258    metas = getattr(args, "meta_repository")
259    if metas is None:
260        metas = args.gerrit("ls-projects", "-m", "meta-")
261        metas = metas.stdout.decode("utf-8").split()
262        metas = [os.path.split(x)[-1] for x in metas]
263
264    for meta in metas:
265        find_and_process_bumps(meta, args)
266
267
268if __name__ == "__main__":
269    sys.exit(0 if main() else 1)
270