1#!/usr/bin/env python3 2 3# Contributors Listed Below - COPYRIGHT 2018 4# [+] International Business Machines Corp. 5# 6# 7# Licensed under the Apache License, Version 2.0 (the "License"); 8# you may not use this file except in compliance with the License. 9# You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, software 14# distributed under the License is distributed on an "AS IS" BASIS, 15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 16# implied. See the License for the specific language governing 17# permissions and limitations under the License. 18 19import argparse 20import os 21import sys 22 23import sh 24 25git = sh.git.bake("--no-pager") 26 27 28def log(msg, args): 29 if args.noisy: 30 sys.stderr.write("{}\n".format(msg)) 31 32 33def git_clone_or_reset(local_name, remote, args): 34 if not os.path.exists(local_name): 35 log("cloning into {}...".format(local_name), args) 36 git.clone(remote, local_name) 37 else: 38 log("{} exists, updating...".format(local_name), args) 39 git.fetch(_cwd=local_name) 40 git.reset("--hard", "FETCH_HEAD", _cwd=local_name) 41 42 43def extract_project_from_uris(uris): 44 # remove SRC_URI = and quotes (does not handle escaped quotes) 45 uris = uris.split('"')[1] 46 for uri in uris.split(): 47 if "github.com/openbmc" not in uri: 48 continue 49 50 # remove fetcher arguments 51 uri = uri.split(";")[0] 52 # the project is the right-most path segment 53 return uri.split("/")[-1].replace(".git", "") 54 55 return None 56 57 58def extract_sha_from_recipe(recipe): 59 with open(recipe) as fp: 60 uris = "" 61 project = None 62 sha = None 63 64 for line in fp: 65 line = line.rstrip() 66 if "SRCREV" in line: 67 sha = line.split("=")[-1].replace('"', "").strip() 68 elif not project and uris or "_URI" in line: 69 uris += line.split("\\")[0] 70 if "\\" not in line: 71 # In uris we've gathered a complete (possibly multi-line) 72 # assignment to a bitbake variable that ends with _URI. 73 # Try to pull an OpenBMC project out of it. 74 project = extract_project_from_uris(uris) 75 if project is None: 76 # We didn't find a project. Unset uris and look for 77 # another bitbake variable that ends with _URI. 78 uris = "" 79 80 if project and sha: 81 return (project, sha) 82 83 raise RuntimeError("No SRCREV or URI found in {}".format(recipe)) 84 85 86def find_candidate_recipes(meta, args): 87 remote_fmt_args = (args.ssh_config_host, meta) 88 remote = "ssh://{}/openbmc/{}".format(*remote_fmt_args) 89 try: 90 git_clone_or_reset(meta, remote, args) 91 except sh.ErrorReturnCode as e: 92 log("{}".format(e), args) 93 return [] 94 95 match_suffixes = ("bb", "bbclass", "inc") 96 pathspecs = ("*.{}".format(x) for x in match_suffixes) 97 grep_args = ("-l", "-e", "_URI", "--and", "-e", "github.com/openbmc") 98 grep_args = (*grep_args, *pathspecs) 99 try: 100 return git.grep(*grep_args, _cwd=meta).stdout.decode("utf-8").split() 101 except sh.ErrorReturnCode_1: 102 pass 103 except sh.ErrorReturnCode as e: 104 log("{}".format(e), args) 105 106 return [] 107 108 109def find_and_process_bumps(meta, args): 110 candidate_recipes = find_candidate_recipes(meta, args) 111 112 for recipe in candidate_recipes: 113 full_recipe_path = os.path.join(meta, recipe) 114 recipe_basename = os.path.basename(full_recipe_path) 115 project_name, recipe_sha = extract_sha_from_recipe(full_recipe_path) 116 117 remote_fmt_args = (args.ssh_config_host, project_name) 118 remote = "ssh://{}/openbmc/{}".format(*remote_fmt_args) 119 ls_remote_args = [remote, "refs/heads/{}".format(args.branch)] 120 try: 121 project_sha = git("ls-remote", *ls_remote_args) 122 project_sha = project_sha.stdout.decode("utf-8").split()[0] 123 except sh.ErrorReturnCode as e: 124 log("{}".format(e), args) 125 continue 126 127 if project_sha == recipe_sha: 128 message_args = (recipe_basename, recipe_sha[:10]) 129 print("{} is up to date ({})".format(*message_args)) 130 continue 131 132 change_id = "autobump {} {} {}".format(recipe, recipe_sha, project_sha) 133 hash_object_args = ["-t", "blob", "--stdin"] 134 change_id = git(sh.echo(change_id), "hash-object", *hash_object_args) 135 change_id = "I{}".format(change_id.strip()) 136 137 query_args = ["query", "change:{}".format(change_id)] 138 gerrit_query_result = args.gerrit(*query_args) 139 gerrit_query_result = gerrit_query_result.stdout.decode("utf-8") 140 141 if change_id in gerrit_query_result: 142 message_args = (recipe_basename, change_id) 143 print("{} {} already exists".format(*message_args)) 144 continue 145 146 message_args = (recipe_basename, recipe_sha[:10], project_sha[:10]) 147 print("{} updating from {} to {}".format(*message_args)) 148 149 remote_args = (args.ssh_config_host, project_name) 150 remote = "ssh://{}/openbmc/{}".format(*remote_args) 151 git_clone_or_reset(project_name, remote, args) 152 153 try: 154 revlist = "{}..{}".format(recipe_sha, project_sha) 155 shortlog = git.shortlog(revlist, _cwd=project_name) 156 shortlog = shortlog.stdout.decode("utf-8") 157 except sh.ErrorReturnCode as e: 158 log("{}".format(e), args) 159 continue 160 161 reset_args = ["--hard", "origin/{}".format(args.branch)] 162 git.reset(*reset_args, _cwd=meta) 163 164 recipe_content = None 165 with open(full_recipe_path) as fd: 166 recipe_content = fd.read() 167 168 recipe_content = recipe_content.replace(recipe_sha, project_sha) 169 with open(full_recipe_path, "w") as fd: 170 fd.write(recipe_content) 171 172 git.add(recipe, _cwd=meta) 173 174 commit_summary_args = (project_name, recipe_sha[:10], project_sha[:10]) 175 commit_msg = "{}: srcrev bump {}..{}".format(*commit_summary_args) 176 commit_msg += "\n\n{}".format(shortlog) 177 commit_msg += "\n\nChange-Id: {}".format(change_id) 178 179 git.commit(sh.echo(commit_msg), "-s", "-F", "-", _cwd=meta) 180 181 push_args = [ 182 "origin", 183 "HEAD:refs/for/{}%topic=autobump".format(args.branch), 184 ] 185 if not args.dry_run: 186 git.push(*push_args, _cwd=meta) 187 188 189def main(): 190 app_description = """OpenBMC bitbake recipe bumping tool. 191 192Find bitbake metadata files (recipes) that use the git fetcher 193and check the project repository for newer revisions. 194 195Generate commits that update bitbake metadata files with SRCREV. 196 197Push generated commits to the OpenBMC Gerrit instance for review. 198 """ 199 parser = argparse.ArgumentParser( 200 description=app_description, 201 formatter_class=argparse.RawDescriptionHelpFormatter, 202 ) 203 204 parser.set_defaults(branch="master") 205 parser.add_argument( 206 "-d", 207 "--dry-run", 208 dest="dry_run", 209 action="store_true", 210 help="perform a dry run only", 211 ) 212 parser.add_argument( 213 "-m", 214 "--meta-repository", 215 dest="meta_repository", 216 action="append", 217 help="meta repository to check for updates", 218 ) 219 parser.add_argument( 220 "-v", 221 "--verbose", 222 dest="noisy", 223 action="store_true", 224 help="enable verbose status messages", 225 ) 226 parser.add_argument( 227 "ssh_config_host", 228 metavar="SSH_CONFIG_HOST_ENTRY", 229 help="SSH config host entry for Gerrit connectivity", 230 ) 231 232 args = parser.parse_args() 233 setattr(args, "gerrit", sh.ssh.bake(args.ssh_config_host, "gerrit")) 234 235 metas = getattr(args, "meta_repository") 236 if metas is None: 237 metas = args.gerrit("ls-projects", "-m", "meta-") 238 metas = metas.stdout.decode("utf-8").split() 239 metas = [os.path.split(x)[-1] for x in metas] 240 241 for meta in metas: 242 find_and_process_bumps(meta, args) 243 244 245if __name__ == "__main__": 246 sys.exit(0 if main() else 1) 247