1cf3c1e67SAndrew Jeffery#!/usr/bin/env python3 2cf3c1e67SAndrew Jeffery 3cf3c1e67SAndrew Jeffery# Contributors Listed Below - COPYRIGHT 2018 4cf3c1e67SAndrew Jeffery# [+] International Business Machines Corp. 5cf3c1e67SAndrew Jeffery# 6cf3c1e67SAndrew Jeffery# 7cf3c1e67SAndrew Jeffery# Licensed under the Apache License, Version 2.0 (the "License"); 8cf3c1e67SAndrew Jeffery# you may not use this file except in compliance with the License. 9cf3c1e67SAndrew Jeffery# You may obtain a copy of the License at 10cf3c1e67SAndrew Jeffery# 11cf3c1e67SAndrew Jeffery# http://www.apache.org/licenses/LICENSE-2.0 12cf3c1e67SAndrew Jeffery# 13cf3c1e67SAndrew Jeffery# Unless required by applicable law or agreed to in writing, software 14cf3c1e67SAndrew Jeffery# distributed under the License is distributed on an "AS IS" BASIS, 15cf3c1e67SAndrew Jeffery# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 16cf3c1e67SAndrew Jeffery# implied. See the License for the specific language governing 17cf3c1e67SAndrew Jeffery# permissions and limitations under the License. 18cf3c1e67SAndrew Jeffery 19cf3c1e67SAndrew Jefferyimport argparse 20cf3c1e67SAndrew Jefferyimport os 21cf3c1e67SAndrew Jefferyimport sys 22cf3c1e67SAndrew Jeffery 23e310dd91SPatrick Williamsimport sh 24e310dd91SPatrick Williams 25e310dd91SPatrick Williamsgit = sh.git.bake("--no-pager") 26cf3c1e67SAndrew Jeffery 27cf3c1e67SAndrew Jeffery 28cf3c1e67SAndrew Jefferydef log(msg, args): 29cf3c1e67SAndrew Jeffery if args.noisy: 30e310dd91SPatrick Williams sys.stderr.write("{}\n".format(msg)) 31cf3c1e67SAndrew Jeffery 32cf3c1e67SAndrew Jeffery 33cf3c1e67SAndrew Jefferydef git_clone_or_reset(local_name, remote, args): 34cf3c1e67SAndrew Jeffery if not os.path.exists(local_name): 35e310dd91SPatrick Williams log("cloning into {}...".format(local_name), args) 36cf3c1e67SAndrew Jeffery git.clone(remote, local_name) 37cf3c1e67SAndrew Jeffery else: 38e310dd91SPatrick Williams log("{} exists, updating...".format(local_name), args) 39cf3c1e67SAndrew Jeffery git.fetch(_cwd=local_name) 40e310dd91SPatrick Williams git.reset("--hard", "FETCH_HEAD", _cwd=local_name) 41cf3c1e67SAndrew Jeffery 42cf3c1e67SAndrew Jeffery 43*ffb4d52eSPatrick Williamsdef extract_project_from_uris(uris, args): 44cf3c1e67SAndrew Jeffery # remove SRC_URI = and quotes (does not handle escaped quotes) 45cf3c1e67SAndrew Jeffery uris = uris.split('"')[1] 46cf3c1e67SAndrew Jeffery for uri in uris.split(): 47e310dd91SPatrick Williams if "github.com/openbmc" not in uri: 48cf3c1e67SAndrew Jeffery continue 49cf3c1e67SAndrew Jeffery 50*ffb4d52eSPatrick Williams segments = uri.split(";") 51*ffb4d52eSPatrick Williams branch = args.branch 52*ffb4d52eSPatrick Williams 53*ffb4d52eSPatrick Williams for s in segments[1:]: 54*ffb4d52eSPatrick Williams if "branch=" not in s: 55*ffb4d52eSPatrick Williams continue 56*ffb4d52eSPatrick Williams if "nobranch=" in s: 57*ffb4d52eSPatrick Williams continue 58*ffb4d52eSPatrick Williams branch = s.split("=")[1] 59*ffb4d52eSPatrick Williams 60cf3c1e67SAndrew Jeffery # remove fetcher arguments 61*ffb4d52eSPatrick Williams uri = segments[0] 62cf3c1e67SAndrew Jeffery # the project is the right-most path segment 63*ffb4d52eSPatrick Williams return (uri.split("/")[-1].replace(".git", ""), branch) 64cf3c1e67SAndrew Jeffery 65cf3c1e67SAndrew Jeffery return None 66cf3c1e67SAndrew Jeffery 67cf3c1e67SAndrew Jeffery 68*ffb4d52eSPatrick Williamsdef extract_sha_from_recipe(recipe, args): 69cf3c1e67SAndrew Jeffery with open(recipe) as fp: 70e310dd91SPatrick Williams uris = "" 71cf3c1e67SAndrew Jeffery project = None 72*ffb4d52eSPatrick Williams branch = None 73cf3c1e67SAndrew Jeffery sha = None 74cf3c1e67SAndrew Jeffery 75cf3c1e67SAndrew Jeffery for line in fp: 76cf3c1e67SAndrew Jeffery line = line.rstrip() 77e310dd91SPatrick Williams if "SRCREV" in line: 78e310dd91SPatrick Williams sha = line.split("=")[-1].replace('"', "").strip() 79e310dd91SPatrick Williams elif not project and uris or "_URI" in line: 80e310dd91SPatrick Williams uris += line.split("\\")[0] 81e310dd91SPatrick Williams if "\\" not in line: 82cf3c1e67SAndrew Jeffery # In uris we've gathered a complete (possibly multi-line) 83cf3c1e67SAndrew Jeffery # assignment to a bitbake variable that ends with _URI. 84cf3c1e67SAndrew Jeffery # Try to pull an OpenBMC project out of it. 85*ffb4d52eSPatrick Williams uri = extract_project_from_uris(uris, args) 86*ffb4d52eSPatrick Williams if uri is None: 87cf3c1e67SAndrew Jeffery # We didn't find a project. Unset uris and look for 88cf3c1e67SAndrew Jeffery # another bitbake variable that ends with _URI. 89e310dd91SPatrick Williams uris = "" 90*ffb4d52eSPatrick Williams else: 91*ffb4d52eSPatrick Williams (project, branch) = uri 92cf3c1e67SAndrew Jeffery 93*ffb4d52eSPatrick Williams if project and branch and sha: 94*ffb4d52eSPatrick Williams return (project, branch, sha) 95cf3c1e67SAndrew Jeffery 96e310dd91SPatrick Williams raise RuntimeError("No SRCREV or URI found in {}".format(recipe)) 97cf3c1e67SAndrew Jeffery 98cf3c1e67SAndrew Jeffery 99cf3c1e67SAndrew Jefferydef find_candidate_recipes(meta, args): 100cf3c1e67SAndrew Jeffery remote_fmt_args = (args.ssh_config_host, meta) 101e310dd91SPatrick Williams remote = "ssh://{}/openbmc/{}".format(*remote_fmt_args) 102cf3c1e67SAndrew Jeffery try: 103cf3c1e67SAndrew Jeffery git_clone_or_reset(meta, remote, args) 104cf3c1e67SAndrew Jeffery except sh.ErrorReturnCode as e: 105e310dd91SPatrick Williams log("{}".format(e), args) 106cf3c1e67SAndrew Jeffery return [] 107cf3c1e67SAndrew Jeffery 108e310dd91SPatrick Williams match_suffixes = ("bb", "bbclass", "inc") 109e310dd91SPatrick Williams pathspecs = ("*.{}".format(x) for x in match_suffixes) 110*ffb4d52eSPatrick Williams grep_args = ( 111*ffb4d52eSPatrick Williams "--no-color", 112*ffb4d52eSPatrick Williams "-l", 113*ffb4d52eSPatrick Williams "-e", 114*ffb4d52eSPatrick Williams "_URI", 115*ffb4d52eSPatrick Williams "--and", 116*ffb4d52eSPatrick Williams "-e", 117*ffb4d52eSPatrick Williams "github.com/openbmc", 118*ffb4d52eSPatrick Williams ) 1192b6a546bSBrad Bishop grep_args = (*grep_args, *pathspecs) 120cf3c1e67SAndrew Jeffery try: 121e310dd91SPatrick Williams return git.grep(*grep_args, _cwd=meta).stdout.decode("utf-8").split() 122cf3c1e67SAndrew Jeffery except sh.ErrorReturnCode_1: 123cf3c1e67SAndrew Jeffery pass 124cf3c1e67SAndrew Jeffery except sh.ErrorReturnCode as e: 125e310dd91SPatrick Williams log("{}".format(e), args) 126cf3c1e67SAndrew Jeffery 127cf3c1e67SAndrew Jeffery return [] 128cf3c1e67SAndrew Jeffery 129cf3c1e67SAndrew Jeffery 130cf3c1e67SAndrew Jefferydef find_and_process_bumps(meta, args): 131cf3c1e67SAndrew Jeffery candidate_recipes = find_candidate_recipes(meta, args) 132cf3c1e67SAndrew Jeffery 133cf3c1e67SAndrew Jeffery for recipe in candidate_recipes: 134cf3c1e67SAndrew Jeffery full_recipe_path = os.path.join(meta, recipe) 135cf3c1e67SAndrew Jeffery recipe_basename = os.path.basename(full_recipe_path) 136*ffb4d52eSPatrick Williams project_name, recipe_branch, recipe_sha = extract_sha_from_recipe( 137*ffb4d52eSPatrick Williams full_recipe_path, args 138*ffb4d52eSPatrick Williams ) 139cf3c1e67SAndrew Jeffery 140cf3c1e67SAndrew Jeffery remote_fmt_args = (args.ssh_config_host, project_name) 141e310dd91SPatrick Williams remote = "ssh://{}/openbmc/{}".format(*remote_fmt_args) 142*ffb4d52eSPatrick Williams ls_remote_args = [remote, "refs/heads/{}".format(recipe_branch)] 143cf3c1e67SAndrew Jeffery try: 144e310dd91SPatrick Williams project_sha = git("ls-remote", *ls_remote_args) 145e310dd91SPatrick Williams project_sha = project_sha.stdout.decode("utf-8").split()[0] 146cf3c1e67SAndrew Jeffery except sh.ErrorReturnCode as e: 147e310dd91SPatrick Williams log("{}".format(e), args) 148cf3c1e67SAndrew Jeffery continue 149cf3c1e67SAndrew Jeffery 150cf3c1e67SAndrew Jeffery if project_sha == recipe_sha: 151cf3c1e67SAndrew Jeffery message_args = (recipe_basename, recipe_sha[:10]) 152e310dd91SPatrick Williams print("{} is up to date ({})".format(*message_args)) 153cf3c1e67SAndrew Jeffery continue 154cf3c1e67SAndrew Jeffery 155e310dd91SPatrick Williams change_id = "autobump {} {} {}".format(recipe, recipe_sha, project_sha) 156e310dd91SPatrick Williams hash_object_args = ["-t", "blob", "--stdin"] 157e310dd91SPatrick Williams change_id = git(sh.echo(change_id), "hash-object", *hash_object_args) 158e310dd91SPatrick Williams change_id = "I{}".format(change_id.strip()) 159cf3c1e67SAndrew Jeffery 160e310dd91SPatrick Williams query_args = ["query", "change:{}".format(change_id)] 161cf3c1e67SAndrew Jeffery gerrit_query_result = args.gerrit(*query_args) 162e310dd91SPatrick Williams gerrit_query_result = gerrit_query_result.stdout.decode("utf-8") 163cf3c1e67SAndrew Jeffery 164e310dd91SPatrick Williams if change_id in gerrit_query_result: 165cf3c1e67SAndrew Jeffery message_args = (recipe_basename, change_id) 166e310dd91SPatrick Williams print("{} {} already exists".format(*message_args)) 167cf3c1e67SAndrew Jeffery continue 168cf3c1e67SAndrew Jeffery 169cf3c1e67SAndrew Jeffery message_args = (recipe_basename, recipe_sha[:10], project_sha[:10]) 170e310dd91SPatrick Williams print("{} updating from {} to {}".format(*message_args)) 171cf3c1e67SAndrew Jeffery 172cf3c1e67SAndrew Jeffery remote_args = (args.ssh_config_host, project_name) 173e310dd91SPatrick Williams remote = "ssh://{}/openbmc/{}".format(*remote_args) 174cf3c1e67SAndrew Jeffery git_clone_or_reset(project_name, remote, args) 175cf3c1e67SAndrew Jeffery 176cf3c1e67SAndrew Jeffery try: 177e310dd91SPatrick Williams revlist = "{}..{}".format(recipe_sha, project_sha) 178cf3c1e67SAndrew Jeffery shortlog = git.shortlog(revlist, _cwd=project_name) 179e310dd91SPatrick Williams shortlog = shortlog.stdout.decode("utf-8") 180cf3c1e67SAndrew Jeffery except sh.ErrorReturnCode as e: 181e310dd91SPatrick Williams log("{}".format(e), args) 182cf3c1e67SAndrew Jeffery continue 183cf3c1e67SAndrew Jeffery 184e310dd91SPatrick Williams reset_args = ["--hard", "origin/{}".format(args.branch)] 185cf3c1e67SAndrew Jeffery git.reset(*reset_args, _cwd=meta) 186cf3c1e67SAndrew Jeffery 187cf3c1e67SAndrew Jeffery recipe_content = None 188cf3c1e67SAndrew Jeffery with open(full_recipe_path) as fd: 189cf3c1e67SAndrew Jeffery recipe_content = fd.read() 190cf3c1e67SAndrew Jeffery 191cf3c1e67SAndrew Jeffery recipe_content = recipe_content.replace(recipe_sha, project_sha) 192e310dd91SPatrick Williams with open(full_recipe_path, "w") as fd: 193cf3c1e67SAndrew Jeffery fd.write(recipe_content) 194cf3c1e67SAndrew Jeffery 195cf3c1e67SAndrew Jeffery git.add(recipe, _cwd=meta) 196cf3c1e67SAndrew Jeffery 197cf3c1e67SAndrew Jeffery commit_summary_args = (project_name, recipe_sha[:10], project_sha[:10]) 198e310dd91SPatrick Williams commit_msg = "{}: srcrev bump {}..{}".format(*commit_summary_args) 199e310dd91SPatrick Williams commit_msg += "\n\n{}".format(shortlog) 200e310dd91SPatrick Williams commit_msg += "\n\nChange-Id: {}".format(change_id) 201cf3c1e67SAndrew Jeffery 202e310dd91SPatrick Williams git.commit(sh.echo(commit_msg), "-s", "-F", "-", _cwd=meta) 203cf3c1e67SAndrew Jeffery 204199f7959SBrad Bishop push_args = [ 205e310dd91SPatrick Williams "origin", 206e310dd91SPatrick Williams "HEAD:refs/for/{}%topic=autobump".format(args.branch), 207199f7959SBrad Bishop ] 208cf3c1e67SAndrew Jeffery if not args.dry_run: 209cf3c1e67SAndrew Jeffery git.push(*push_args, _cwd=meta) 210cf3c1e67SAndrew Jeffery 211cf3c1e67SAndrew Jeffery 212cf3c1e67SAndrew Jefferydef main(): 213e310dd91SPatrick Williams app_description = """OpenBMC bitbake recipe bumping tool. 214cf3c1e67SAndrew Jeffery 215cf3c1e67SAndrew JefferyFind bitbake metadata files (recipes) that use the git fetcher 216cf3c1e67SAndrew Jefferyand check the project repository for newer revisions. 217cf3c1e67SAndrew Jeffery 218cf3c1e67SAndrew JefferyGenerate commits that update bitbake metadata files with SRCREV. 219cf3c1e67SAndrew Jeffery 220cf3c1e67SAndrew JefferyPush generated commits to the OpenBMC Gerrit instance for review. 221e310dd91SPatrick Williams """ 222cf3c1e67SAndrew Jeffery parser = argparse.ArgumentParser( 223cf3c1e67SAndrew Jeffery description=app_description, 224e310dd91SPatrick Williams formatter_class=argparse.RawDescriptionHelpFormatter, 225e310dd91SPatrick Williams ) 226cf3c1e67SAndrew Jeffery 227e310dd91SPatrick Williams parser.set_defaults(branch="master") 228cf3c1e67SAndrew Jeffery parser.add_argument( 229e310dd91SPatrick Williams "-d", 230e310dd91SPatrick Williams "--dry-run", 231e310dd91SPatrick Williams dest="dry_run", 232e310dd91SPatrick Williams action="store_true", 233e310dd91SPatrick Williams help="perform a dry run only", 234e310dd91SPatrick Williams ) 235cf3c1e67SAndrew Jeffery parser.add_argument( 236e310dd91SPatrick Williams "-m", 237e310dd91SPatrick Williams "--meta-repository", 238e310dd91SPatrick Williams dest="meta_repository", 239e310dd91SPatrick Williams action="append", 240e310dd91SPatrick Williams help="meta repository to check for updates", 241e310dd91SPatrick Williams ) 242cf3c1e67SAndrew Jeffery parser.add_argument( 243e310dd91SPatrick Williams "-v", 244e310dd91SPatrick Williams "--verbose", 245e310dd91SPatrick Williams dest="noisy", 246e310dd91SPatrick Williams action="store_true", 247e310dd91SPatrick Williams help="enable verbose status messages", 248e310dd91SPatrick Williams ) 249cf3c1e67SAndrew Jeffery parser.add_argument( 250e310dd91SPatrick Williams "ssh_config_host", 251e310dd91SPatrick Williams metavar="SSH_CONFIG_HOST_ENTRY", 252e310dd91SPatrick Williams help="SSH config host entry for Gerrit connectivity", 253e310dd91SPatrick Williams ) 254cf3c1e67SAndrew Jeffery 255cf3c1e67SAndrew Jeffery args = parser.parse_args() 256e310dd91SPatrick Williams setattr(args, "gerrit", sh.ssh.bake(args.ssh_config_host, "gerrit")) 257cf3c1e67SAndrew Jeffery 258e310dd91SPatrick Williams metas = getattr(args, "meta_repository") 259cf3c1e67SAndrew Jeffery if metas is None: 260e310dd91SPatrick Williams metas = args.gerrit("ls-projects", "-m", "meta-") 261e310dd91SPatrick Williams metas = metas.stdout.decode("utf-8").split() 262cf3c1e67SAndrew Jeffery metas = [os.path.split(x)[-1] for x in metas] 263cf3c1e67SAndrew Jeffery 264cf3c1e67SAndrew Jeffery for meta in metas: 265cf3c1e67SAndrew Jeffery find_and_process_bumps(meta, args) 266cf3c1e67SAndrew Jeffery 267cf3c1e67SAndrew Jeffery 268e310dd91SPatrick Williamsif __name__ == "__main__": 269cf3c1e67SAndrew Jeffery sys.exit(0 if main() else 1) 270