1# Development tool - upgrade command plugin 2# 3# Copyright (C) 2014-2017 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool upgrade plugin""" 8 9import os 10import sys 11import re 12import shutil 13import tempfile 14import logging 15import argparse 16import scriptutils 17import errno 18import bb 19 20devtool_path = os.path.dirname(os.path.realpath(__file__)) + '/../../../meta/lib' 21sys.path = sys.path + [devtool_path] 22 23import oe.recipeutils 24from devtool import standard 25from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build, update_unlockedsigs, check_prerelease_version 26 27logger = logging.getLogger('devtool') 28 29def _run(cmd, cwd=''): 30 logger.debug("Running command %s> %s" % (cwd,cmd)) 31 return bb.process.run('%s' % cmd, cwd=cwd) 32 33def _get_srctree(tmpdir): 34 srctree = tmpdir 35 dirs = os.listdir(tmpdir) 36 if len(dirs) == 1: 37 srctree = os.path.join(tmpdir, dirs[0]) 38 else: 39 raise DevtoolError("Cannot determine where the source tree is after unpacking in {}: {}".format(tmpdir,dirs)) 40 return srctree 41 42def _copy_source_code(orig, dest): 43 for path in standard._ls_tree(orig): 44 dest_dir = os.path.join(dest, os.path.dirname(path)) 45 bb.utils.mkdirhier(dest_dir) 46 dest_path = os.path.join(dest, path) 47 shutil.move(os.path.join(orig, path), dest_path) 48 49def _remove_patch_dirs(recipefolder): 50 for root, dirs, files in os.walk(recipefolder): 51 for d in dirs: 52 shutil.rmtree(os.path.join(root,d)) 53 54def _recipe_contains(rd, var): 55 rf = rd.getVar('FILE') 56 varfiles = oe.recipeutils.get_var_files(rf, [var], rd) 57 for var, fn in varfiles.items(): 58 if fn and fn.startswith(os.path.dirname(rf) + os.sep): 59 return True 60 return False 61 62def _rename_recipe_dirs(oldpv, newpv, path): 63 for root, dirs, files in os.walk(path): 64 # Rename directories with the version in their name 65 for olddir in dirs: 66 if olddir.find(oldpv) != -1: 67 newdir = olddir.replace(oldpv, newpv) 68 if olddir != newdir: 69 shutil.move(os.path.join(path, olddir), os.path.join(path, newdir)) 70 # Rename any inc files with the version in their name (unusual, but possible) 71 for oldfile in files: 72 if oldfile.endswith('.inc'): 73 if oldfile.find(oldpv) != -1: 74 newfile = oldfile.replace(oldpv, newpv) 75 if oldfile != newfile: 76 bb.utils.rename(os.path.join(path, oldfile), 77 os.path.join(path, newfile)) 78 79def _rename_recipe_file(oldrecipe, pn, oldpv, newpv, path): 80 oldrecipe = os.path.basename(oldrecipe) 81 if oldrecipe.endswith('_%s.bb' % oldpv): 82 newrecipe = '%s_%s.bb' % (pn, newpv) 83 if oldrecipe != newrecipe: 84 shutil.move(os.path.join(path, oldrecipe), os.path.join(path, newrecipe)) 85 else: 86 newrecipe = oldrecipe 87 return os.path.join(path, newrecipe) 88 89def _rename_recipe_files(oldrecipe, pn, oldpv, newpv, path): 90 _rename_recipe_dirs(oldpv, newpv, path) 91 return _rename_recipe_file(oldrecipe, pn, oldpv, newpv, path) 92 93def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, revs, copied, workspace, d): 94 """Writes an append file""" 95 if not os.path.exists(rc): 96 raise DevtoolError("bbappend not created because %s does not exist" % rc) 97 98 appendpath = os.path.join(workspace, 'appends') 99 if not os.path.exists(appendpath): 100 bb.utils.mkdirhier(appendpath) 101 102 brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename 103 104 srctree = os.path.abspath(srctree) 105 pn = d.getVar('PN') 106 af = os.path.join(appendpath, '%s.bbappend' % brf) 107 with open(af, 'w') as f: 108 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n\n') 109 # Local files can be modified/tracked in separate subdir under srctree 110 # Mostly useful for packages with S != WORKDIR 111 f.write('FILESPATH:prepend := "%s:"\n' % 112 os.path.join(srctreebase, 'oe-local-files')) 113 f.write('# srctreebase: %s\n' % srctreebase) 114 f.write('inherit externalsrc\n') 115 f.write(('# NOTE: We use pn- overrides here to avoid affecting' 116 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n')) 117 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) 118 b_is_s = use_external_build(same_dir, no_same_dir, d) 119 if b_is_s: 120 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) 121 f.write('\n') 122 if revs: 123 for name, rev in revs.items(): 124 f.write('# initial_rev %s: %s\n' % (name, rev)) 125 if copied: 126 f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE'))) 127 f.write('# original_files: %s\n' % ' '.join(copied)) 128 return af 129 130def _cleanup_on_error(rd, srctree): 131 if os.path.exists(rd): 132 shutil.rmtree(rd) 133 srctree = os.path.abspath(srctree) 134 if os.path.exists(srctree): 135 shutil.rmtree(srctree) 136 137def _upgrade_error(e, rd, srctree, keep_failure=False, extramsg=None): 138 if not keep_failure: 139 _cleanup_on_error(rd, srctree) 140 logger.error(e) 141 if extramsg: 142 logger.error(extramsg) 143 if keep_failure: 144 logger.info('Preserving failed upgrade files (--keep-failure)') 145 sys.exit(1) 146 147def _get_uri(rd): 148 srcuris = rd.getVar('SRC_URI').split() 149 if not len(srcuris): 150 raise DevtoolError('SRC_URI not found on recipe') 151 # Get first non-local entry in SRC_URI - usually by convention it's 152 # the first entry, but not always! 153 srcuri = None 154 for entry in srcuris: 155 if not entry.startswith('file://'): 156 srcuri = entry 157 break 158 if not srcuri: 159 raise DevtoolError('Unable to find non-local entry in SRC_URI') 160 srcrev = '${AUTOREV}' 161 if '://' in srcuri: 162 # Fetch a URL 163 rev_re = re.compile(';rev=([^;]+)') 164 res = rev_re.search(srcuri) 165 if res: 166 srcrev = res.group(1) 167 srcuri = rev_re.sub('', srcuri) 168 return srcuri, srcrev 169 170def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, keep_temp, tinfoil, rd): 171 """Extract sources of a recipe with a new version""" 172 import oe.patch 173 174 def __run(cmd): 175 """Simple wrapper which calls _run with srctree as cwd""" 176 return _run(cmd, srctree) 177 178 crd = rd.createCopy() 179 180 pv = crd.getVar('PV') 181 crd.setVar('PV', newpv) 182 183 tmpsrctree = None 184 uri, rev = _get_uri(crd) 185 if srcrev: 186 rev = srcrev 187 paths = [srctree] 188 if uri.startswith('git://') or uri.startswith('gitsm://'): 189 __run('git fetch') 190 __run('git checkout %s' % rev) 191 __run('git tag -f devtool-base-new') 192 __run('git submodule update --recursive') 193 __run('git submodule foreach \'git tag -f devtool-base-new\'') 194 (stdout, _) = __run('git submodule --quiet foreach \'echo $sm_path\'') 195 paths += [os.path.join(srctree, p) for p in stdout.splitlines()] 196 checksums = {} 197 _, _, _, _, _, params = bb.fetch2.decodeurl(uri) 198 srcsubdir_rel = params.get('destsuffix', 'git') 199 if not srcbranch: 200 check_branch, check_branch_err = __run('git branch -r --contains %s' % srcrev) 201 get_branch = [x.strip() for x in check_branch.splitlines()] 202 # Remove HEAD reference point and drop remote prefix 203 get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] 204 if len(get_branch) == 1: 205 # If srcrev is on only ONE branch, then use that branch 206 srcbranch = get_branch[0] 207 elif 'main' in get_branch: 208 # If srcrev is on multiple branches, then choose 'main' if it is one of them 209 srcbranch = 'main' 210 elif 'master' in get_branch: 211 # Otherwise choose 'master' if it is one of the branches 212 srcbranch = 'master' 213 else: 214 # If get_branch contains more than one objects, then display error and exit. 215 mbrch = '\n ' + '\n '.join(get_branch) 216 raise DevtoolError('Revision %s was found on multiple branches: %s\nPlease provide the correct branch in the devtool command with "--srcbranch" or "-B" option.' % (srcrev, mbrch)) 217 else: 218 __run('git checkout devtool-base -b devtool-%s' % newpv) 219 220 tmpdir = tempfile.mkdtemp(prefix='devtool') 221 try: 222 checksums, ftmpdir = scriptutils.fetch_url(tinfoil, uri, rev, tmpdir, logger, preserve_tmp=keep_temp) 223 except scriptutils.FetchUrlFailure as e: 224 raise DevtoolError(e) 225 226 if ftmpdir and keep_temp: 227 logger.info('Fetch temp directory is %s' % ftmpdir) 228 229 tmpsrctree = _get_srctree(tmpdir) 230 srctree = os.path.abspath(srctree) 231 srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir) 232 233 # Delete all sources so we ensure no stray files are left over 234 for item in os.listdir(srctree): 235 if item in ['.git', 'oe-local-files']: 236 continue 237 itempath = os.path.join(srctree, item) 238 if os.path.isdir(itempath): 239 shutil.rmtree(itempath) 240 else: 241 os.remove(itempath) 242 243 # Copy in new ones 244 _copy_source_code(tmpsrctree, srctree) 245 246 (stdout,_) = __run('git ls-files --modified --others') 247 filelist = stdout.splitlines() 248 pbar = bb.ui.knotty.BBProgress('Adding changed files', len(filelist)) 249 pbar.start() 250 batchsize = 100 251 for i in range(0, len(filelist), batchsize): 252 batch = filelist[i:i+batchsize] 253 __run('git add -f -A %s' % ' '.join(['"%s"' % item for item in batch])) 254 pbar.update(i) 255 pbar.finish() 256 257 useroptions = [] 258 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) 259 __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv)) 260 __run('git tag -f devtool-base-%s' % newpv) 261 262 revs = {} 263 for path in paths: 264 (stdout, _) = _run('git rev-parse HEAD', cwd=path) 265 revs[os.path.relpath(path, srctree)] = stdout.rstrip() 266 267 if no_patch: 268 patches = oe.recipeutils.get_recipe_patches(crd) 269 if patches: 270 logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n %s' % '\n '.join([os.path.basename(patch) for patch in patches])) 271 else: 272 for path in paths: 273 _run('git checkout devtool-patched -b %s' % branch, cwd=path) 274 (stdout, _) = _run('git branch --list devtool-override-*', cwd=path) 275 branches_to_rebase = [branch] + stdout.split() 276 target_branch = revs[os.path.relpath(path, srctree)] 277 278 # There is a bug (or feature?) in git rebase where if a commit with 279 # a note is fully rebased away by being part of an old commit, the 280 # note is still attached to the old commit. Avoid this by making 281 # sure all old devtool related commits have a note attached to them 282 # (this assumes git config notes.rewriteMode is set to ignore). 283 (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch) 284 for rev in stdout.splitlines(): 285 if not oe.patch.GitApplyTree.getNotes(path, rev): 286 oe.patch.GitApplyTree.addNote(path, rev, "dummy") 287 288 for b in branches_to_rebase: 289 logger.info("Rebasing {} onto {}".format(b, target_branch)) 290 _run('git checkout %s' % b, cwd=path) 291 try: 292 _run('git rebase %s' % target_branch, cwd=path) 293 except bb.process.ExecutionError as e: 294 if 'conflict' in e.stdout: 295 logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) 296 _run('git rebase --abort', cwd=path) 297 else: 298 logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) 299 300 # Remove any dummy notes added above. 301 (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch) 302 for rev in stdout.splitlines(): 303 oe.patch.GitApplyTree.removeNote(path, rev, "dummy") 304 305 _run('git checkout %s' % branch, cwd=path) 306 307 if tmpsrctree: 308 if keep_temp: 309 logger.info('Preserving temporary directory %s' % tmpsrctree) 310 else: 311 shutil.rmtree(tmpsrctree) 312 if tmpdir != tmpsrctree: 313 shutil.rmtree(tmpdir) 314 315 return (revs, checksums, srcbranch, srcsubdir_rel) 316 317def _add_license_diff_to_recipe(path, diff): 318 notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'. 319# The following is the difference between the old and the new license text. 320# Please update the LICENSE value if needed, and summarize the changes in 321# the commit message via 'License-Update:' tag. 322# (example: 'License-Update: copyright years updated.') 323# 324# The changes: 325# 326""" 327 commented_diff = "\n".join(["# {}".format(l) for l in diff.split('\n')]) 328 with open(path, 'rb') as f: 329 orig_content = f.read() 330 with open(path, 'wb') as f: 331 f.write(notice_text.encode()) 332 f.write(commented_diff.encode()) 333 f.write("\n#\n\n".encode()) 334 f.write(orig_content) 335 336def _create_new_recipe(newpv, checksums, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): 337 """Creates the new recipe under workspace""" 338 339 pn = rd.getVar('PN') 340 path = os.path.join(workspace, 'recipes', pn) 341 bb.utils.mkdirhier(path) 342 copied, _ = oe.recipeutils.copy_recipe_files(rd, path, all_variants=True) 343 if not copied: 344 raise DevtoolError('Internal error - no files were copied for recipe %s' % pn) 345 logger.debug('Copied %s to %s' % (copied, path)) 346 347 oldpv = rd.getVar('PV') 348 if not newpv: 349 newpv = oldpv 350 origpath = rd.getVar('FILE') 351 fullpath = _rename_recipe_files(origpath, pn, oldpv, newpv, path) 352 logger.debug('Upgraded %s => %s' % (origpath, fullpath)) 353 354 newvalues = {} 355 if _recipe_contains(rd, 'PV') and newpv != oldpv: 356 newvalues['PV'] = newpv 357 358 if srcrev: 359 newvalues['SRCREV'] = srcrev 360 361 if srcbranch: 362 src_uri = oe.recipeutils.split_var_value(rd.getVar('SRC_URI', False) or '') 363 changed = False 364 replacing = True 365 new_src_uri = [] 366 for entry in src_uri: 367 try: 368 scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry) 369 except bb.fetch2.MalformedUrl as e: 370 raise DevtoolError("Could not decode SRC_URI: {}".format(e)) 371 if replacing and scheme in ['git', 'gitsm']: 372 branch = params.get('branch', 'master') 373 if rd.expand(branch) != srcbranch: 374 # Handle case where branch is set through a variable 375 res = re.match(r'\$\{([^}@]+)\}', branch) 376 if res: 377 newvalues[res.group(1)] = srcbranch 378 # We know we won't change SRC_URI now, so break out 379 break 380 else: 381 params['branch'] = srcbranch 382 entry = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params)) 383 changed = True 384 replacing = False 385 new_src_uri.append(entry) 386 if changed: 387 newvalues['SRC_URI'] = ' '.join(new_src_uri) 388 389 newvalues['PR'] = None 390 391 # Work out which SRC_URI entries have changed in case the entry uses a name 392 crd = rd.createCopy() 393 crd.setVar('PV', newpv) 394 for var, value in newvalues.items(): 395 crd.setVar(var, value) 396 old_src_uri = (rd.getVar('SRC_URI') or '').split() 397 new_src_uri = (crd.getVar('SRC_URI') or '').split() 398 newnames = [] 399 addnames = [] 400 for newentry in new_src_uri: 401 _, _, _, _, _, params = bb.fetch2.decodeurl(newentry) 402 if 'name' in params: 403 newnames.append(params['name']) 404 if newentry not in old_src_uri: 405 addnames.append(params['name']) 406 # Find what's been set in the original recipe 407 oldnames = [] 408 oldsums = [] 409 noname = False 410 for varflag in rd.getVarFlags('SRC_URI'): 411 for checksum in checksums: 412 if varflag.endswith('.' + checksum): 413 name = varflag.rsplit('.', 1)[0] 414 if name not in oldnames: 415 oldnames.append(name) 416 oldsums.append(checksum) 417 elif varflag == checksum: 418 noname = True 419 oldsums.append(checksum) 420 # Even if SRC_URI has named entries it doesn't have to actually use the name 421 if noname and addnames and addnames[0] not in oldnames: 422 addnames = [] 423 # Drop any old names (the name actually might include ${PV}) 424 for name in oldnames: 425 if name not in newnames: 426 for checksum in oldsums: 427 newvalues['SRC_URI[%s.%s]' % (name, checksum)] = None 428 429 nameprefix = '%s.' % addnames[0] if addnames else '' 430 431 # md5sum is deprecated, remove any traces of it. If it was the only old 432 # checksum, then replace it with the default checksums. 433 if 'md5sum' in oldsums: 434 newvalues['SRC_URI[%smd5sum]' % nameprefix] = None 435 oldsums.remove('md5sum') 436 if not oldsums: 437 oldsums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST] 438 439 for checksum in oldsums: 440 newvalues['SRC_URI[%s%s]' % (nameprefix, checksum)] = checksums[checksum] 441 442 if srcsubdir_new != srcsubdir_old: 443 s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR')) 444 s_subdir_new = os.path.relpath(os.path.abspath(crd.getVar('S')), crd.getVar('WORKDIR')) 445 if srcsubdir_old == s_subdir_old and srcsubdir_new != s_subdir_new: 446 # Subdir for old extracted source matches what S points to (it should!) 447 # but subdir for new extracted source doesn't match what S will be 448 newvalues['S'] = '${WORKDIR}/%s' % srcsubdir_new.replace(newpv, '${PV}') 449 if crd.expand(newvalues['S']) == crd.expand('${WORKDIR}/${BP}'): 450 # It's the default, drop it 451 # FIXME what if S is being set in a .inc? 452 newvalues['S'] = None 453 logger.info('Source subdirectory has changed, dropping S value since it now matches the default ("${WORKDIR}/${BP}")') 454 else: 455 logger.info('Source subdirectory has changed, updating S value') 456 457 if license_diff: 458 newlicchksum = " ".join(["file://{}".format(l['path']) + 459 (";beginline={}".format(l['beginline']) if l['beginline'] else "") + 460 (";endline={}".format(l['endline']) if l['endline'] else "") + 461 (";md5={}".format(l['actual_md5'])) for l in new_licenses]) 462 newvalues["LIC_FILES_CHKSUM"] = newlicchksum 463 _add_license_diff_to_recipe(fullpath, license_diff) 464 465 tinfoil.modified_files() 466 try: 467 rd = tinfoil.parse_recipe_file(fullpath, False) 468 except bb.tinfoil.TinfoilCommandFailed as e: 469 _upgrade_error(e, os.path.dirname(fullpath), srctree, keep_failure, 'Parsing of upgraded recipe failed') 470 oe.recipeutils.patch_recipe(rd, fullpath, newvalues) 471 472 return fullpath, copied 473 474 475def _check_git_config(): 476 def getconfig(name): 477 try: 478 value = bb.process.run('git config %s' % name)[0].strip() 479 except bb.process.ExecutionError as e: 480 if e.exitcode == 1: 481 value = None 482 else: 483 raise 484 return value 485 486 username = getconfig('user.name') 487 useremail = getconfig('user.email') 488 configerr = [] 489 if not username: 490 configerr.append('Please set your name using:\n git config --global user.name') 491 if not useremail: 492 configerr.append('Please set your email using:\n git config --global user.email') 493 if configerr: 494 raise DevtoolError('Your git configuration is incomplete which will prevent rebases from working:\n' + '\n'.join(configerr)) 495 496def _extract_licenses(srcpath, recipe_licenses): 497 licenses = [] 498 for url in recipe_licenses.split(): 499 license = {} 500 (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) 501 license['path'] = path 502 license['md5'] = parm.get('md5', '') 503 license['beginline'], license['endline'] = 0, 0 504 if 'beginline' in parm: 505 license['beginline'] = int(parm['beginline']) 506 if 'endline' in parm: 507 license['endline'] = int(parm['endline']) 508 license['text'] = [] 509 with open(os.path.join(srcpath, path), 'rb') as f: 510 import hashlib 511 actual_md5 = hashlib.md5() 512 lineno = 0 513 for line in f: 514 lineno += 1 515 if (lineno >= license['beginline']) and ((lineno <= license['endline']) or not license['endline']): 516 license['text'].append(line.decode(errors='ignore')) 517 actual_md5.update(line) 518 license['actual_md5'] = actual_md5.hexdigest() 519 licenses.append(license) 520 return licenses 521 522def _generate_license_diff(old_licenses, new_licenses): 523 need_diff = False 524 for l in new_licenses: 525 if l['md5'] != l['actual_md5']: 526 need_diff = True 527 break 528 if need_diff == False: 529 return None 530 531 import difflib 532 diff = '' 533 for old, new in zip(old_licenses, new_licenses): 534 for line in difflib.unified_diff(old['text'], new['text'], old['path'], new['path']): 535 diff = diff + line 536 return diff 537 538def _run_recipe_upgrade_extra_tasks(pn, rd, tinfoil): 539 tasks = [] 540 for task in (rd.getVar('RECIPE_UPGRADE_EXTRA_TASKS') or '').split(): 541 logger.info('Running extra recipe upgrade task: %s' % task) 542 res = tinfoil.build_targets(pn, task, handle_events=True) 543 544 if not res: 545 raise DevtoolError('Running extra recipe upgrade task %s for %s failed' % (task, pn)) 546 547def upgrade(args, config, basepath, workspace): 548 """Entry point for the devtool 'upgrade' subcommand""" 549 550 if args.recipename in workspace: 551 raise DevtoolError("recipe %s is already in your workspace" % args.recipename) 552 if args.srcbranch and not args.srcrev: 553 raise DevtoolError("If you specify --srcbranch/-B then you must use --srcrev/-S to specify the revision" % args.recipename) 554 555 _check_git_config() 556 557 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 558 try: 559 rd = parse_recipe(config, tinfoil, args.recipename, True) 560 if not rd: 561 return 1 562 563 pn = rd.getVar('PN') 564 if pn != args.recipename: 565 logger.info('Mapping %s to %s' % (args.recipename, pn)) 566 if pn in workspace: 567 raise DevtoolError("recipe %s is already in your workspace" % pn) 568 569 if args.srctree: 570 srctree = os.path.abspath(args.srctree) 571 else: 572 srctree = standard.get_default_srctree(config, pn) 573 574 srctree_s = standard.get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) 575 576 # try to automatically discover latest version and revision if not provided on command line 577 if not args.version and not args.srcrev: 578 version_info = oe.recipeutils.get_recipe_upstream_version(rd) 579 if version_info['version'] and not version_info['version'].endswith("new-commits-available"): 580 args.version = version_info['version'] 581 if version_info['revision']: 582 args.srcrev = version_info['revision'] 583 if not args.version and not args.srcrev: 584 raise DevtoolError("Automatic discovery of latest version/revision failed - you must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option.") 585 586 standard._check_compatible_recipe(pn, rd) 587 old_srcrev = rd.getVar('SRCREV') 588 if old_srcrev == 'INVALID': 589 old_srcrev = None 590 if old_srcrev and not args.srcrev: 591 raise DevtoolError("Recipe specifies a SRCREV value; you must specify a new one when upgrading") 592 old_ver = rd.getVar('PV') 593 if old_ver == args.version and old_srcrev == args.srcrev: 594 raise DevtoolError("Current and upgrade versions are the same version") 595 if args.version: 596 if bb.utils.vercmp_string(args.version, old_ver) < 0: 597 logger.warning('Upgrade version %s compares as less than the current version %s. If you are using a package feed for on-target upgrades or providing this recipe for general consumption, then you should increment PE in the recipe (or if there is no current PE value set, set it to "1")' % (args.version, old_ver)) 598 check_prerelease_version(args.version, 'devtool upgrade') 599 600 rf = None 601 license_diff = None 602 try: 603 logger.info('Extracting current version source...') 604 rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 605 old_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or "")) 606 logger.info('Extracting upgraded version source...') 607 rev2, checksums, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch, 608 args.srcrev, args.srcbranch, args.branch, args.keep_temp, 609 tinfoil, rd) 610 new_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or "")) 611 license_diff = _generate_license_diff(old_licenses, new_licenses) 612 rf, copied = _create_new_recipe(args.version, checksums, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) 613 except (bb.process.CmdError, DevtoolError) as e: 614 recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('PN')) 615 _upgrade_error(e, recipedir, srctree, args.keep_failure) 616 standard._add_md5(config, pn, os.path.dirname(rf)) 617 618 af = _write_append(rf, srctree, srctree_s, args.same_dir, args.no_same_dir, rev2, 619 copied, config.workspace_path, rd) 620 standard._add_md5(config, pn, af) 621 622 _run_recipe_upgrade_extra_tasks(pn, rd, tinfoil) 623 624 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 625 626 logger.info('Upgraded source extracted to %s' % srctree) 627 logger.info('New recipe is %s' % rf) 628 if license_diff: 629 logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.') 630 preferred_version = rd.getVar('PREFERRED_VERSION_%s' % rd.getVar('PN')) 631 if preferred_version: 632 logger.warning('Version is pinned to %s via PREFERRED_VERSION; it may need adjustment to match the new version before any further steps are taken' % preferred_version) 633 finally: 634 tinfoil.shutdown() 635 return 0 636 637def latest_version(args, config, basepath, workspace): 638 """Entry point for the devtool 'latest_version' subcommand""" 639 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 640 try: 641 rd = parse_recipe(config, tinfoil, args.recipename, True) 642 if not rd: 643 return 1 644 version_info = oe.recipeutils.get_recipe_upstream_version(rd) 645 # "new-commits-available" is an indication that upstream never issues version tags 646 if not version_info['version'].endswith("new-commits-available"): 647 logger.info("Current version: {}".format(version_info['current_version'])) 648 logger.info("Latest version: {}".format(version_info['version'])) 649 if version_info['revision']: 650 logger.info("Latest version's commit: {}".format(version_info['revision'])) 651 else: 652 logger.info("Latest commit: {}".format(version_info['revision'])) 653 finally: 654 tinfoil.shutdown() 655 return 0 656 657def check_upgrade_status(args, config, basepath, workspace): 658 def _print_status(recipe): 659 print("{:25} {:15} {:15} {} {} {}".format( recipe['pn'], 660 recipe['cur_ver'], 661 recipe['status'] if recipe['status'] != 'UPDATE' else (recipe['next_ver'] if not recipe['next_ver'].endswith("new-commits-available") else "new commits"), 662 recipe['maintainer'], 663 recipe['revision'] if recipe['revision'] != 'N/A' else "", 664 "cannot be updated due to: %s" %(recipe['no_upgrade_reason']) if recipe['no_upgrade_reason'] else "")) 665 if not args.recipe: 666 logger.info("Checking the upstream status for all recipes may take a few minutes") 667 results = oe.recipeutils.get_recipe_upgrade_status(args.recipe) 668 for recipegroup in results: 669 upgrades = [r for r in recipegroup if r['status'] != 'MATCH'] 670 currents = [r for r in recipegroup if r['status'] == 'MATCH'] 671 if len(upgrades) > 1: 672 print("These recipes need to be upgraded together {") 673 for r in upgrades: 674 _print_status(r) 675 if len(upgrades) > 1: 676 print("}") 677 for r in currents: 678 if args.all: 679 _print_status(r) 680 681def register_commands(subparsers, context): 682 """Register devtool subcommands from this plugin""" 683 684 defsrctree = standard.get_default_srctree(context.config) 685 686 parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe', 687 description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).', 688 group='starting') 689 parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)') 690 parser_upgrade.add_argument('srctree', nargs='?', help='Path to where to extract the source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 691 parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV). If omitted, latest upstream version will be determined and used, if possible.') 692 parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (useful when fetching from an SCM such as git)') 693 parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)') 694 parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")') 695 parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code') 696 parser_upgrade.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 697 group = parser_upgrade.add_mutually_exclusive_group() 698 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 699 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 700 parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 701 parser_upgrade.add_argument('--keep-failure', action="store_true", help='Keep failed upgrade recipe and associated files (for debugging)') 702 parser_upgrade.set_defaults(func=upgrade, fixed_setup=context.fixed_setup) 703 704 parser_latest_version = subparsers.add_parser('latest-version', help='Report the latest version of an existing recipe', 705 description='Queries the upstream server for what the latest upstream release is (for git, tags are checked, for tarballs, a list of them is obtained, and one with the highest version number is reported)', 706 group='info') 707 parser_latest_version.add_argument('recipename', help='Name of recipe to query (just name - no version, path or extension)') 708 parser_latest_version.set_defaults(func=latest_version) 709 710 parser_check_upgrade_status = subparsers.add_parser('check-upgrade-status', help="Report upgradability for multiple (or all) recipes", 711 description="Prints a table of recipes together with versions currently provided by recipes, and latest upstream versions, when there is a later version available", 712 group='info') 713 parser_check_upgrade_status.add_argument('recipe', help='Name of the recipe to report (omit to report upgrade info for all recipes)', nargs='*') 714 parser_check_upgrade_status.add_argument('--all', '-a', help='Show all recipes, not just recipes needing upgrade', action="store_true") 715 parser_check_upgrade_status.set_defaults(func=check_upgrade_status) 716