1# 2# Copyright BitBake Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import collections 8import fnmatch 9import logging 10import sys 11import os 12import re 13 14import bb.utils 15 16from bblayers.common import LayerPlugin 17 18logger = logging.getLogger('bitbake-layers') 19 20 21def plugin_init(plugins): 22 return QueryPlugin() 23 24 25class QueryPlugin(LayerPlugin): 26 def __init__(self): 27 super(QueryPlugin, self).__init__() 28 self.collection_res = {} 29 30 def do_show_layers(self, args): 31 """show current configured layers.""" 32 logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(70), "priority")) 33 logger.plain('=' * 104) 34 for layer, _, regex, pri in self.tinfoil.cooker.bbfile_config_priorities: 35 layerdir = self.bbfile_collections.get(layer, None) 36 layername = layer 37 logger.plain("%s %s %s" % (layername.ljust(20), layerdir.ljust(70), pri)) 38 39 def version_str(self, pe, pv, pr = None): 40 verstr = "%s" % pv 41 if pr: 42 verstr = "%s-%s" % (verstr, pr) 43 if pe: 44 verstr = "%s:%s" % (pe, verstr) 45 return verstr 46 47 def do_show_overlayed(self, args): 48 """list overlayed recipes (where the same recipe exists in another layer) 49 50Lists the names of overlayed recipes and the available versions in each 51layer, with the preferred version first. Note that skipped recipes that 52are overlayed will also be listed, with a " (skipped)" suffix. 53""" 54 55 items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, False, True, None, False, None, args.mc) 56 57 # Check for overlayed .bbclass files 58 classes = collections.defaultdict(list) 59 for layerdir in self.bblayers: 60 for c in ["classes-global", "classes-recipe", "classes"]: 61 classdir = os.path.join(layerdir, c) 62 if os.path.exists(classdir): 63 for classfile in os.listdir(classdir): 64 if os.path.splitext(classfile)[1] == '.bbclass': 65 classes[classfile].append(classdir) 66 67 # Locating classes and other files is a bit more complicated than recipes - 68 # layer priority is not a factor; instead BitBake uses the first matching 69 # file in BBPATH, which is manipulated directly by each layer's 70 # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a 71 # factor - however, each layer.conf is free to either prepend or append to 72 # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might 73 # not be exactly the order present in bblayers.conf either. 74 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 75 overlayed_class_found = False 76 for (classfile, classdirs) in classes.items(): 77 if len(classdirs) > 1: 78 if not overlayed_class_found: 79 logger.plain('=== Overlayed classes ===') 80 overlayed_class_found = True 81 82 mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile)) 83 if args.filenames: 84 logger.plain('%s' % mainfile) 85 else: 86 # We effectively have to guess the layer here 87 logger.plain('%s:' % classfile) 88 mainlayername = '?' 89 for layerdir in self.bblayers: 90 classdir = os.path.join(layerdir, 'classes') 91 if mainfile.startswith(classdir): 92 mainlayername = self.get_layer_name(layerdir) 93 logger.plain(' %s' % mainlayername) 94 for classdir in classdirs: 95 fullpath = os.path.join(classdir, classfile) 96 if fullpath != mainfile: 97 if args.filenames: 98 print(' %s' % fullpath) 99 else: 100 print(' %s' % self.get_layer_name(os.path.dirname(classdir))) 101 102 if overlayed_class_found: 103 items_listed = True; 104 105 if not items_listed: 106 logger.plain('No overlayed files found.') 107 108 def do_show_recipes(self, args): 109 """list available recipes, showing the layer they are provided by 110 111Lists the names of recipes and the available versions in each 112layer, with the preferred version first. Optionally you may specify 113pnspec to match a specified recipe name (supports wildcards). Note that 114skipped recipes will also be listed, with a " (skipped)" suffix. 115""" 116 117 inheritlist = args.inherits.split(',') if args.inherits else [] 118 if inheritlist or args.pnspec or args.multiple: 119 title = 'Matching recipes:' 120 else: 121 title = 'Available recipes:' 122 self.list_recipes(title, args.pnspec, False, False, args.filenames, args.recipes_only, args.multiple, args.layer, args.bare, inheritlist, args.mc) 123 124 def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_recipes_only, show_multi_provider_only, selected_layer, bare, inherits, mc): 125 if inherits: 126 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 127 for classname in inherits: 128 found = False 129 for c in ["classes-global", "classes-recipe", "classes"]: 130 cfile = c + '/%s.bbclass' % classname 131 if bb.utils.which(bbpath, cfile, history=False): 132 found = True 133 break 134 if not found: 135 logger.error('No class named %s found in BBPATH', classname) 136 sys.exit(1) 137 138 pkg_pn = self.tinfoil.cooker.recipecaches[mc].pkg_pn 139 (latest_versions, preferred_versions, required_versions) = self.tinfoil.find_providers(mc) 140 allproviders = self.tinfoil.get_all_providers(mc) 141 142 # Ensure we list skipped recipes 143 # We are largely guessing about PN, PV and the preferred version here, 144 # but we have no choice since skipped recipes are not fully parsed 145 skiplist = list(self.tinfoil.cooker.skiplist.keys()) 146 mcspec = 'mc:%s:' % mc 147 if mc: 148 skiplist = [s[len(mcspec):] for s in skiplist if s.startswith(mcspec)] 149 150 for fn in skiplist: 151 recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_') 152 p = recipe_parts[0] 153 if len(recipe_parts) > 1: 154 ver = (None, recipe_parts[1], None) 155 else: 156 ver = (None, 'unknown', None) 157 allproviders[p].append((ver, fn)) 158 if not p in pkg_pn: 159 pkg_pn[p] = 'dummy' 160 preferred_versions[p] = (ver, fn) 161 162 def print_item(f, pn, ver, layer, ispref): 163 if not selected_layer or layer == selected_layer: 164 if not bare and f in skiplist: 165 skipped = ' (skipped: %s)' % self.tinfoil.cooker.skiplist[f].skipreason 166 else: 167 skipped = '' 168 if show_filenames: 169 if ispref: 170 logger.plain("%s%s", f, skipped) 171 else: 172 logger.plain(" %s%s", f, skipped) 173 elif show_recipes_only: 174 if pn not in show_unique_pn: 175 show_unique_pn.append(pn) 176 logger.plain("%s%s", pn, skipped) 177 else: 178 if ispref: 179 logger.plain("%s:", pn) 180 logger.plain(" %s %s%s", layer.ljust(20), ver, skipped) 181 182 global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split() 183 cls_re = re.compile('classes.*/') 184 185 preffiles = [] 186 show_unique_pn = [] 187 items_listed = False 188 for p in sorted(pkg_pn): 189 if pnspec: 190 found=False 191 for pnm in pnspec: 192 if fnmatch.fnmatch(p, pnm): 193 found=True 194 break 195 if not found: 196 continue 197 198 if len(allproviders[p]) > 1 or not show_multi_provider_only: 199 pref = preferred_versions[p] 200 realfn = bb.cache.virtualfn2realfn(pref[1]) 201 preffile = realfn[0] 202 203 # We only display once per recipe, we should prefer non extended versions of the 204 # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl 205 # which would otherwise sort first). 206 if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecaches[mc].pkg_fn: 207 continue 208 209 if inherits: 210 matchcount = 0 211 recipe_inherits = self.tinfoil.cooker_data.inherits.get(preffile, []) 212 for cls in recipe_inherits: 213 if cls_re.match(cls): 214 continue 215 classname = os.path.splitext(os.path.basename(cls))[0] 216 if classname in global_inherit: 217 continue 218 elif classname in inherits: 219 matchcount += 1 220 if matchcount != len(inherits): 221 # No match - skip this recipe 222 continue 223 224 if preffile not in preffiles: 225 preflayer = self.get_file_layer(preffile) 226 multilayer = False 227 same_ver = True 228 provs = [] 229 for prov in allproviders[p]: 230 provfile = bb.cache.virtualfn2realfn(prov[1])[0] 231 provlayer = self.get_file_layer(provfile) 232 provs.append((provfile, provlayer, prov[0])) 233 if provlayer != preflayer: 234 multilayer = True 235 if prov[0] != pref[0]: 236 same_ver = False 237 if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only): 238 if not items_listed: 239 logger.plain('=== %s ===' % title) 240 items_listed = True 241 print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True) 242 for (provfile, provlayer, provver) in provs: 243 if provfile != preffile: 244 print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False) 245 # Ensure we don't show two entries for BBCLASSEXTENDed recipes 246 preffiles.append(preffile) 247 248 return items_listed 249 250 def get_file_layer(self, filename): 251 layerdir = self.get_file_layerdir(filename) 252 if layerdir: 253 return self.get_layer_name(layerdir) 254 else: 255 return '?' 256 257 def get_collection_res(self): 258 if not self.collection_res: 259 self.collection_res = bb.utils.get_collection_res(self.tinfoil.config_data) 260 return self.collection_res 261 262 def get_file_layerdir(self, filename): 263 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data, self.get_collection_res()) 264 return self.bbfile_collections.get(layer, None) 265 266 def remove_layer_prefix(self, f): 267 """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the 268 return value will be: layer_dir/foo/blah""" 269 f_layerdir = self.get_file_layerdir(f) 270 if not f_layerdir: 271 return f 272 prefix = os.path.join(os.path.dirname(f_layerdir), '') 273 return f[len(prefix):] if f.startswith(prefix) else f 274 275 def do_show_appends(self, args): 276 """list bbappend files and recipe files they apply to 277 278Lists recipes with the bbappends that apply to them as subitems. 279""" 280 if args.pnspec: 281 logger.plain('=== Matched appended recipes ===') 282 else: 283 logger.plain('=== Appended recipes ===') 284 285 286 cooker_data = self.tinfoil.cooker.recipecaches[args.mc] 287 288 pnlist = list(cooker_data.pkg_pn.keys()) 289 pnlist.sort() 290 appends = False 291 for pn in pnlist: 292 if args.pnspec: 293 found=False 294 for pnm in args.pnspec: 295 if fnmatch.fnmatch(pn, pnm): 296 found=True 297 break 298 if not found: 299 continue 300 301 if self.show_appends_for_pn(pn, cooker_data, args.mc): 302 appends = True 303 304 if not args.pnspec and self.show_appends_for_skipped(): 305 appends = True 306 307 if not appends: 308 logger.plain('No append files found') 309 310 def show_appends_for_pn(self, pn, cooker_data, mc): 311 filenames = cooker_data.pkg_pn[pn] 312 if mc: 313 pn = "mc:%s:%s" % (mc, pn) 314 315 best = self.tinfoil.find_best_provider(pn) 316 best_filename = os.path.basename(best[3]) 317 318 return self.show_appends_output(filenames, best_filename) 319 320 def show_appends_for_skipped(self): 321 filenames = [os.path.basename(f) 322 for f in self.tinfoil.cooker.skiplist.keys()] 323 return self.show_appends_output(filenames, None, " (skipped)") 324 325 def show_appends_output(self, filenames, best_filename, name_suffix = ''): 326 appended, missing = self.get_appends_for_files(filenames) 327 if appended: 328 for basename, appends in appended: 329 logger.plain('%s%s:', basename, name_suffix) 330 for append in appends: 331 logger.plain(' %s', append) 332 333 if best_filename: 334 if best_filename in missing: 335 logger.warning('%s: missing append for preferred version', 336 best_filename) 337 return True 338 else: 339 return False 340 341 def get_appends_for_files(self, filenames): 342 appended, notappended = [], [] 343 for filename in filenames: 344 _, cls, mc = bb.cache.virtualfn2realfn(filename) 345 if cls: 346 continue 347 348 basename = os.path.basename(filename) 349 appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) 350 if appends: 351 appended.append((basename, list(appends))) 352 else: 353 notappended.append(basename) 354 return appended, notappended 355 356 def do_show_cross_depends(self, args): 357 """Show dependencies between recipes that cross layer boundaries. 358 359Figure out the dependencies between recipes that cross layer boundaries. 360 361NOTE: .bbappend files can impact the dependencies. 362""" 363 ignore_layers = (args.ignore or '').split(',') 364 365 pkg_fn = self.tinfoil.cooker_data.pkg_fn 366 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 367 self.require_re = re.compile(r"require\s+(.+)") 368 self.include_re = re.compile(r"include\s+(.+)") 369 self.inherit_re = re.compile(r"inherit\s+(.+)") 370 371 global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split() 372 373 # The bb's DEPENDS and RDEPENDS 374 for f in pkg_fn: 375 f = bb.cache.virtualfn2realfn(f)[0] 376 # Get the layername that the file is in 377 layername = self.get_file_layer(f) 378 379 # The DEPENDS 380 deps = self.tinfoil.cooker_data.deps[f] 381 for pn in deps: 382 if pn in self.tinfoil.cooker_data.pkg_pn: 383 best = self.tinfoil.find_best_provider(pn) 384 self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers) 385 386 # The RDPENDS 387 all_rdeps = self.tinfoil.cooker_data.rundeps[f].values() 388 # Remove the duplicated or null one. 389 sorted_rdeps = {} 390 # The all_rdeps is the list in list, so we need two for loops 391 for k1 in all_rdeps: 392 for k2 in k1: 393 sorted_rdeps[k2] = 1 394 all_rdeps = sorted_rdeps.keys() 395 for rdep in all_rdeps: 396 all_p, best = self.tinfoil.get_runtime_providers(rdep) 397 if all_p: 398 if f in all_p: 399 # The recipe provides this one itself, ignore 400 continue 401 self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers) 402 403 # The RRECOMMENDS 404 all_rrecs = self.tinfoil.cooker_data.runrecs[f].values() 405 # Remove the duplicated or null one. 406 sorted_rrecs = {} 407 # The all_rrecs is the list in list, so we need two for loops 408 for k1 in all_rrecs: 409 for k2 in k1: 410 sorted_rrecs[k2] = 1 411 all_rrecs = sorted_rrecs.keys() 412 for rrec in all_rrecs: 413 all_p, best = self.tinfoil.get_runtime_providers(rrec) 414 if all_p: 415 if f in all_p: 416 # The recipe provides this one itself, ignore 417 continue 418 self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers) 419 420 # The inherit class 421 cls_re = re.compile('classes.*/') 422 if f in self.tinfoil.cooker_data.inherits: 423 inherits = self.tinfoil.cooker_data.inherits[f] 424 for cls in inherits: 425 # The inherits' format is [classes/cls, /path/to/classes/cls] 426 # ignore the classes/cls. 427 if not cls_re.match(cls): 428 classname = os.path.splitext(os.path.basename(cls))[0] 429 if classname in global_inherit: 430 continue 431 inherit_layername = self.get_file_layer(cls) 432 if inherit_layername != layername and not inherit_layername in ignore_layers: 433 if not args.filenames: 434 f_short = self.remove_layer_prefix(f) 435 cls = self.remove_layer_prefix(cls) 436 else: 437 f_short = f 438 logger.plain("%s inherits %s" % (f_short, cls)) 439 440 # The 'require/include xxx' in the bb file 441 pv_re = re.compile(r"\${PV}") 442 with open(f, 'r') as fnfile: 443 line = fnfile.readline() 444 while line: 445 m, keyword = self.match_require_include(line) 446 # Found the 'require/include xxxx' 447 if m: 448 needed_file = m.group(1) 449 # Replace the ${PV} with the real PV 450 if pv_re.search(needed_file) and f in self.tinfoil.cooker_data.pkg_pepvpr: 451 pv = self.tinfoil.cooker_data.pkg_pepvpr[f][1] 452 needed_file = re.sub(r"\${PV}", pv, needed_file) 453 self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers) 454 line = fnfile.readline() 455 456 # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass 457 conf_re = re.compile(r".*/conf/machine/[^\/]*\.conf$") 458 inc_re = re.compile(r".*\.inc$") 459 # The "inherit xxx" in .bbclass 460 bbclass_re = re.compile(r".*\.bbclass$") 461 for layerdir in self.bblayers: 462 layername = self.get_layer_name(layerdir) 463 for dirpath, dirnames, filenames in os.walk(layerdir): 464 for name in filenames: 465 f = os.path.join(dirpath, name) 466 s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f) 467 if s: 468 with open(f, 'r') as ffile: 469 line = ffile.readline() 470 while line: 471 m, keyword = self.match_require_include(line) 472 # Only bbclass has the "inherit xxx" here. 473 bbclass="" 474 if not m and f.endswith(".bbclass"): 475 m, keyword = self.match_inherit(line) 476 bbclass=".bbclass" 477 # Find a 'require/include xxxx' 478 if m: 479 self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers) 480 line = ffile.readline() 481 482 def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers): 483 """Print the depends that crosses a layer boundary""" 484 needed_file = bb.utils.which(bbpath, needed_filename) 485 if needed_file: 486 # Which layer is this file from 487 needed_layername = self.get_file_layer(needed_file) 488 if needed_layername != layername and not needed_layername in ignore_layers: 489 if not show_filenames: 490 f = self.remove_layer_prefix(f) 491 needed_file = self.remove_layer_prefix(needed_file) 492 logger.plain("%s %s %s" %(f, keyword, needed_file)) 493 494 def match_inherit(self, line): 495 """Match the inherit xxx line""" 496 return (self.inherit_re.match(line), "inherits") 497 498 def match_require_include(self, line): 499 """Match the require/include xxx line""" 500 m = self.require_re.match(line) 501 keyword = "requires" 502 if not m: 503 m = self.include_re.match(line) 504 keyword = "includes" 505 return (m, keyword) 506 507 def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers): 508 """Print the DEPENDS/RDEPENDS file that crosses a layer boundary""" 509 best_realfn = bb.cache.virtualfn2realfn(needed_file)[0] 510 needed_layername = self.get_file_layer(best_realfn) 511 if needed_layername != layername and not needed_layername in ignore_layers: 512 if not show_filenames: 513 f = self.remove_layer_prefix(f) 514 best_realfn = self.remove_layer_prefix(best_realfn) 515 516 logger.plain("%s %s %s" % (f, keyword, best_realfn)) 517 518 def register_commands(self, sp): 519 self.add_command(sp, 'show-layers', self.do_show_layers, parserecipes=False) 520 521 parser_show_overlayed = self.add_command(sp, 'show-overlayed', self.do_show_overlayed) 522 parser_show_overlayed.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') 523 parser_show_overlayed.add_argument('-s', '--same-version', help='only list overlayed recipes where the version is the same', action='store_true') 524 parser_show_overlayed.add_argument('--mc', help='use specified multiconfig', default='') 525 526 parser_show_recipes = self.add_command(sp, 'show-recipes', self.do_show_recipes) 527 parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') 528 parser_show_recipes.add_argument('-r', '--recipes-only', help='instead of the default formatting, list recipes only', action='store_true') 529 parser_show_recipes.add_argument('-m', '--multiple', help='only list where multiple recipes (in the same layer or different layers) exist for the same recipe name', action='store_true') 530 parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class(es) - separate multiple classes using , (without spaces)', metavar='CLASS', default='') 531 parser_show_recipes.add_argument('-l', '--layer', help='only list recipes from the selected layer', default='') 532 parser_show_recipes.add_argument('-b', '--bare', help='output just names without the "(skipped)" marker', action='store_true') 533 parser_show_recipes.add_argument('--mc', help='use specified multiconfig', default='') 534 parser_show_recipes.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') 535 536 parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends) 537 parser_show_appends.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') 538 parser_show_appends.add_argument('--mc', help='use specified multiconfig', default='') 539 540 parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends) 541 parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true') 542 parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME') 543