1# 2# Copyright BitBake Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import collections 8import fnmatch 9import logging 10import sys 11import os 12import re 13 14import bb.utils 15 16from bblayers.common import LayerPlugin 17 18logger = logging.getLogger('bitbake-layers') 19 20 21def plugin_init(plugins): 22 return QueryPlugin() 23 24 25class QueryPlugin(LayerPlugin): 26 def __init__(self): 27 super(QueryPlugin, self).__init__() 28 self.collection_res = {} 29 30 def do_show_layers(self, args): 31 """show current configured layers.""" 32 logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(70), "priority")) 33 logger.plain('=' * 104) 34 for layer, _, regex, pri in self.tinfoil.cooker.bbfile_config_priorities: 35 layerdir = self.bbfile_collections.get(layer, None) 36 layername = layer 37 logger.plain("%s %s %s" % (layername.ljust(20), layerdir.ljust(70), pri)) 38 39 def version_str(self, pe, pv, pr = None): 40 verstr = "%s" % pv 41 if pr: 42 verstr = "%s-%s" % (verstr, pr) 43 if pe: 44 verstr = "%s:%s" % (pe, verstr) 45 return verstr 46 47 def do_show_overlayed(self, args): 48 """list overlayed recipes (where the same recipe exists in another layer) 49 50Lists the names of overlayed recipes and the available versions in each 51layer, with the preferred version first. Note that skipped recipes that 52are overlayed will also be listed, with a " (skipped)" suffix. 53""" 54 55 items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, False, True, None, False, None, args.mc) 56 57 # Check for overlayed .bbclass files 58 classes = collections.defaultdict(list) 59 for layerdir in self.bblayers: 60 for c in ["classes-global", "classes-recipe", "classes"]: 61 classdir = os.path.join(layerdir, c) 62 if os.path.exists(classdir): 63 for classfile in os.listdir(classdir): 64 if os.path.splitext(classfile)[1] == '.bbclass': 65 classes[classfile].append(classdir) 66 67 # Locating classes and other files is a bit more complicated than recipes - 68 # layer priority is not a factor; instead BitBake uses the first matching 69 # file in BBPATH, which is manipulated directly by each layer's 70 # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a 71 # factor - however, each layer.conf is free to either prepend or append to 72 # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might 73 # not be exactly the order present in bblayers.conf either. 74 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 75 overlayed_class_found = False 76 for (classfile, classdirs) in classes.items(): 77 if len(classdirs) > 1: 78 if not overlayed_class_found: 79 logger.plain('=== Overlayed classes ===') 80 overlayed_class_found = True 81 82 mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile)) 83 if args.filenames: 84 logger.plain('%s' % mainfile) 85 else: 86 # We effectively have to guess the layer here 87 logger.plain('%s:' % classfile) 88 mainlayername = '?' 89 for layerdir in self.bblayers: 90 classdir = os.path.join(layerdir, 'classes') 91 if mainfile.startswith(classdir): 92 mainlayername = self.get_layer_name(layerdir) 93 logger.plain(' %s' % mainlayername) 94 for classdir in classdirs: 95 fullpath = os.path.join(classdir, classfile) 96 if fullpath != mainfile: 97 if args.filenames: 98 print(' %s' % fullpath) 99 else: 100 print(' %s' % self.get_layer_name(os.path.dirname(classdir))) 101 102 if overlayed_class_found: 103 items_listed = True; 104 105 if not items_listed: 106 logger.plain('No overlayed files found.') 107 108 def do_show_recipes(self, args): 109 """list available recipes, showing the layer they are provided by 110 111Lists the names of recipes and the available versions in each 112layer, with the preferred version first. Optionally you may specify 113pnspec to match a specified recipe name (supports wildcards). Note that 114skipped recipes will also be listed, with a " (skipped)" suffix. 115""" 116 117 inheritlist = args.inherits.split(',') if args.inherits else [] 118 if inheritlist or args.pnspec or args.multiple: 119 title = 'Matching recipes:' 120 else: 121 title = 'Available recipes:' 122 self.list_recipes(title, args.pnspec, False, False, args.filenames, args.recipes_only, args.multiple, args.layer, args.bare, inheritlist, args.mc) 123 124 def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_recipes_only, show_multi_provider_only, selected_layer, bare, inherits, mc): 125 if inherits: 126 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 127 for classname in inherits: 128 found = False 129 for c in ["classes-global", "classes-recipe", "classes"]: 130 cfile = c + '/%s.bbclass' % classname 131 if bb.utils.which(bbpath, cfile, history=False): 132 found = True 133 break 134 if not found: 135 logger.error('No class named %s found in BBPATH', classname) 136 sys.exit(1) 137 138 pkg_pn = self.tinfoil.cooker.recipecaches[mc].pkg_pn 139 (latest_versions, preferred_versions, required_versions) = self.tinfoil.find_providers(mc) 140 allproviders = self.tinfoil.get_all_providers(mc) 141 142 # Ensure we list skipped recipes 143 # We are largely guessing about PN, PV and the preferred version here, 144 # but we have no choice since skipped recipes are not fully parsed 145 skiplist = list(self.tinfoil.cooker.skiplist.keys()) 146 mcspec = 'mc:%s:' % mc 147 if mc: 148 skiplist = [s[len(mcspec):] for s in skiplist if s.startswith(mcspec)] 149 150 for fn in skiplist: 151 recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_') 152 p = recipe_parts[0] 153 if len(recipe_parts) > 1: 154 ver = (None, recipe_parts[1], None) 155 else: 156 ver = (None, 'unknown', None) 157 allproviders[p].append((ver, fn)) 158 if not p in pkg_pn: 159 pkg_pn[p] = 'dummy' 160 preferred_versions[p] = (ver, fn) 161 162 def print_item(f, pn, ver, layer, ispref): 163 if not selected_layer or layer == selected_layer: 164 if not bare and f in skiplist: 165 skipped = ' (skipped: %s)' % self.tinfoil.cooker.skiplist[f].skipreason 166 else: 167 skipped = '' 168 if show_filenames: 169 if ispref: 170 logger.plain("%s%s", f, skipped) 171 else: 172 logger.plain(" %s%s", f, skipped) 173 elif show_recipes_only: 174 if pn not in show_unique_pn: 175 show_unique_pn.append(pn) 176 logger.plain("%s%s", pn, skipped) 177 else: 178 if ispref: 179 logger.plain("%s:", pn) 180 logger.plain(" %s %s%s", layer.ljust(20), ver, skipped) 181 182 global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split() 183 cls_re = re.compile('classes.*/') 184 185 preffiles = [] 186 show_unique_pn = [] 187 items_listed = False 188 for p in sorted(pkg_pn): 189 if pnspec: 190 found=False 191 for pnm in pnspec: 192 if fnmatch.fnmatch(p, pnm): 193 found=True 194 break 195 if not found: 196 continue 197 198 if len(allproviders[p]) > 1 or not show_multi_provider_only: 199 pref = preferred_versions[p] 200 realfn = bb.cache.virtualfn2realfn(pref[1]) 201 preffile = realfn[0] 202 203 # We only display once per recipe, we should prefer non extended versions of the 204 # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl 205 # which would otherwise sort first). 206 if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecaches[mc].pkg_fn: 207 continue 208 209 if inherits: 210 matchcount = 0 211 recipe_inherits = self.tinfoil.cooker_data.inherits.get(preffile, []) 212 for cls in recipe_inherits: 213 if cls_re.match(cls): 214 continue 215 classname = os.path.splitext(os.path.basename(cls))[0] 216 if classname in global_inherit: 217 continue 218 elif classname in inherits: 219 matchcount += 1 220 if matchcount != len(inherits): 221 # No match - skip this recipe 222 continue 223 224 if preffile not in preffiles: 225 preflayer = self.get_file_layer(preffile) 226 multilayer = False 227 same_ver = True 228 provs = [] 229 for prov in allproviders[p]: 230 provfile = bb.cache.virtualfn2realfn(prov[1])[0] 231 provlayer = self.get_file_layer(provfile) 232 provs.append((provfile, provlayer, prov[0])) 233 if provlayer != preflayer: 234 multilayer = True 235 if prov[0] != pref[0]: 236 same_ver = False 237 if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only): 238 if not items_listed: 239 logger.plain('=== %s ===' % title) 240 items_listed = True 241 print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True) 242 for (provfile, provlayer, provver) in provs: 243 if provfile != preffile: 244 print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False) 245 # Ensure we don't show two entries for BBCLASSEXTENDed recipes 246 preffiles.append(preffile) 247 248 return items_listed 249 250 def get_file_layer(self, filename): 251 layerdir = self.get_file_layerdir(filename) 252 if layerdir: 253 return self.get_layer_name(layerdir) 254 else: 255 return '?' 256 257 def get_collection_res(self): 258 if not self.collection_res: 259 self.collection_res = bb.utils.get_collection_res(self.tinfoil.config_data) 260 return self.collection_res 261 262 def get_file_layerdir(self, filename): 263 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data, self.get_collection_res()) 264 return self.bbfile_collections.get(layer, None) 265 266 def remove_layer_prefix(self, f): 267 """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the 268 return value will be: layer_dir/foo/blah""" 269 f_layerdir = self.get_file_layerdir(f) 270 if not f_layerdir: 271 return f 272 prefix = os.path.join(os.path.dirname(f_layerdir), '') 273 return f[len(prefix):] if f.startswith(prefix) else f 274 275 def do_show_appends(self, args): 276 """list bbappend files and recipe files they apply to 277 278Lists recipes with the bbappends that apply to them as subitems. 279""" 280 if args.pnspec: 281 logger.plain('=== Matched appended recipes ===') 282 else: 283 logger.plain('=== Appended recipes ===') 284 285 pnlist = list(self.tinfoil.cooker_data.pkg_pn.keys()) 286 pnlist.sort() 287 appends = False 288 for pn in pnlist: 289 if args.pnspec: 290 found=False 291 for pnm in args.pnspec: 292 if fnmatch.fnmatch(pn, pnm): 293 found=True 294 break 295 if not found: 296 continue 297 298 if self.show_appends_for_pn(pn): 299 appends = True 300 301 if not args.pnspec and self.show_appends_for_skipped(): 302 appends = True 303 304 if not appends: 305 logger.plain('No append files found') 306 307 def show_appends_for_pn(self, pn): 308 filenames = self.tinfoil.cooker_data.pkg_pn[pn] 309 310 best = self.tinfoil.find_best_provider(pn) 311 best_filename = os.path.basename(best[3]) 312 313 return self.show_appends_output(filenames, best_filename) 314 315 def show_appends_for_skipped(self): 316 filenames = [os.path.basename(f) 317 for f in self.tinfoil.cooker.skiplist.keys()] 318 return self.show_appends_output(filenames, None, " (skipped)") 319 320 def show_appends_output(self, filenames, best_filename, name_suffix = ''): 321 appended, missing = self.get_appends_for_files(filenames) 322 if appended: 323 for basename, appends in appended: 324 logger.plain('%s%s:', basename, name_suffix) 325 for append in appends: 326 logger.plain(' %s', append) 327 328 if best_filename: 329 if best_filename in missing: 330 logger.warning('%s: missing append for preferred version', 331 best_filename) 332 return True 333 else: 334 return False 335 336 def get_appends_for_files(self, filenames): 337 appended, notappended = [], [] 338 for filename in filenames: 339 _, cls, mc = bb.cache.virtualfn2realfn(filename) 340 if cls: 341 continue 342 343 basename = os.path.basename(filename) 344 appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) 345 if appends: 346 appended.append((basename, list(appends))) 347 else: 348 notappended.append(basename) 349 return appended, notappended 350 351 def do_show_cross_depends(self, args): 352 """Show dependencies between recipes that cross layer boundaries. 353 354Figure out the dependencies between recipes that cross layer boundaries. 355 356NOTE: .bbappend files can impact the dependencies. 357""" 358 ignore_layers = (args.ignore or '').split(',') 359 360 pkg_fn = self.tinfoil.cooker_data.pkg_fn 361 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 362 self.require_re = re.compile(r"require\s+(.+)") 363 self.include_re = re.compile(r"include\s+(.+)") 364 self.inherit_re = re.compile(r"inherit\s+(.+)") 365 366 global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split() 367 368 # The bb's DEPENDS and RDEPENDS 369 for f in pkg_fn: 370 f = bb.cache.virtualfn2realfn(f)[0] 371 # Get the layername that the file is in 372 layername = self.get_file_layer(f) 373 374 # The DEPENDS 375 deps = self.tinfoil.cooker_data.deps[f] 376 for pn in deps: 377 if pn in self.tinfoil.cooker_data.pkg_pn: 378 best = self.tinfoil.find_best_provider(pn) 379 self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers) 380 381 # The RDPENDS 382 all_rdeps = self.tinfoil.cooker_data.rundeps[f].values() 383 # Remove the duplicated or null one. 384 sorted_rdeps = {} 385 # The all_rdeps is the list in list, so we need two for loops 386 for k1 in all_rdeps: 387 for k2 in k1: 388 sorted_rdeps[k2] = 1 389 all_rdeps = sorted_rdeps.keys() 390 for rdep in all_rdeps: 391 all_p, best = self.tinfoil.get_runtime_providers(rdep) 392 if all_p: 393 if f in all_p: 394 # The recipe provides this one itself, ignore 395 continue 396 self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers) 397 398 # The RRECOMMENDS 399 all_rrecs = self.tinfoil.cooker_data.runrecs[f].values() 400 # Remove the duplicated or null one. 401 sorted_rrecs = {} 402 # The all_rrecs is the list in list, so we need two for loops 403 for k1 in all_rrecs: 404 for k2 in k1: 405 sorted_rrecs[k2] = 1 406 all_rrecs = sorted_rrecs.keys() 407 for rrec in all_rrecs: 408 all_p, best = self.tinfoil.get_runtime_providers(rrec) 409 if all_p: 410 if f in all_p: 411 # The recipe provides this one itself, ignore 412 continue 413 self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers) 414 415 # The inherit class 416 cls_re = re.compile('classes.*/') 417 if f in self.tinfoil.cooker_data.inherits: 418 inherits = self.tinfoil.cooker_data.inherits[f] 419 for cls in inherits: 420 # The inherits' format is [classes/cls, /path/to/classes/cls] 421 # ignore the classes/cls. 422 if not cls_re.match(cls): 423 classname = os.path.splitext(os.path.basename(cls))[0] 424 if classname in global_inherit: 425 continue 426 inherit_layername = self.get_file_layer(cls) 427 if inherit_layername != layername and not inherit_layername in ignore_layers: 428 if not args.filenames: 429 f_short = self.remove_layer_prefix(f) 430 cls = self.remove_layer_prefix(cls) 431 else: 432 f_short = f 433 logger.plain("%s inherits %s" % (f_short, cls)) 434 435 # The 'require/include xxx' in the bb file 436 pv_re = re.compile(r"\${PV}") 437 with open(f, 'r') as fnfile: 438 line = fnfile.readline() 439 while line: 440 m, keyword = self.match_require_include(line) 441 # Found the 'require/include xxxx' 442 if m: 443 needed_file = m.group(1) 444 # Replace the ${PV} with the real PV 445 if pv_re.search(needed_file) and f in self.tinfoil.cooker_data.pkg_pepvpr: 446 pv = self.tinfoil.cooker_data.pkg_pepvpr[f][1] 447 needed_file = re.sub(r"\${PV}", pv, needed_file) 448 self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers) 449 line = fnfile.readline() 450 451 # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass 452 conf_re = re.compile(r".*/conf/machine/[^\/]*\.conf$") 453 inc_re = re.compile(r".*\.inc$") 454 # The "inherit xxx" in .bbclass 455 bbclass_re = re.compile(r".*\.bbclass$") 456 for layerdir in self.bblayers: 457 layername = self.get_layer_name(layerdir) 458 for dirpath, dirnames, filenames in os.walk(layerdir): 459 for name in filenames: 460 f = os.path.join(dirpath, name) 461 s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f) 462 if s: 463 with open(f, 'r') as ffile: 464 line = ffile.readline() 465 while line: 466 m, keyword = self.match_require_include(line) 467 # Only bbclass has the "inherit xxx" here. 468 bbclass="" 469 if not m and f.endswith(".bbclass"): 470 m, keyword = self.match_inherit(line) 471 bbclass=".bbclass" 472 # Find a 'require/include xxxx' 473 if m: 474 self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers) 475 line = ffile.readline() 476 477 def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers): 478 """Print the depends that crosses a layer boundary""" 479 needed_file = bb.utils.which(bbpath, needed_filename) 480 if needed_file: 481 # Which layer is this file from 482 needed_layername = self.get_file_layer(needed_file) 483 if needed_layername != layername and not needed_layername in ignore_layers: 484 if not show_filenames: 485 f = self.remove_layer_prefix(f) 486 needed_file = self.remove_layer_prefix(needed_file) 487 logger.plain("%s %s %s" %(f, keyword, needed_file)) 488 489 def match_inherit(self, line): 490 """Match the inherit xxx line""" 491 return (self.inherit_re.match(line), "inherits") 492 493 def match_require_include(self, line): 494 """Match the require/include xxx line""" 495 m = self.require_re.match(line) 496 keyword = "requires" 497 if not m: 498 m = self.include_re.match(line) 499 keyword = "includes" 500 return (m, keyword) 501 502 def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers): 503 """Print the DEPENDS/RDEPENDS file that crosses a layer boundary""" 504 best_realfn = bb.cache.virtualfn2realfn(needed_file)[0] 505 needed_layername = self.get_file_layer(best_realfn) 506 if needed_layername != layername and not needed_layername in ignore_layers: 507 if not show_filenames: 508 f = self.remove_layer_prefix(f) 509 best_realfn = self.remove_layer_prefix(best_realfn) 510 511 logger.plain("%s %s %s" % (f, keyword, best_realfn)) 512 513 def register_commands(self, sp): 514 self.add_command(sp, 'show-layers', self.do_show_layers, parserecipes=False) 515 516 parser_show_overlayed = self.add_command(sp, 'show-overlayed', self.do_show_overlayed) 517 parser_show_overlayed.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') 518 parser_show_overlayed.add_argument('-s', '--same-version', help='only list overlayed recipes where the version is the same', action='store_true') 519 parser_show_overlayed.add_argument('--mc', help='use specified multiconfig', default='') 520 521 parser_show_recipes = self.add_command(sp, 'show-recipes', self.do_show_recipes) 522 parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') 523 parser_show_recipes.add_argument('-r', '--recipes-only', help='instead of the default formatting, list recipes only', action='store_true') 524 parser_show_recipes.add_argument('-m', '--multiple', help='only list where multiple recipes (in the same layer or different layers) exist for the same recipe name', action='store_true') 525 parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class(es) - separate multiple classes using , (without spaces)', metavar='CLASS', default='') 526 parser_show_recipes.add_argument('-l', '--layer', help='only list recipes from the selected layer', default='') 527 parser_show_recipes.add_argument('-b', '--bare', help='output just names without the "(skipped)" marker', action='store_true') 528 parser_show_recipes.add_argument('--mc', help='use specified multiconfig', default='') 529 parser_show_recipes.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') 530 531 parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends) 532 parser_show_appends.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') 533 534 parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends) 535 parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true') 536 parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME') 537