1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4 5import collections 6import fnmatch 7import logging 8import sys 9import os 10import re 11 12import bb.utils 13 14from bblayers.common import LayerPlugin 15 16logger = logging.getLogger('bitbake-layers') 17 18 19def plugin_init(plugins): 20 return QueryPlugin() 21 22 23class QueryPlugin(LayerPlugin): 24 def __init__(self): 25 super(QueryPlugin, self).__init__() 26 self.collection_res = {} 27 28 def do_show_layers(self, args): 29 """show current configured layers.""" 30 logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(40), "priority")) 31 logger.plain('=' * 74) 32 for layer, _, regex, pri in self.tinfoil.cooker.bbfile_config_priorities: 33 layerdir = self.bbfile_collections.get(layer, None) 34 layername = self.get_layer_name(layerdir) 35 logger.plain("%s %s %d" % (layername.ljust(20), layerdir.ljust(40), pri)) 36 37 def version_str(self, pe, pv, pr = None): 38 verstr = "%s" % pv 39 if pr: 40 verstr = "%s-%s" % (verstr, pr) 41 if pe: 42 verstr = "%s:%s" % (pe, verstr) 43 return verstr 44 45 def do_show_overlayed(self, args): 46 """list overlayed recipes (where the same recipe exists in another layer) 47 48Lists the names of overlayed recipes and the available versions in each 49layer, with the preferred version first. Note that skipped recipes that 50are overlayed will also be listed, with a " (skipped)" suffix. 51""" 52 53 items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, False, True, None, False, None, args.mc) 54 55 # Check for overlayed .bbclass files 56 classes = collections.defaultdict(list) 57 for layerdir in self.bblayers: 58 classdir = os.path.join(layerdir, 'classes') 59 if os.path.exists(classdir): 60 for classfile in os.listdir(classdir): 61 if os.path.splitext(classfile)[1] == '.bbclass': 62 classes[classfile].append(classdir) 63 64 # Locating classes and other files is a bit more complicated than recipes - 65 # layer priority is not a factor; instead BitBake uses the first matching 66 # file in BBPATH, which is manipulated directly by each layer's 67 # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a 68 # factor - however, each layer.conf is free to either prepend or append to 69 # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might 70 # not be exactly the order present in bblayers.conf either. 71 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 72 overlayed_class_found = False 73 for (classfile, classdirs) in classes.items(): 74 if len(classdirs) > 1: 75 if not overlayed_class_found: 76 logger.plain('=== Overlayed classes ===') 77 overlayed_class_found = True 78 79 mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile)) 80 if args.filenames: 81 logger.plain('%s' % mainfile) 82 else: 83 # We effectively have to guess the layer here 84 logger.plain('%s:' % classfile) 85 mainlayername = '?' 86 for layerdir in self.bblayers: 87 classdir = os.path.join(layerdir, 'classes') 88 if mainfile.startswith(classdir): 89 mainlayername = self.get_layer_name(layerdir) 90 logger.plain(' %s' % mainlayername) 91 for classdir in classdirs: 92 fullpath = os.path.join(classdir, classfile) 93 if fullpath != mainfile: 94 if args.filenames: 95 print(' %s' % fullpath) 96 else: 97 print(' %s' % self.get_layer_name(os.path.dirname(classdir))) 98 99 if overlayed_class_found: 100 items_listed = True; 101 102 if not items_listed: 103 logger.plain('No overlayed files found.') 104 105 def do_show_recipes(self, args): 106 """list available recipes, showing the layer they are provided by 107 108Lists the names of recipes and the available versions in each 109layer, with the preferred version first. Optionally you may specify 110pnspec to match a specified recipe name (supports wildcards). Note that 111skipped recipes will also be listed, with a " (skipped)" suffix. 112""" 113 114 inheritlist = args.inherits.split(',') if args.inherits else [] 115 if inheritlist or args.pnspec or args.multiple: 116 title = 'Matching recipes:' 117 else: 118 title = 'Available recipes:' 119 self.list_recipes(title, args.pnspec, False, False, args.filenames, args.recipes_only, args.multiple, args.layer, args.bare, inheritlist, args.mc) 120 121 def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_recipes_only, show_multi_provider_only, selected_layer, bare, inherits, mc): 122 if inherits: 123 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 124 for classname in inherits: 125 classfile = 'classes/%s.bbclass' % classname 126 if not bb.utils.which(bbpath, classfile, history=False): 127 logger.error('No class named %s found in BBPATH', classfile) 128 sys.exit(1) 129 130 pkg_pn = self.tinfoil.cooker.recipecaches[mc].pkg_pn 131 (latest_versions, preferred_versions, required_versions) = self.tinfoil.find_providers(mc) 132 allproviders = self.tinfoil.get_all_providers(mc) 133 134 # Ensure we list skipped recipes 135 # We are largely guessing about PN, PV and the preferred version here, 136 # but we have no choice since skipped recipes are not fully parsed 137 skiplist = list(self.tinfoil.cooker.skiplist.keys()) 138 mcspec = 'mc:%s:' % mc 139 if mc: 140 skiplist = [s[len(mcspec):] for s in skiplist if s.startswith(mcspec)] 141 142 for fn in skiplist: 143 recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_') 144 p = recipe_parts[0] 145 if len(recipe_parts) > 1: 146 ver = (None, recipe_parts[1], None) 147 else: 148 ver = (None, 'unknown', None) 149 allproviders[p].append((ver, fn)) 150 if not p in pkg_pn: 151 pkg_pn[p] = 'dummy' 152 preferred_versions[p] = (ver, fn) 153 154 def print_item(f, pn, ver, layer, ispref): 155 if not selected_layer or layer == selected_layer: 156 if not bare and f in skiplist: 157 skipped = ' (skipped: %s)' % self.tinfoil.cooker.skiplist[f].skipreason 158 else: 159 skipped = '' 160 if show_filenames: 161 if ispref: 162 logger.plain("%s%s", f, skipped) 163 else: 164 logger.plain(" %s%s", f, skipped) 165 elif show_recipes_only: 166 if pn not in show_unique_pn: 167 show_unique_pn.append(pn) 168 logger.plain("%s%s", pn, skipped) 169 else: 170 if ispref: 171 logger.plain("%s:", pn) 172 logger.plain(" %s %s%s", layer.ljust(20), ver, skipped) 173 174 global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split() 175 cls_re = re.compile('classes/') 176 177 preffiles = [] 178 show_unique_pn = [] 179 items_listed = False 180 for p in sorted(pkg_pn): 181 if pnspec: 182 found=False 183 for pnm in pnspec: 184 if fnmatch.fnmatch(p, pnm): 185 found=True 186 break 187 if not found: 188 continue 189 190 if len(allproviders[p]) > 1 or not show_multi_provider_only: 191 pref = preferred_versions[p] 192 realfn = bb.cache.virtualfn2realfn(pref[1]) 193 preffile = realfn[0] 194 195 # We only display once per recipe, we should prefer non extended versions of the 196 # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl 197 # which would otherwise sort first). 198 if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecaches[mc].pkg_fn: 199 continue 200 201 if inherits: 202 matchcount = 0 203 recipe_inherits = self.tinfoil.cooker_data.inherits.get(preffile, []) 204 for cls in recipe_inherits: 205 if cls_re.match(cls): 206 continue 207 classname = os.path.splitext(os.path.basename(cls))[0] 208 if classname in global_inherit: 209 continue 210 elif classname in inherits: 211 matchcount += 1 212 if matchcount != len(inherits): 213 # No match - skip this recipe 214 continue 215 216 if preffile not in preffiles: 217 preflayer = self.get_file_layer(preffile) 218 multilayer = False 219 same_ver = True 220 provs = [] 221 for prov in allproviders[p]: 222 provfile = bb.cache.virtualfn2realfn(prov[1])[0] 223 provlayer = self.get_file_layer(provfile) 224 provs.append((provfile, provlayer, prov[0])) 225 if provlayer != preflayer: 226 multilayer = True 227 if prov[0] != pref[0]: 228 same_ver = False 229 if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only): 230 if not items_listed: 231 logger.plain('=== %s ===' % title) 232 items_listed = True 233 print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True) 234 for (provfile, provlayer, provver) in provs: 235 if provfile != preffile: 236 print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False) 237 # Ensure we don't show two entries for BBCLASSEXTENDed recipes 238 preffiles.append(preffile) 239 240 return items_listed 241 242 def get_file_layer(self, filename): 243 layerdir = self.get_file_layerdir(filename) 244 if layerdir: 245 return self.get_layer_name(layerdir) 246 else: 247 return '?' 248 249 def get_collection_res(self): 250 if not self.collection_res: 251 self.collection_res = bb.utils.get_collection_res(self.tinfoil.config_data) 252 return self.collection_res 253 254 def get_file_layerdir(self, filename): 255 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data, self.get_collection_res()) 256 return self.bbfile_collections.get(layer, None) 257 258 def remove_layer_prefix(self, f): 259 """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the 260 return value will be: layer_dir/foo/blah""" 261 f_layerdir = self.get_file_layerdir(f) 262 if not f_layerdir: 263 return f 264 prefix = os.path.join(os.path.dirname(f_layerdir), '') 265 return f[len(prefix):] if f.startswith(prefix) else f 266 267 def do_show_appends(self, args): 268 """list bbappend files and recipe files they apply to 269 270Lists recipes with the bbappends that apply to them as subitems. 271""" 272 if args.pnspec: 273 logger.plain('=== Matched appended recipes ===') 274 else: 275 logger.plain('=== Appended recipes ===') 276 277 pnlist = list(self.tinfoil.cooker_data.pkg_pn.keys()) 278 pnlist.sort() 279 appends = False 280 for pn in pnlist: 281 if args.pnspec: 282 found=False 283 for pnm in args.pnspec: 284 if fnmatch.fnmatch(pn, pnm): 285 found=True 286 break 287 if not found: 288 continue 289 290 if self.show_appends_for_pn(pn): 291 appends = True 292 293 if not args.pnspec and self.show_appends_for_skipped(): 294 appends = True 295 296 if not appends: 297 logger.plain('No append files found') 298 299 def show_appends_for_pn(self, pn): 300 filenames = self.tinfoil.cooker_data.pkg_pn[pn] 301 302 best = self.tinfoil.find_best_provider(pn) 303 best_filename = os.path.basename(best[3]) 304 305 return self.show_appends_output(filenames, best_filename) 306 307 def show_appends_for_skipped(self): 308 filenames = [os.path.basename(f) 309 for f in self.tinfoil.cooker.skiplist.keys()] 310 return self.show_appends_output(filenames, None, " (skipped)") 311 312 def show_appends_output(self, filenames, best_filename, name_suffix = ''): 313 appended, missing = self.get_appends_for_files(filenames) 314 if appended: 315 for basename, appends in appended: 316 logger.plain('%s%s:', basename, name_suffix) 317 for append in appends: 318 logger.plain(' %s', append) 319 320 if best_filename: 321 if best_filename in missing: 322 logger.warning('%s: missing append for preferred version', 323 best_filename) 324 return True 325 else: 326 return False 327 328 def get_appends_for_files(self, filenames): 329 appended, notappended = [], [] 330 for filename in filenames: 331 _, cls, mc = bb.cache.virtualfn2realfn(filename) 332 if cls: 333 continue 334 335 basename = os.path.basename(filename) 336 appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) 337 if appends: 338 appended.append((basename, list(appends))) 339 else: 340 notappended.append(basename) 341 return appended, notappended 342 343 def do_show_cross_depends(self, args): 344 """Show dependencies between recipes that cross layer boundaries. 345 346Figure out the dependencies between recipes that cross layer boundaries. 347 348NOTE: .bbappend files can impact the dependencies. 349""" 350 ignore_layers = (args.ignore or '').split(',') 351 352 pkg_fn = self.tinfoil.cooker_data.pkg_fn 353 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 354 self.require_re = re.compile(r"require\s+(.+)") 355 self.include_re = re.compile(r"include\s+(.+)") 356 self.inherit_re = re.compile(r"inherit\s+(.+)") 357 358 global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split() 359 360 # The bb's DEPENDS and RDEPENDS 361 for f in pkg_fn: 362 f = bb.cache.virtualfn2realfn(f)[0] 363 # Get the layername that the file is in 364 layername = self.get_file_layer(f) 365 366 # The DEPENDS 367 deps = self.tinfoil.cooker_data.deps[f] 368 for pn in deps: 369 if pn in self.tinfoil.cooker_data.pkg_pn: 370 best = self.tinfoil.find_best_provider(pn) 371 self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers) 372 373 # The RDPENDS 374 all_rdeps = self.tinfoil.cooker_data.rundeps[f].values() 375 # Remove the duplicated or null one. 376 sorted_rdeps = {} 377 # The all_rdeps is the list in list, so we need two for loops 378 for k1 in all_rdeps: 379 for k2 in k1: 380 sorted_rdeps[k2] = 1 381 all_rdeps = sorted_rdeps.keys() 382 for rdep in all_rdeps: 383 all_p, best = self.tinfoil.get_runtime_providers(rdep) 384 if all_p: 385 if f in all_p: 386 # The recipe provides this one itself, ignore 387 continue 388 self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers) 389 390 # The RRECOMMENDS 391 all_rrecs = self.tinfoil.cooker_data.runrecs[f].values() 392 # Remove the duplicated or null one. 393 sorted_rrecs = {} 394 # The all_rrecs is the list in list, so we need two for loops 395 for k1 in all_rrecs: 396 for k2 in k1: 397 sorted_rrecs[k2] = 1 398 all_rrecs = sorted_rrecs.keys() 399 for rrec in all_rrecs: 400 all_p, best = self.tinfoil.get_runtime_providers(rrec) 401 if all_p: 402 if f in all_p: 403 # The recipe provides this one itself, ignore 404 continue 405 self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers) 406 407 # The inherit class 408 cls_re = re.compile('classes/') 409 if f in self.tinfoil.cooker_data.inherits: 410 inherits = self.tinfoil.cooker_data.inherits[f] 411 for cls in inherits: 412 # The inherits' format is [classes/cls, /path/to/classes/cls] 413 # ignore the classes/cls. 414 if not cls_re.match(cls): 415 classname = os.path.splitext(os.path.basename(cls))[0] 416 if classname in global_inherit: 417 continue 418 inherit_layername = self.get_file_layer(cls) 419 if inherit_layername != layername and not inherit_layername in ignore_layers: 420 if not args.filenames: 421 f_short = self.remove_layer_prefix(f) 422 cls = self.remove_layer_prefix(cls) 423 else: 424 f_short = f 425 logger.plain("%s inherits %s" % (f_short, cls)) 426 427 # The 'require/include xxx' in the bb file 428 pv_re = re.compile(r"\${PV}") 429 with open(f, 'r') as fnfile: 430 line = fnfile.readline() 431 while line: 432 m, keyword = self.match_require_include(line) 433 # Found the 'require/include xxxx' 434 if m: 435 needed_file = m.group(1) 436 # Replace the ${PV} with the real PV 437 if pv_re.search(needed_file) and f in self.tinfoil.cooker_data.pkg_pepvpr: 438 pv = self.tinfoil.cooker_data.pkg_pepvpr[f][1] 439 needed_file = re.sub(r"\${PV}", pv, needed_file) 440 self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers) 441 line = fnfile.readline() 442 443 # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass 444 conf_re = re.compile(r".*/conf/machine/[^\/]*\.conf$") 445 inc_re = re.compile(r".*\.inc$") 446 # The "inherit xxx" in .bbclass 447 bbclass_re = re.compile(r".*\.bbclass$") 448 for layerdir in self.bblayers: 449 layername = self.get_layer_name(layerdir) 450 for dirpath, dirnames, filenames in os.walk(layerdir): 451 for name in filenames: 452 f = os.path.join(dirpath, name) 453 s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f) 454 if s: 455 with open(f, 'r') as ffile: 456 line = ffile.readline() 457 while line: 458 m, keyword = self.match_require_include(line) 459 # Only bbclass has the "inherit xxx" here. 460 bbclass="" 461 if not m and f.endswith(".bbclass"): 462 m, keyword = self.match_inherit(line) 463 bbclass=".bbclass" 464 # Find a 'require/include xxxx' 465 if m: 466 self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers) 467 line = ffile.readline() 468 469 def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers): 470 """Print the depends that crosses a layer boundary""" 471 needed_file = bb.utils.which(bbpath, needed_filename) 472 if needed_file: 473 # Which layer is this file from 474 needed_layername = self.get_file_layer(needed_file) 475 if needed_layername != layername and not needed_layername in ignore_layers: 476 if not show_filenames: 477 f = self.remove_layer_prefix(f) 478 needed_file = self.remove_layer_prefix(needed_file) 479 logger.plain("%s %s %s" %(f, keyword, needed_file)) 480 481 def match_inherit(self, line): 482 """Match the inherit xxx line""" 483 return (self.inherit_re.match(line), "inherits") 484 485 def match_require_include(self, line): 486 """Match the require/include xxx line""" 487 m = self.require_re.match(line) 488 keyword = "requires" 489 if not m: 490 m = self.include_re.match(line) 491 keyword = "includes" 492 return (m, keyword) 493 494 def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers): 495 """Print the DEPENDS/RDEPENDS file that crosses a layer boundary""" 496 best_realfn = bb.cache.virtualfn2realfn(needed_file)[0] 497 needed_layername = self.get_file_layer(best_realfn) 498 if needed_layername != layername and not needed_layername in ignore_layers: 499 if not show_filenames: 500 f = self.remove_layer_prefix(f) 501 best_realfn = self.remove_layer_prefix(best_realfn) 502 503 logger.plain("%s %s %s" % (f, keyword, best_realfn)) 504 505 def register_commands(self, sp): 506 self.add_command(sp, 'show-layers', self.do_show_layers, parserecipes=False) 507 508 parser_show_overlayed = self.add_command(sp, 'show-overlayed', self.do_show_overlayed) 509 parser_show_overlayed.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') 510 parser_show_overlayed.add_argument('-s', '--same-version', help='only list overlayed recipes where the version is the same', action='store_true') 511 parser_show_overlayed.add_argument('--mc', help='use specified multiconfig', default='') 512 513 parser_show_recipes = self.add_command(sp, 'show-recipes', self.do_show_recipes) 514 parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') 515 parser_show_recipes.add_argument('-r', '--recipes-only', help='instead of the default formatting, list recipes only', action='store_true') 516 parser_show_recipes.add_argument('-m', '--multiple', help='only list where multiple recipes (in the same layer or different layers) exist for the same recipe name', action='store_true') 517 parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class(es) - separate multiple classes using , (without spaces)', metavar='CLASS', default='') 518 parser_show_recipes.add_argument('-l', '--layer', help='only list recipes from the selected layer', default='') 519 parser_show_recipes.add_argument('-b', '--bare', help='output just names without the "(skipped)" marker', action='store_true') 520 parser_show_recipes.add_argument('--mc', help='use specified multiconfig', default='') 521 parser_show_recipes.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') 522 523 parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends) 524 parser_show_appends.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') 525 526 parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends) 527 parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true') 528 parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME') 529