1# Development tool - ide-sdk command plugin 2# 3# Copyright (C) 2023-2024 Siemens AG 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool ide-sdk plugin""" 8 9import json 10import logging 11import os 12import re 13import shutil 14import stat 15import subprocess 16import sys 17from argparse import RawTextHelpFormatter 18from enum import Enum 19 20import scriptutils 21import bb 22from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, DevtoolError, parse_recipe 23from devtool.standard import get_real_srctree 24from devtool.ide_plugins import BuildTool 25 26 27logger = logging.getLogger('devtool') 28 29# dict of classes derived from IdeBase 30ide_plugins = {} 31 32 33class DevtoolIdeMode(Enum): 34 """Different modes are supported by the ide-sdk plugin. 35 36 The enum might be extended by more advanced modes in the future. Some ideas: 37 - auto: modified if all recipes are modified, shared if none of the recipes is modified. 38 - mixed: modified mode for modified recipes, shared mode for all other recipes. 39 """ 40 41 modified = 'modified' 42 shared = 'shared' 43 44 45class TargetDevice: 46 """SSH remote login parameters""" 47 48 def __init__(self, args): 49 self.extraoptions = '' 50 if args.no_host_check: 51 self.extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 52 self.ssh_sshexec = 'ssh' 53 if args.ssh_exec: 54 self.ssh_sshexec = args.ssh_exec 55 self.ssh_port = '' 56 if args.port: 57 self.ssh_port = "-p %s" % args.port 58 if args.key: 59 self.extraoptions += ' -i %s' % args.key 60 61 self.target = args.target 62 target_sp = args.target.split('@') 63 if len(target_sp) == 1: 64 self.login = "" 65 self.host = target_sp[0] 66 elif len(target_sp) == 2: 67 self.login = target_sp[0] 68 self.host = target_sp[1] 69 else: 70 logger.error("Invalid target argument: %s" % args.target) 71 72 73class RecipeNative: 74 """Base class for calling bitbake to provide a -native recipe""" 75 76 def __init__(self, name, target_arch=None): 77 self.name = name 78 self.target_arch = target_arch 79 self.bootstrap_tasks = [self.name + ':do_addto_recipe_sysroot'] 80 self.staging_bindir_native = None 81 self.target_sys = None 82 self.__native_bin = None 83 84 def _initialize(self, config, workspace, tinfoil): 85 """Get the parsed recipe""" 86 recipe_d = parse_recipe( 87 config, tinfoil, self.name, appends=True, filter_workspace=False) 88 if not recipe_d: 89 raise DevtoolError("Parsing %s recipe failed" % self.name) 90 self.staging_bindir_native = os.path.realpath( 91 recipe_d.getVar('STAGING_BINDIR_NATIVE')) 92 self.target_sys = recipe_d.getVar('TARGET_SYS') 93 return recipe_d 94 95 def initialize(self, config, workspace, tinfoil): 96 """Basic initialization that can be overridden by a derived class""" 97 self._initialize(config, workspace, tinfoil) 98 99 @property 100 def native_bin(self): 101 if not self.__native_bin: 102 raise DevtoolError("native binary name is not defined.") 103 return self.__native_bin 104 105 106class RecipeGdbCross(RecipeNative): 107 """Handle handle gdb-cross on the host and the gdbserver on the target device""" 108 109 def __init__(self, args, target_arch, target_device): 110 super().__init__('gdb-cross-' + target_arch, target_arch) 111 self.target_device = target_device 112 self.gdb = None 113 self.gdbserver_port_next = int(args.gdbserver_port_start) 114 self.config_db = {} 115 116 def __find_gdbserver(self, config, tinfoil): 117 """Absolute path of the gdbserver""" 118 recipe_d_gdb = parse_recipe( 119 config, tinfoil, 'gdb', appends=True, filter_workspace=False) 120 if not recipe_d_gdb: 121 raise DevtoolError("Parsing gdb recipe failed") 122 return os.path.join(recipe_d_gdb.getVar('bindir'), 'gdbserver') 123 124 def initialize(self, config, workspace, tinfoil): 125 super()._initialize(config, workspace, tinfoil) 126 gdb_bin = self.target_sys + '-gdb' 127 gdb_path = os.path.join( 128 self.staging_bindir_native, self.target_sys, gdb_bin) 129 self.gdb = gdb_path 130 self.gdbserver_path = self.__find_gdbserver(config, tinfoil) 131 132 @property 133 def host(self): 134 return self.target_device.host 135 136 137class RecipeImage: 138 """Handle some image recipe related properties 139 140 Most workflows require firmware that runs on the target device. 141 This firmware must be consistent with the setup of the host system. 142 In particular, the debug symbols must be compatible. For this, the 143 rootfs must be created as part of the SDK. 144 """ 145 146 def __init__(self, name): 147 self.combine_dbg_image = False 148 self.gdbserver_missing = False 149 self.name = name 150 self.rootfs = None 151 self.__rootfs_dbg = None 152 self.bootstrap_tasks = [self.name + ':do_build'] 153 154 def initialize(self, config, tinfoil): 155 image_d = parse_recipe( 156 config, tinfoil, self.name, appends=True, filter_workspace=False) 157 if not image_d: 158 raise DevtoolError( 159 "Parsing image recipe %s failed" % self.name) 160 161 self.combine_dbg_image = bb.data.inherits_class( 162 'image-combined-dbg', image_d) 163 164 workdir = image_d.getVar('WORKDIR') 165 self.rootfs = os.path.join(workdir, 'rootfs') 166 if image_d.getVar('IMAGE_GEN_DEBUGFS') == "1": 167 self.__rootfs_dbg = os.path.join(workdir, 'rootfs-dbg') 168 169 self.gdbserver_missing = 'gdbserver' not in image_d.getVar( 170 'IMAGE_INSTALL') 171 172 @property 173 def debug_support(self): 174 return bool(self.rootfs_dbg) 175 176 @property 177 def rootfs_dbg(self): 178 if self.__rootfs_dbg and os.path.isdir(self.__rootfs_dbg): 179 return self.__rootfs_dbg 180 return None 181 182 183class RecipeMetaIdeSupport: 184 """For the shared sysroots mode meta-ide-support is needed 185 186 For use cases where just a cross tool-chain is required but 187 no recipe is used, devtool ide-sdk abstracts calling bitbake meta-ide-support 188 and bitbake build-sysroots. This also allows to expose the cross-toolchains 189 to IDEs. For example VSCode support different tool-chains with e.g. cmake-kits. 190 """ 191 192 def __init__(self): 193 self.bootstrap_tasks = ['meta-ide-support:do_build'] 194 self.topdir = None 195 self.datadir = None 196 self.deploy_dir_image = None 197 self.build_sys = None 198 # From toolchain-scripts 199 self.real_multimach_target_sys = None 200 201 def initialize(self, config, tinfoil): 202 meta_ide_support_d = parse_recipe( 203 config, tinfoil, 'meta-ide-support', appends=True, filter_workspace=False) 204 if not meta_ide_support_d: 205 raise DevtoolError("Parsing meta-ide-support recipe failed") 206 207 self.topdir = meta_ide_support_d.getVar('TOPDIR') 208 self.datadir = meta_ide_support_d.getVar('datadir') 209 self.deploy_dir_image = meta_ide_support_d.getVar( 210 'DEPLOY_DIR_IMAGE') 211 self.build_sys = meta_ide_support_d.getVar('BUILD_SYS') 212 self.real_multimach_target_sys = meta_ide_support_d.getVar( 213 'REAL_MULTIMACH_TARGET_SYS') 214 215 216class RecipeBuildSysroots: 217 """For the shared sysroots mode build-sysroots is needed""" 218 219 def __init__(self): 220 self.standalone_sysroot = None 221 self.standalone_sysroot_native = None 222 self.bootstrap_tasks = [ 223 'build-sysroots:do_build_target_sysroot', 224 'build-sysroots:do_build_native_sysroot' 225 ] 226 227 def initialize(self, config, tinfoil): 228 build_sysroots_d = parse_recipe( 229 config, tinfoil, 'build-sysroots', appends=True, filter_workspace=False) 230 if not build_sysroots_d: 231 raise DevtoolError("Parsing build-sysroots recipe failed") 232 self.standalone_sysroot = build_sysroots_d.getVar( 233 'STANDALONE_SYSROOT') 234 self.standalone_sysroot_native = build_sysroots_d.getVar( 235 'STANDALONE_SYSROOT_NATIVE') 236 237 238class SharedSysrootsEnv: 239 """Handle the shared sysroots based workflow 240 241 Support the workflow with just a tool-chain without a recipe. 242 It's basically like: 243 bitbake some-dependencies 244 bitbake meta-ide-support 245 bitbake build-sysroots 246 Use the environment-* file found in the deploy folder 247 """ 248 249 def __init__(self): 250 self.ide_support = None 251 self.build_sysroots = None 252 253 def initialize(self, ide_support, build_sysroots): 254 self.ide_support = ide_support 255 self.build_sysroots = build_sysroots 256 257 def setup_ide(self, ide): 258 ide.setup(self) 259 260 261class RecipeNotModified: 262 """Handling of recipes added to the Direct DSK shared sysroots.""" 263 264 def __init__(self, name): 265 self.name = name 266 self.bootstrap_tasks = [name + ':do_populate_sysroot'] 267 268 269class RecipeModified: 270 """Handling of recipes in the workspace created by devtool modify""" 271 OE_INIT_BUILD_ENV = 'oe-init-build-env' 272 273 VALID_BASH_ENV_NAME_CHARS = re.compile(r"^[a-zA-Z0-9_]*$") 274 275 def __init__(self, name): 276 self.name = name 277 self.bootstrap_tasks = [name + ':do_install'] 278 self.gdb_cross = None 279 # workspace 280 self.real_srctree = None 281 self.srctree = None 282 self.ide_sdk_dir = None 283 self.ide_sdk_scripts_dir = None 284 self.bbappend = None 285 # recipe variables from d.getVar 286 self.b = None 287 self.base_libdir = None 288 self.bblayers = None 289 self.bpn = None 290 self.d = None 291 self.fakerootcmd = None 292 self.fakerootenv = None 293 self.libdir = None 294 self.max_process = None 295 self.package_arch = None 296 self.package_debug_split_style = None 297 self.path = None 298 self.pn = None 299 self.recipe_sysroot = None 300 self.recipe_sysroot_native = None 301 self.staging_incdir = None 302 self.strip_cmd = None 303 self.target_arch = None 304 self.target_dbgsrc_dir = None 305 self.topdir = None 306 self.workdir = None 307 self.recipe_id = None 308 # replicate bitbake build environment 309 self.exported_vars = None 310 self.cmd_compile = None 311 self.__oe_init_dir = None 312 # main build tool used by this recipe 313 self.build_tool = BuildTool.UNDEFINED 314 # build_tool = cmake 315 self.oecmake_generator = None 316 self.cmake_cache_vars = None 317 # build_tool = meson 318 self.meson_buildtype = None 319 self.meson_wrapper = None 320 self.mesonopts = None 321 self.extra_oemeson = None 322 self.meson_cross_file = None 323 324 def initialize(self, config, workspace, tinfoil): 325 recipe_d = parse_recipe( 326 config, tinfoil, self.name, appends=True, filter_workspace=False) 327 if not recipe_d: 328 raise DevtoolError("Parsing %s recipe failed" % self.name) 329 330 # Verify this recipe is built as externalsrc setup by devtool modify 331 workspacepn = check_workspace_recipe( 332 workspace, self.name, bbclassextend=True) 333 self.srctree = workspace[workspacepn]['srctree'] 334 # Need to grab this here in case the source is within a subdirectory 335 self.real_srctree = get_real_srctree( 336 self.srctree, recipe_d.getVar('S'), recipe_d.getVar('WORKDIR')) 337 self.bbappend = workspace[workspacepn]['bbappend'] 338 339 self.ide_sdk_dir = os.path.join( 340 config.workspace_path, 'ide-sdk', self.name) 341 if os.path.exists(self.ide_sdk_dir): 342 shutil.rmtree(self.ide_sdk_dir) 343 self.ide_sdk_scripts_dir = os.path.join(self.ide_sdk_dir, 'scripts') 344 345 self.b = recipe_d.getVar('B') 346 self.base_libdir = recipe_d.getVar('base_libdir') 347 self.bblayers = recipe_d.getVar('BBLAYERS').split() 348 self.bpn = recipe_d.getVar('BPN') 349 self.cxx = recipe_d.getVar('CXX') 350 self.d = recipe_d.getVar('D') 351 self.fakerootcmd = recipe_d.getVar('FAKEROOTCMD') 352 self.fakerootenv = recipe_d.getVar('FAKEROOTENV') 353 self.libdir = recipe_d.getVar('libdir') 354 self.max_process = int(recipe_d.getVar( 355 "BB_NUMBER_THREADS") or os.cpu_count() or 1) 356 self.package_arch = recipe_d.getVar('PACKAGE_ARCH') 357 self.package_debug_split_style = recipe_d.getVar( 358 'PACKAGE_DEBUG_SPLIT_STYLE') 359 self.path = recipe_d.getVar('PATH') 360 self.pn = recipe_d.getVar('PN') 361 self.recipe_sysroot = os.path.realpath( 362 recipe_d.getVar('RECIPE_SYSROOT')) 363 self.recipe_sysroot_native = os.path.realpath( 364 recipe_d.getVar('RECIPE_SYSROOT_NATIVE')) 365 self.staging_bindir_toolchain = os.path.realpath( 366 recipe_d.getVar('STAGING_BINDIR_TOOLCHAIN')) 367 self.staging_incdir = os.path.realpath( 368 recipe_d.getVar('STAGING_INCDIR')) 369 self.strip_cmd = recipe_d.getVar('STRIP') 370 self.target_arch = recipe_d.getVar('TARGET_ARCH') 371 self.target_dbgsrc_dir = recipe_d.getVar('TARGET_DBGSRC_DIR') 372 self.topdir = recipe_d.getVar('TOPDIR') 373 self.workdir = os.path.realpath(recipe_d.getVar('WORKDIR')) 374 375 self.__init_exported_variables(recipe_d) 376 377 if bb.data.inherits_class('cmake', recipe_d): 378 self.oecmake_generator = recipe_d.getVar('OECMAKE_GENERATOR') 379 self.__init_cmake_preset_cache(recipe_d) 380 self.build_tool = BuildTool.CMAKE 381 elif bb.data.inherits_class('meson', recipe_d): 382 self.meson_buildtype = recipe_d.getVar('MESON_BUILDTYPE') 383 self.mesonopts = recipe_d.getVar('MESONOPTS') 384 self.extra_oemeson = recipe_d.getVar('EXTRA_OEMESON') 385 self.meson_cross_file = recipe_d.getVar('MESON_CROSS_FILE') 386 self.build_tool = BuildTool.MESON 387 388 # Recipe ID is the identifier for IDE config sections 389 self.recipe_id = self.bpn + "-" + self.package_arch 390 self.recipe_id_pretty = self.bpn + ": " + self.package_arch 391 392 def append_to_bbappend(self, append_text): 393 with open(self.bbappend, 'a') as bbap: 394 bbap.write(append_text) 395 396 def remove_from_bbappend(self, append_text): 397 with open(self.bbappend, 'r') as bbap: 398 text = bbap.read() 399 new_text = text.replace(append_text, '') 400 with open(self.bbappend, 'w') as bbap: 401 bbap.write(new_text) 402 403 @staticmethod 404 def is_valid_shell_variable(var): 405 """Skip strange shell variables like systemd 406 407 prevent from strange bugs because of strange variables which 408 are not used in this context but break various tools. 409 """ 410 if RecipeModified.VALID_BASH_ENV_NAME_CHARS.match(var): 411 bb.debug(1, "ignoring variable: %s" % var) 412 return True 413 return False 414 415 def debug_build_config(self, args): 416 """Explicitely set for example CMAKE_BUILD_TYPE to Debug if not defined otherwise""" 417 if self.build_tool is BuildTool.CMAKE: 418 append_text = os.linesep + \ 419 'OECMAKE_ARGS:append = " -DCMAKE_BUILD_TYPE:STRING=Debug"' + os.linesep 420 if args.debug_build_config and not 'CMAKE_BUILD_TYPE' in self.cmake_cache_vars: 421 self.cmake_cache_vars['CMAKE_BUILD_TYPE'] = { 422 "type": "STRING", 423 "value": "Debug", 424 } 425 self.append_to_bbappend(append_text) 426 elif 'CMAKE_BUILD_TYPE' in self.cmake_cache_vars: 427 del self.cmake_cache_vars['CMAKE_BUILD_TYPE'] 428 self.remove_from_bbappend(append_text) 429 elif self.build_tool is BuildTool.MESON: 430 append_text = os.linesep + 'MESON_BUILDTYPE = "debug"' + os.linesep 431 if args.debug_build_config and self.meson_buildtype != "debug": 432 self.mesonopts.replace( 433 '--buildtype ' + self.meson_buildtype, '--buildtype debug') 434 self.append_to_bbappend(append_text) 435 elif self.meson_buildtype == "debug": 436 self.mesonopts.replace( 437 '--buildtype debug', '--buildtype plain') 438 self.remove_from_bbappend(append_text) 439 elif args.debug_build_config: 440 logger.warn( 441 "--debug-build-config is not implemented for this build tool yet.") 442 443 def solib_search_path(self, image): 444 """Search for debug symbols in the rootfs and rootfs-dbg 445 446 The debug symbols of shared libraries which are provided by other packages 447 are grabbed from the -dbg packages in the rootfs-dbg. 448 449 But most cross debugging tools like gdb, perf, and systemtap need to find 450 executable/library first and through it debuglink note find corresponding 451 symbols file. Therefore the library paths from the rootfs are added as well. 452 453 Note: For the devtool modified recipe compiled from the IDE, the debug 454 symbols are taken from the unstripped binaries in the image folder. 455 Also, devtool deploy-target takes the files from the image folder. 456 debug symbols in the image folder refer to the corresponding source files 457 with absolute paths of the build machine. Debug symbols found in the 458 rootfs-dbg are relocated and contain paths which refer to the source files 459 installed on the target device e.g. /usr/src/... 460 """ 461 base_libdir = self.base_libdir.lstrip('/') 462 libdir = self.libdir.lstrip('/') 463 so_paths = [ 464 # debug symbols for package_debug_split_style: debug-with-srcpkg or .debug 465 os.path.join(image.rootfs_dbg, base_libdir, ".debug"), 466 os.path.join(image.rootfs_dbg, libdir, ".debug"), 467 # debug symbols for package_debug_split_style: debug-file-directory 468 os.path.join(image.rootfs_dbg, "usr", "lib", "debug"), 469 470 # The binaries are required as well, the debug packages are not enough 471 # With image-combined-dbg.bbclass the binaries are copied into rootfs-dbg 472 os.path.join(image.rootfs_dbg, base_libdir), 473 os.path.join(image.rootfs_dbg, libdir), 474 # Without image-combined-dbg.bbclass the binaries are only in rootfs. 475 # Note: Stepping into source files located in rootfs-dbg does not 476 # work without image-combined-dbg.bbclass yet. 477 os.path.join(image.rootfs, base_libdir), 478 os.path.join(image.rootfs, libdir) 479 ] 480 return so_paths 481 482 def solib_search_path_str(self, image): 483 """Return a : separated list of paths usable by GDB's set solib-search-path""" 484 return ':'.join(self.solib_search_path(image)) 485 486 def __init_exported_variables(self, d): 487 """Find all variables with export flag set. 488 489 This allows to generate IDE configurations which compile with the same 490 environment as bitbake does. That's at least a reasonable default behavior. 491 """ 492 exported_vars = {} 493 494 vars = (key for key in d.keys() if not key.startswith( 495 "__") and not d.getVarFlag(key, "func", False)) 496 for var in vars: 497 func = d.getVarFlag(var, "func", False) 498 if d.getVarFlag(var, 'python', False) and func: 499 continue 500 export = d.getVarFlag(var, "export", False) 501 unexport = d.getVarFlag(var, "unexport", False) 502 if not export and not unexport and not func: 503 continue 504 if unexport: 505 continue 506 507 val = d.getVar(var) 508 if val is None: 509 continue 510 if set(var) & set("-.{}+"): 511 logger.warn( 512 "Warning: Found invalid character in variable name %s", str(var)) 513 continue 514 varExpanded = d.expand(var) 515 val = str(val) 516 517 if not RecipeModified.is_valid_shell_variable(varExpanded): 518 continue 519 520 if func: 521 code_line = "line: {0}, file: {1}\n".format( 522 d.getVarFlag(var, "lineno", False), 523 d.getVarFlag(var, "filename", False)) 524 val = val.rstrip('\n') 525 logger.warn("Warning: exported shell function %s() is not exported (%s)" % 526 (varExpanded, code_line)) 527 continue 528 529 if export: 530 exported_vars[varExpanded] = val.strip() 531 continue 532 533 self.exported_vars = exported_vars 534 535 def __init_cmake_preset_cache(self, d): 536 """Get the arguments passed to cmake 537 538 Replicate the cmake configure arguments with all details to 539 share on build folder between bitbake and SDK. 540 """ 541 site_file = os.path.join(self.workdir, 'site-file.cmake') 542 if os.path.exists(site_file): 543 print("Warning: site-file.cmake is not supported") 544 545 cache_vars = {} 546 oecmake_args = d.getVar('OECMAKE_ARGS').split() 547 extra_oecmake = d.getVar('EXTRA_OECMAKE').split() 548 for param in oecmake_args + extra_oecmake: 549 d_pref = "-D" 550 if param.startswith(d_pref): 551 param = param[len(d_pref):] 552 else: 553 print("Error: expected a -D") 554 param_s = param.split('=', 1) 555 param_nt = param_s[0].split(':', 1) 556 557 def handle_undefined_variable(var): 558 if var.startswith('${') and var.endswith('}'): 559 return '' 560 else: 561 return var 562 # Example: FOO=ON 563 if len(param_nt) == 1: 564 cache_vars[param_s[0]] = handle_undefined_variable(param_s[1]) 565 # Example: FOO:PATH=/tmp 566 elif len(param_nt) == 2: 567 cache_vars[param_nt[0]] = { 568 "type": param_nt[1], 569 "value": handle_undefined_variable(param_s[1]), 570 } 571 else: 572 print("Error: cannot parse %s" % param) 573 self.cmake_cache_vars = cache_vars 574 575 def cmake_preset(self): 576 """Create a preset for cmake that mimics how bitbake calls cmake""" 577 toolchain_file = os.path.join(self.workdir, 'toolchain.cmake') 578 cmake_executable = os.path.join( 579 self.recipe_sysroot_native, 'usr', 'bin', 'cmake') 580 self.cmd_compile = cmake_executable + " --build --preset " + self.recipe_id 581 582 preset_dict_configure = { 583 "name": self.recipe_id, 584 "displayName": self.recipe_id_pretty, 585 "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch), 586 "binaryDir": self.b, 587 "generator": self.oecmake_generator, 588 "toolchainFile": toolchain_file, 589 "cacheVariables": self.cmake_cache_vars, 590 "environment": self.exported_vars, 591 "cmakeExecutable": cmake_executable 592 } 593 594 preset_dict_build = { 595 "name": self.recipe_id, 596 "displayName": self.recipe_id_pretty, 597 "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch), 598 "configurePreset": self.recipe_id, 599 "inheritConfigureEnvironment": True 600 } 601 602 preset_dict_test = { 603 "name": self.recipe_id, 604 "displayName": self.recipe_id_pretty, 605 "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch), 606 "configurePreset": self.recipe_id, 607 "inheritConfigureEnvironment": True 608 } 609 610 preset_dict = { 611 "version": 3, # cmake 3.21, backward compatible with kirkstone 612 "configurePresets": [preset_dict_configure], 613 "buildPresets": [preset_dict_build], 614 "testPresets": [preset_dict_test] 615 } 616 617 # Finally write the json file 618 json_file = 'CMakeUserPresets.json' 619 json_path = os.path.join(self.real_srctree, json_file) 620 logger.info("Updating CMake preset: %s (%s)" % (json_file, json_path)) 621 if not os.path.exists(self.real_srctree): 622 os.makedirs(self.real_srctree) 623 try: 624 with open(json_path) as f: 625 orig_dict = json.load(f) 626 except json.decoder.JSONDecodeError: 627 logger.info( 628 "Decoding %s failed. Probably because of comments in the json file" % json_path) 629 orig_dict = {} 630 except FileNotFoundError: 631 orig_dict = {} 632 633 # Add or update the presets for the recipe and keep other presets 634 for k, v in preset_dict.items(): 635 if isinstance(v, list): 636 update_preset = v[0] 637 preset_added = False 638 if k in orig_dict: 639 for index, orig_preset in enumerate(orig_dict[k]): 640 if 'name' in orig_preset: 641 if orig_preset['name'] == update_preset['name']: 642 logger.debug("Updating preset: %s" % 643 orig_preset['name']) 644 orig_dict[k][index] = update_preset 645 preset_added = True 646 break 647 else: 648 logger.debug("keeping preset: %s" % 649 orig_preset['name']) 650 else: 651 logger.warn("preset without a name found") 652 if not preset_added: 653 if not k in orig_dict: 654 orig_dict[k] = [] 655 orig_dict[k].append(update_preset) 656 logger.debug("Added preset: %s" % 657 update_preset['name']) 658 else: 659 orig_dict[k] = v 660 661 with open(json_path, 'w') as f: 662 json.dump(orig_dict, f, indent=4) 663 664 def gen_meson_wrapper(self): 665 """Generate a wrapper script to call meson with the cross environment""" 666 bb.utils.mkdirhier(self.ide_sdk_scripts_dir) 667 meson_wrapper = os.path.join(self.ide_sdk_scripts_dir, 'meson') 668 meson_real = os.path.join( 669 self.recipe_sysroot_native, 'usr', 'bin', 'meson.real') 670 with open(meson_wrapper, 'w') as mwrap: 671 mwrap.write("#!/bin/sh" + os.linesep) 672 for var, val in self.exported_vars.items(): 673 mwrap.write('export %s="%s"' % (var, val) + os.linesep) 674 mwrap.write("unset CC CXX CPP LD AR NM STRIP" + os.linesep) 675 private_temp = os.path.join(self.b, "meson-private", "tmp") 676 mwrap.write('mkdir -p "%s"' % private_temp + os.linesep) 677 mwrap.write('export TMPDIR="%s"' % private_temp + os.linesep) 678 mwrap.write('exec "%s" "$@"' % meson_real + os.linesep) 679 st = os.stat(meson_wrapper) 680 os.chmod(meson_wrapper, st.st_mode | stat.S_IEXEC) 681 self.meson_wrapper = meson_wrapper 682 self.cmd_compile = meson_wrapper + " compile -C " + self.b 683 684 def which(self, executable): 685 bin_path = shutil.which(executable, path=self.path) 686 if not bin_path: 687 raise DevtoolError( 688 'Cannot find %s. Probably the recipe %s is not built yet.' % (executable, self.bpn)) 689 return bin_path 690 691 @staticmethod 692 def is_elf_file(file_path): 693 with open(file_path, "rb") as f: 694 data = f.read(4) 695 if data == b'\x7fELF': 696 return True 697 return False 698 699 def find_installed_binaries(self): 700 """find all executable elf files in the image directory""" 701 binaries = [] 702 d_len = len(self.d) 703 re_so = re.compile(r'.*\.so[.0-9]*$') 704 for root, _, files in os.walk(self.d, followlinks=False): 705 for file in files: 706 if os.path.islink(file): 707 continue 708 if re_so.match(file): 709 continue 710 abs_name = os.path.join(root, file) 711 if os.access(abs_name, os.X_OK) and RecipeModified.is_elf_file(abs_name): 712 binaries.append(abs_name[d_len:]) 713 return sorted(binaries) 714 715 def gen_delete_package_dirs(self): 716 """delete folders of package tasks 717 718 This is a workaround for and issue with recipes having their sources 719 downloaded as file:// 720 This likely breaks pseudo like: 721 path mismatch [3 links]: ino 79147802 db 722 .../build/tmp/.../cmake-example/1.0/package/usr/src/debug/ 723 cmake-example/1.0-r0/oe-local-files/cpp-example-lib.cpp 724 .../build/workspace/sources/cmake-example/oe-local-files/cpp-example-lib.cpp 725 Since the files are anyway outdated lets deleted them (also from pseudo's db) to workaround this issue. 726 """ 727 cmd_lines = ['#!/bin/sh'] 728 729 # Set up the appropriate environment 730 newenv = dict(os.environ) 731 for varvalue in self.fakerootenv.split(): 732 if '=' in varvalue: 733 splitval = varvalue.split('=', 1) 734 newenv[splitval[0]] = splitval[1] 735 736 # Replicate the environment variables from bitbake 737 for var, val in newenv.items(): 738 if not RecipeModified.is_valid_shell_variable(var): 739 continue 740 cmd_lines.append('%s="%s"' % (var, val)) 741 cmd_lines.append('export %s' % var) 742 743 # Delete the folders 744 pkg_dirs = ' '.join([os.path.join(self.workdir, d) for d in [ 745 "package", "packages-split", "pkgdata", "sstate-install-package", "debugsources.list", "*.spec"]]) 746 cmd = "%s rm -rf %s" % (self.fakerootcmd, pkg_dirs) 747 cmd_lines.append('%s || { "%s failed"; exit 1; }' % (cmd, cmd)) 748 749 return self.write_script(cmd_lines, 'delete_package_dirs') 750 751 def gen_deploy_target_script(self, args): 752 """Generate a script which does what devtool deploy-target does 753 754 This script is much quicker than devtool target-deploy. Because it 755 does not need to start a bitbake server. All information from tinfoil 756 is hard-coded in the generated script. 757 """ 758 cmd_lines = ['#!%s' % str(sys.executable)] 759 cmd_lines.append('import sys') 760 cmd_lines.append('devtool_sys_path = %s' % str(sys.path)) 761 cmd_lines.append('devtool_sys_path.reverse()') 762 cmd_lines.append('for p in devtool_sys_path:') 763 cmd_lines.append(' if p not in sys.path:') 764 cmd_lines.append(' sys.path.insert(0, p)') 765 cmd_lines.append('from devtool.deploy import deploy_no_d') 766 args_filter = ['debug', 'dry_run', 'key', 'no_check_space', 'no_host_check', 767 'no_preserve', 'port', 'show_status', 'ssh_exec', 'strip', 'target'] 768 filtered_args_dict = {key: value for key, value in vars( 769 args).items() if key in args_filter} 770 cmd_lines.append('filtered_args_dict = %s' % str(filtered_args_dict)) 771 cmd_lines.append('class Dict2Class(object):') 772 cmd_lines.append(' def __init__(self, my_dict):') 773 cmd_lines.append(' for key in my_dict:') 774 cmd_lines.append(' setattr(self, key, my_dict[key])') 775 cmd_lines.append('filtered_args = Dict2Class(filtered_args_dict)') 776 cmd_lines.append( 777 'setattr(filtered_args, "recipename", "%s")' % self.bpn) 778 cmd_lines.append('deploy_no_d("%s", "%s", "%s", "%s", "%s", "%s", %d, "%s", "%s", filtered_args)' % 779 (self.d, self.workdir, self.path, self.strip_cmd, 780 self.libdir, self.base_libdir, self.max_process, 781 self.fakerootcmd, self.fakerootenv)) 782 return self.write_script(cmd_lines, 'deploy_target') 783 784 def gen_install_deploy_script(self, args): 785 """Generate a script which does install and deploy""" 786 cmd_lines = ['#!/bin/bash'] 787 788 cmd_lines.append(self.gen_delete_package_dirs()) 789 790 # . oe-init-build-env $BUILDDIR 791 # Note: Sourcing scripts with arguments requires bash 792 cmd_lines.append('cd "%s" || { echo "cd %s failed"; exit 1; }' % ( 793 self.oe_init_dir, self.oe_init_dir)) 794 cmd_lines.append('. "%s" "%s" || { echo ". %s %s failed"; exit 1; }' % ( 795 self.oe_init_build_env, self.topdir, self.oe_init_build_env, self.topdir)) 796 797 # bitbake -c install 798 cmd_lines.append( 799 'bitbake %s -c install --force || { echo "bitbake %s -c install --force failed"; exit 1; }' % (self.bpn, self.bpn)) 800 801 # Self contained devtool deploy-target 802 cmd_lines.append(self.gen_deploy_target_script(args)) 803 804 return self.write_script(cmd_lines, 'install_and_deploy') 805 806 def write_script(self, cmd_lines, script_name): 807 bb.utils.mkdirhier(self.ide_sdk_scripts_dir) 808 script_name_arch = script_name + '_' + self.recipe_id 809 script_file = os.path.join(self.ide_sdk_scripts_dir, script_name_arch) 810 with open(script_file, 'w') as script_f: 811 script_f.write(os.linesep.join(cmd_lines)) 812 st = os.stat(script_file) 813 os.chmod(script_file, st.st_mode | stat.S_IEXEC) 814 return script_file 815 816 @property 817 def oe_init_build_env(self): 818 """Find the oe-init-build-env used for this setup""" 819 oe_init_dir = self.oe_init_dir 820 if oe_init_dir: 821 return os.path.join(oe_init_dir, RecipeModified.OE_INIT_BUILD_ENV) 822 return None 823 824 @property 825 def oe_init_dir(self): 826 """Find the directory where the oe-init-build-env is located 827 828 Assumption: There might be a layer with higher priority than poky 829 which provides to oe-init-build-env in the layer's toplevel folder. 830 """ 831 if not self.__oe_init_dir: 832 for layer in reversed(self.bblayers): 833 result = subprocess.run( 834 ['git', 'rev-parse', '--show-toplevel'], cwd=layer, capture_output=True) 835 if result.returncode == 0: 836 oe_init_dir = result.stdout.decode('utf-8').strip() 837 oe_init_path = os.path.join( 838 oe_init_dir, RecipeModified.OE_INIT_BUILD_ENV) 839 if os.path.exists(oe_init_path): 840 logger.debug("Using %s from: %s" % ( 841 RecipeModified.OE_INIT_BUILD_ENV, oe_init_path)) 842 self.__oe_init_dir = oe_init_dir 843 break 844 if not self.__oe_init_dir: 845 logger.error("Cannot find the bitbake top level folder") 846 return self.__oe_init_dir 847 848 849def ide_setup(args, config, basepath, workspace): 850 """Generate the IDE configuration for the workspace""" 851 852 # Explicitely passing some special recipes does not make sense 853 for recipe in args.recipenames: 854 if recipe in ['meta-ide-support', 'build-sysroots']: 855 raise DevtoolError("Invalid recipe: %s." % recipe) 856 857 # Collect information about tasks which need to be bitbaked 858 bootstrap_tasks = [] 859 bootstrap_tasks_late = [] 860 tinfoil = setup_tinfoil(config_only=False, basepath=basepath) 861 try: 862 # define mode depending on recipes which need to be processed 863 recipes_image_names = [] 864 recipes_modified_names = [] 865 recipes_other_names = [] 866 for recipe in args.recipenames: 867 try: 868 check_workspace_recipe( 869 workspace, recipe, bbclassextend=True) 870 recipes_modified_names.append(recipe) 871 except DevtoolError: 872 recipe_d = parse_recipe( 873 config, tinfoil, recipe, appends=True, filter_workspace=False) 874 if not recipe_d: 875 raise DevtoolError("Parsing recipe %s failed" % recipe) 876 if bb.data.inherits_class('image', recipe_d): 877 recipes_image_names.append(recipe) 878 else: 879 recipes_other_names.append(recipe) 880 881 invalid_params = False 882 if args.mode == DevtoolIdeMode.shared: 883 if len(recipes_modified_names): 884 logger.error("In shared sysroots mode modified recipes %s cannot be handled." % str( 885 recipes_modified_names)) 886 invalid_params = True 887 if args.mode == DevtoolIdeMode.modified: 888 if len(recipes_other_names): 889 logger.error("Only in shared sysroots mode not modified recipes %s can be handled." % str( 890 recipes_other_names)) 891 invalid_params = True 892 if len(recipes_image_names) != 1: 893 logger.error( 894 "One image recipe is required as the rootfs for the remote development.") 895 invalid_params = True 896 for modified_recipe_name in recipes_modified_names: 897 if modified_recipe_name.startswith('nativesdk-') or modified_recipe_name.endswith('-native'): 898 logger.error( 899 "Only cross compiled recipes are support. %s is not cross." % modified_recipe_name) 900 invalid_params = True 901 902 if invalid_params: 903 raise DevtoolError("Invalid parameters are passed.") 904 905 # For the shared sysroots mode, add all dependencies of all the images to the sysroots 906 # For the modified mode provide one rootfs and the corresponding debug symbols via rootfs-dbg 907 recipes_images = [] 908 for recipes_image_name in recipes_image_names: 909 logger.info("Using image: %s" % recipes_image_name) 910 recipe_image = RecipeImage(recipes_image_name) 911 recipe_image.initialize(config, tinfoil) 912 bootstrap_tasks += recipe_image.bootstrap_tasks 913 recipes_images.append(recipe_image) 914 915 # Provide a Direct SDK with shared sysroots 916 recipes_not_modified = [] 917 if args.mode == DevtoolIdeMode.shared: 918 ide_support = RecipeMetaIdeSupport() 919 ide_support.initialize(config, tinfoil) 920 bootstrap_tasks += ide_support.bootstrap_tasks 921 922 logger.info("Adding %s to the Direct SDK sysroots." % 923 str(recipes_other_names)) 924 for recipe_name in recipes_other_names: 925 recipe_not_modified = RecipeNotModified(recipe_name) 926 bootstrap_tasks += recipe_not_modified.bootstrap_tasks 927 recipes_not_modified.append(recipe_not_modified) 928 929 build_sysroots = RecipeBuildSysroots() 930 build_sysroots.initialize(config, tinfoil) 931 bootstrap_tasks_late += build_sysroots.bootstrap_tasks 932 shared_env = SharedSysrootsEnv() 933 shared_env.initialize(ide_support, build_sysroots) 934 935 recipes_modified = [] 936 if args.mode == DevtoolIdeMode.modified: 937 logger.info("Setting up workspaces for modified recipe: %s" % 938 str(recipes_modified_names)) 939 gdbs_cross = {} 940 for recipe_name in recipes_modified_names: 941 recipe_modified = RecipeModified(recipe_name) 942 recipe_modified.initialize(config, workspace, tinfoil) 943 bootstrap_tasks += recipe_modified.bootstrap_tasks 944 recipes_modified.append(recipe_modified) 945 946 if recipe_modified.target_arch not in gdbs_cross: 947 target_device = TargetDevice(args) 948 gdb_cross = RecipeGdbCross( 949 args, recipe_modified.target_arch, target_device) 950 gdb_cross.initialize(config, workspace, tinfoil) 951 bootstrap_tasks += gdb_cross.bootstrap_tasks 952 gdbs_cross[recipe_modified.target_arch] = gdb_cross 953 recipe_modified.gdb_cross = gdbs_cross[recipe_modified.target_arch] 954 955 finally: 956 tinfoil.shutdown() 957 958 if not args.skip_bitbake: 959 bb_cmd = 'bitbake ' 960 if args.bitbake_k: 961 bb_cmd += "-k " 962 bb_cmd_early = bb_cmd + ' '.join(bootstrap_tasks) 963 exec_build_env_command( 964 config.init_path, basepath, bb_cmd_early, watch=True) 965 if bootstrap_tasks_late: 966 bb_cmd_late = bb_cmd + ' '.join(bootstrap_tasks_late) 967 exec_build_env_command( 968 config.init_path, basepath, bb_cmd_late, watch=True) 969 970 for recipe_image in recipes_images: 971 if (recipe_image.gdbserver_missing): 972 logger.warning( 973 "gdbserver not installed in image %s. Remote debugging will not be available" % recipe_image) 974 975 if recipe_image.combine_dbg_image is False: 976 logger.warning( 977 'IMAGE_CLASSES += "image-combined-dbg" is missing for image %s. Remote debugging will not find debug symbols from rootfs-dbg.' % recipe_image) 978 979 # Instantiate the active IDE plugin 980 ide = ide_plugins[args.ide]() 981 if args.mode == DevtoolIdeMode.shared: 982 ide.setup_shared_sysroots(shared_env) 983 elif args.mode == DevtoolIdeMode.modified: 984 for recipe_modified in recipes_modified: 985 if recipe_modified.build_tool is BuildTool.CMAKE: 986 recipe_modified.cmake_preset() 987 if recipe_modified.build_tool is BuildTool.MESON: 988 recipe_modified.gen_meson_wrapper() 989 ide.setup_modified_recipe( 990 args, recipe_image, recipe_modified) 991 else: 992 raise DevtoolError("Must not end up here.") 993 994 995def register_commands(subparsers, context): 996 """Register devtool subcommands from this plugin""" 997 998 global ide_plugins 999 1000 # Search for IDE plugins in all sub-folders named ide_plugins where devtool seraches for plugins. 1001 pluginpaths = [os.path.join(path, 'ide_plugins') 1002 for path in context.pluginpaths] 1003 ide_plugin_modules = [] 1004 for pluginpath in pluginpaths: 1005 scriptutils.load_plugins(logger, ide_plugin_modules, pluginpath) 1006 1007 for ide_plugin_module in ide_plugin_modules: 1008 if hasattr(ide_plugin_module, 'register_ide_plugin'): 1009 ide_plugin_module.register_ide_plugin(ide_plugins) 1010 # Sort plugins according to their priority. The first entry is the default IDE plugin. 1011 ide_plugins = dict(sorted(ide_plugins.items(), 1012 key=lambda p: p[1].ide_plugin_priority(), reverse=True)) 1013 1014 parser_ide_sdk = subparsers.add_parser('ide-sdk', group='working', order=50, formatter_class=RawTextHelpFormatter, 1015 help='Setup the SDK and configure the IDE') 1016 parser_ide_sdk.add_argument( 1017 'recipenames', nargs='+', help='Generate an IDE configuration suitable to work on the given recipes.\n' 1018 'Depending on the --mode paramter different types of SDKs and IDE configurations are generated.') 1019 parser_ide_sdk.add_argument( 1020 '-m', '--mode', type=DevtoolIdeMode, default=DevtoolIdeMode.modified, 1021 help='Different SDK types are supported:\n' 1022 '- "' + DevtoolIdeMode.modified.name + '" (default):\n' 1023 ' devtool modify creates a workspace to work on the source code of a recipe.\n' 1024 ' devtool ide-sdk builds the SDK and generates the IDE configuration(s) in the workspace directorie(s)\n' 1025 ' Usage example:\n' 1026 ' devtool modify cmake-example\n' 1027 ' devtool ide-sdk cmake-example core-image-minimal\n' 1028 ' Start the IDE in the workspace folder\n' 1029 ' At least one devtool modified recipe plus one image recipe are required:\n' 1030 ' The image recipe is used to generate the target image and the remote debug configuration.\n' 1031 '- "' + DevtoolIdeMode.shared.name + '":\n' 1032 ' Usage example:\n' 1033 ' devtool ide-sdk -m ' + DevtoolIdeMode.shared.name + ' recipe(s)\n' 1034 ' This command generates a cross-toolchain as well as the corresponding shared sysroot directories.\n' 1035 ' To use this tool-chain the environment-* file found in the deploy..image folder needs to be sourced into a shell.\n' 1036 ' In case of VSCode and cmake the tool-chain is also exposed as a cmake-kit') 1037 default_ide = list(ide_plugins.keys())[0] 1038 parser_ide_sdk.add_argument( 1039 '-i', '--ide', choices=ide_plugins.keys(), default=default_ide, 1040 help='Setup the configuration for this IDE (default: %s)' % default_ide) 1041 parser_ide_sdk.add_argument( 1042 '-t', '--target', default='root@192.168.7.2', 1043 help='Live target machine running an ssh server: user@hostname.') 1044 parser_ide_sdk.add_argument( 1045 '-G', '--gdbserver-port-start', default="1234", help='port where gdbserver is listening.') 1046 parser_ide_sdk.add_argument( 1047 '-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') 1048 parser_ide_sdk.add_argument( 1049 '-e', '--ssh-exec', help='Executable to use in place of ssh') 1050 parser_ide_sdk.add_argument( 1051 '-P', '--port', help='Specify ssh port to use for connection to the target') 1052 parser_ide_sdk.add_argument( 1053 '-I', '--key', help='Specify ssh private key for connection to the target') 1054 parser_ide_sdk.add_argument( 1055 '--skip-bitbake', help='Generate IDE configuration but skip calling bibtake to update the SDK.', action='store_true') 1056 parser_ide_sdk.add_argument( 1057 '-k', '--bitbake-k', help='Pass -k parameter to bitbake', action='store_true') 1058 parser_ide_sdk.add_argument( 1059 '--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') 1060 parser_ide_sdk.add_argument( 1061 '-n', '--dry-run', help='List files to be undeployed only', action='store_true') 1062 parser_ide_sdk.add_argument( 1063 '-s', '--show-status', help='Show progress/status output', action='store_true') 1064 parser_ide_sdk.add_argument( 1065 '-p', '--no-preserve', help='Do not preserve existing files', action='store_true') 1066 parser_ide_sdk.add_argument( 1067 '--no-check-space', help='Do not check for available space before deploying', action='store_true') 1068 parser_ide_sdk.add_argument( 1069 '--debug-build-config', help='Use debug build flags, for example set CMAKE_BUILD_TYPE=Debug', action='store_true') 1070 parser_ide_sdk.set_defaults(func=ide_setup) 1071