1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7# This module provides a place to collect various wic-related utils 8# for the OpenEmbedded Image Tools. 9# 10# AUTHORS 11# Tom Zanussi <tom.zanussi (at] linux.intel.com> 12# 13"""Miscellaneous functions.""" 14 15import logging 16import os 17import re 18import subprocess 19import shutil 20 21from collections import defaultdict 22 23from wic import WicError 24 25logger = logging.getLogger('wic') 26 27# executable -> recipe pairs for exec_native_cmd 28NATIVE_RECIPES = {"bmaptool": "bmaptool", 29 "dumpe2fs": "e2fsprogs", 30 "grub-mkimage": "grub-efi", 31 "isohybrid": "syslinux", 32 "mcopy": "mtools", 33 "mdel" : "mtools", 34 "mdeltree" : "mtools", 35 "mdir" : "mtools", 36 "mkdosfs": "dosfstools", 37 "mkisofs": "cdrtools", 38 "mkfs.btrfs": "btrfs-tools", 39 "mkfs.erofs": "erofs-utils", 40 "mkfs.ext2": "e2fsprogs", 41 "mkfs.ext3": "e2fsprogs", 42 "mkfs.ext4": "e2fsprogs", 43 "mkfs.vfat": "dosfstools", 44 "mksquashfs": "squashfs-tools", 45 "mkswap": "util-linux", 46 "mmd": "mtools", 47 "parted": "parted", 48 "sfdisk": "util-linux", 49 "sgdisk": "gptfdisk", 50 "syslinux": "syslinux", 51 "tar": "tar" 52 } 53 54def runtool(cmdln_or_args): 55 """ wrapper for most of the subprocess calls 56 input: 57 cmdln_or_args: can be both args and cmdln str (shell=True) 58 return: 59 rc, output 60 """ 61 if isinstance(cmdln_or_args, list): 62 cmd = cmdln_or_args[0] 63 shell = False 64 else: 65 import shlex 66 cmd = shlex.split(cmdln_or_args)[0] 67 shell = True 68 69 sout = subprocess.PIPE 70 serr = subprocess.STDOUT 71 72 try: 73 process = subprocess.Popen(cmdln_or_args, stdout=sout, 74 stderr=serr, shell=shell) 75 sout, serr = process.communicate() 76 # combine stdout and stderr, filter None out and decode 77 out = ''.join([out.decode('utf-8') for out in [sout, serr] if out]) 78 except OSError as err: 79 if err.errno == 2: 80 # [Errno 2] No such file or directory 81 raise WicError('Cannot run command: %s, lost dependency?' % cmd) 82 else: 83 raise # relay 84 85 return process.returncode, out 86 87def _exec_cmd(cmd_and_args, as_shell=False): 88 """ 89 Execute command, catching stderr, stdout 90 91 Need to execute as_shell if the command uses wildcards 92 """ 93 logger.debug("_exec_cmd: %s", cmd_and_args) 94 args = cmd_and_args.split() 95 logger.debug(args) 96 97 if as_shell: 98 ret, out = runtool(cmd_and_args) 99 else: 100 ret, out = runtool(args) 101 out = out.strip() 102 if ret != 0: 103 raise WicError("_exec_cmd: %s returned '%s' instead of 0\noutput: %s" % \ 104 (cmd_and_args, ret, out)) 105 106 logger.debug("_exec_cmd: output for %s (rc = %d): %s", 107 cmd_and_args, ret, out) 108 109 return ret, out 110 111 112def exec_cmd(cmd_and_args, as_shell=False): 113 """ 114 Execute command, return output 115 """ 116 return _exec_cmd(cmd_and_args, as_shell)[1] 117 118def find_executable(cmd, paths): 119 recipe = cmd 120 if recipe in NATIVE_RECIPES: 121 recipe = NATIVE_RECIPES[recipe] 122 provided = get_bitbake_var("ASSUME_PROVIDED") 123 if provided and "%s-native" % recipe in provided: 124 return True 125 126 return shutil.which(cmd, path=paths) 127 128def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): 129 """ 130 Execute native command, catching stderr, stdout 131 132 Need to execute as_shell if the command uses wildcards 133 134 Always need to execute native commands as_shell 135 """ 136 # The reason -1 is used is because there may be "export" commands. 137 args = cmd_and_args.split(';')[-1].split() 138 logger.debug(args) 139 140 if pseudo: 141 cmd_and_args = pseudo + cmd_and_args 142 143 hosttools_dir = get_bitbake_var("HOSTTOOLS_DIR") 144 target_sys = get_bitbake_var("TARGET_SYS") 145 146 native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/usr/bin/%s:%s/bin:%s" % \ 147 (native_sysroot, native_sysroot, 148 native_sysroot, native_sysroot, target_sys, 149 native_sysroot, hosttools_dir) 150 151 native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ 152 (native_paths, cmd_and_args) 153 logger.debug("exec_native_cmd: %s", native_cmd_and_args) 154 155 # If the command isn't in the native sysroot say we failed. 156 if find_executable(args[0], native_paths): 157 ret, out = _exec_cmd(native_cmd_and_args, True) 158 else: 159 ret = 127 160 out = "can't find native executable %s in %s" % (args[0], native_paths) 161 162 prog = args[0] 163 # shell command-not-found 164 if ret == 127 \ 165 or (pseudo and ret == 1 and out == "Can't find '%s' in $PATH." % prog): 166 msg = "A native program %s required to build the image "\ 167 "was not found (see details above).\n\n" % prog 168 recipe = NATIVE_RECIPES.get(prog) 169 if recipe: 170 msg += "Please make sure wic-tools have %s-native in its DEPENDS, "\ 171 "build it with 'bitbake wic-tools' and try again.\n" % recipe 172 else: 173 msg += "Wic failed to find a recipe to build native %s. Please "\ 174 "file a bug against wic.\n" % prog 175 raise WicError(msg) 176 177 return ret, out 178 179BOOTDD_EXTRA_SPACE = 16384 180 181class BitbakeVars(defaultdict): 182 """ 183 Container for Bitbake variables. 184 """ 185 def __init__(self): 186 defaultdict.__init__(self, dict) 187 188 # default_image and vars_dir attributes should be set from outside 189 self.default_image = None 190 self.vars_dir = None 191 192 def _parse_line(self, line, image, matcher=re.compile(r"^([a-zA-Z0-9\-_+./~]+)=(.*)")): 193 """ 194 Parse one line from bitbake -e output or from .env file. 195 Put result key-value pair into the storage. 196 """ 197 if "=" not in line: 198 return 199 match = matcher.match(line) 200 if not match: 201 return 202 key, val = match.groups() 203 self[image][key] = val.strip('"') 204 205 def get_var(self, var, image=None, cache=True): 206 """ 207 Get bitbake variable from 'bitbake -e' output or from .env file. 208 This is a lazy method, i.e. it runs bitbake or parses file only when 209 only when variable is requested. It also caches results. 210 """ 211 if not image: 212 image = self.default_image 213 214 if image not in self: 215 if image and self.vars_dir: 216 fname = os.path.join(self.vars_dir, image + '.env') 217 if os.path.isfile(fname): 218 # parse .env file 219 with open(fname) as varsfile: 220 for line in varsfile: 221 self._parse_line(line, image) 222 else: 223 print("Couldn't get bitbake variable from %s." % fname) 224 print("File %s doesn't exist." % fname) 225 return 226 else: 227 # Get bitbake -e output 228 cmd = "bitbake -e" 229 if image: 230 cmd += " %s" % image 231 232 log_level = logger.getEffectiveLevel() 233 logger.setLevel(logging.INFO) 234 ret, lines = _exec_cmd(cmd) 235 logger.setLevel(log_level) 236 237 if ret: 238 logger.error("Couldn't get '%s' output.", cmd) 239 logger.error("Bitbake failed with error:\n%s\n", lines) 240 return 241 242 # Parse bitbake -e output 243 for line in lines.split('\n'): 244 self._parse_line(line, image) 245 246 # Make first image a default set of variables 247 if cache: 248 images = [key for key in self if key] 249 if len(images) == 1: 250 self[None] = self[image] 251 252 result = self[image].get(var) 253 if not cache: 254 self.pop(image, None) 255 256 return result 257 258# Create BB_VARS singleton 259BB_VARS = BitbakeVars() 260 261def get_bitbake_var(var, image=None, cache=True): 262 """ 263 Provide old get_bitbake_var API by wrapping 264 get_var method of BB_VARS singleton. 265 """ 266 return BB_VARS.get_var(var, image, cache) 267