1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7# This module provides a place to collect various wic-related utils 8# for the OpenEmbedded Image Tools. 9# 10# AUTHORS 11# Tom Zanussi <tom.zanussi (at] linux.intel.com> 12# 13"""Miscellaneous functions.""" 14 15import logging 16import os 17import re 18import subprocess 19 20from collections import defaultdict 21from distutils import spawn 22 23from wic import WicError 24 25logger = logging.getLogger('wic') 26 27# executable -> recipe pairs for exec_native_cmd 28NATIVE_RECIPES = {"bmaptool": "bmap-tools", 29 "dumpe2fs": "e2fsprogs", 30 "grub-mkimage": "grub-efi", 31 "isohybrid": "syslinux", 32 "mcopy": "mtools", 33 "mdel" : "mtools", 34 "mdeltree" : "mtools", 35 "mdir" : "mtools", 36 "mkdosfs": "dosfstools", 37 "mkisofs": "cdrtools", 38 "mkfs.btrfs": "btrfs-tools", 39 "mkfs.ext2": "e2fsprogs", 40 "mkfs.ext3": "e2fsprogs", 41 "mkfs.ext4": "e2fsprogs", 42 "mkfs.vfat": "dosfstools", 43 "mksquashfs": "squashfs-tools", 44 "mkswap": "util-linux", 45 "mmd": "mtools", 46 "parted": "parted", 47 "sfdisk": "util-linux", 48 "sgdisk": "gptfdisk", 49 "syslinux": "syslinux", 50 "tar": "tar" 51 } 52 53def runtool(cmdln_or_args): 54 """ wrapper for most of the subprocess calls 55 input: 56 cmdln_or_args: can be both args and cmdln str (shell=True) 57 return: 58 rc, output 59 """ 60 if isinstance(cmdln_or_args, list): 61 cmd = cmdln_or_args[0] 62 shell = False 63 else: 64 import shlex 65 cmd = shlex.split(cmdln_or_args)[0] 66 shell = True 67 68 sout = subprocess.PIPE 69 serr = subprocess.STDOUT 70 71 try: 72 process = subprocess.Popen(cmdln_or_args, stdout=sout, 73 stderr=serr, shell=shell) 74 sout, serr = process.communicate() 75 # combine stdout and stderr, filter None out and decode 76 out = ''.join([out.decode('utf-8') for out in [sout, serr] if out]) 77 except OSError as err: 78 if err.errno == 2: 79 # [Errno 2] No such file or directory 80 raise WicError('Cannot run command: %s, lost dependency?' % cmd) 81 else: 82 raise # relay 83 84 return process.returncode, out 85 86def _exec_cmd(cmd_and_args, as_shell=False): 87 """ 88 Execute command, catching stderr, stdout 89 90 Need to execute as_shell if the command uses wildcards 91 """ 92 logger.debug("_exec_cmd: %s", cmd_and_args) 93 args = cmd_and_args.split() 94 logger.debug(args) 95 96 if as_shell: 97 ret, out = runtool(cmd_and_args) 98 else: 99 ret, out = runtool(args) 100 out = out.strip() 101 if ret != 0: 102 raise WicError("_exec_cmd: %s returned '%s' instead of 0\noutput: %s" % \ 103 (cmd_and_args, ret, out)) 104 105 logger.debug("_exec_cmd: output for %s (rc = %d): %s", 106 cmd_and_args, ret, out) 107 108 return ret, out 109 110 111def exec_cmd(cmd_and_args, as_shell=False): 112 """ 113 Execute command, return output 114 """ 115 return _exec_cmd(cmd_and_args, as_shell)[1] 116 117def find_executable(cmd, paths): 118 recipe = cmd 119 if recipe in NATIVE_RECIPES: 120 recipe = NATIVE_RECIPES[recipe] 121 provided = get_bitbake_var("ASSUME_PROVIDED") 122 if provided and "%s-native" % recipe in provided: 123 return True 124 125 return spawn.find_executable(cmd, paths) 126 127def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): 128 """ 129 Execute native command, catching stderr, stdout 130 131 Need to execute as_shell if the command uses wildcards 132 133 Always need to execute native commands as_shell 134 """ 135 # The reason -1 is used is because there may be "export" commands. 136 args = cmd_and_args.split(';')[-1].split() 137 logger.debug(args) 138 139 if pseudo: 140 cmd_and_args = pseudo + cmd_and_args 141 142 hosttools_dir = get_bitbake_var("HOSTTOOLS_DIR") 143 144 native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/bin:%s" % \ 145 (native_sysroot, native_sysroot, 146 native_sysroot, native_sysroot, 147 hosttools_dir) 148 149 native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ 150 (native_paths, cmd_and_args) 151 logger.debug("exec_native_cmd: %s", native_cmd_and_args) 152 153 # If the command isn't in the native sysroot say we failed. 154 if find_executable(args[0], native_paths): 155 ret, out = _exec_cmd(native_cmd_and_args, True) 156 else: 157 ret = 127 158 out = "can't find native executable %s in %s" % (args[0], native_paths) 159 160 prog = args[0] 161 # shell command-not-found 162 if ret == 127 \ 163 or (pseudo and ret == 1 and out == "Can't find '%s' in $PATH." % prog): 164 msg = "A native program %s required to build the image "\ 165 "was not found (see details above).\n\n" % prog 166 recipe = NATIVE_RECIPES.get(prog) 167 if recipe: 168 msg += "Please make sure wic-tools have %s-native in its DEPENDS, "\ 169 "build it with 'bitbake wic-tools' and try again.\n" % recipe 170 else: 171 msg += "Wic failed to find a recipe to build native %s. Please "\ 172 "file a bug against wic.\n" % prog 173 raise WicError(msg) 174 175 return ret, out 176 177BOOTDD_EXTRA_SPACE = 16384 178 179class BitbakeVars(defaultdict): 180 """ 181 Container for Bitbake variables. 182 """ 183 def __init__(self): 184 defaultdict.__init__(self, dict) 185 186 # default_image and vars_dir attributes should be set from outside 187 self.default_image = None 188 self.vars_dir = None 189 190 def _parse_line(self, line, image, matcher=re.compile(r"^([a-zA-Z0-9\-_+./~]+)=(.*)")): 191 """ 192 Parse one line from bitbake -e output or from .env file. 193 Put result key-value pair into the storage. 194 """ 195 if "=" not in line: 196 return 197 match = matcher.match(line) 198 if not match: 199 return 200 key, val = match.groups() 201 self[image][key] = val.strip('"') 202 203 def get_var(self, var, image=None, cache=True): 204 """ 205 Get bitbake variable from 'bitbake -e' output or from .env file. 206 This is a lazy method, i.e. it runs bitbake or parses file only when 207 only when variable is requested. It also caches results. 208 """ 209 if not image: 210 image = self.default_image 211 212 if image not in self: 213 if image and self.vars_dir: 214 fname = os.path.join(self.vars_dir, image + '.env') 215 if os.path.isfile(fname): 216 # parse .env file 217 with open(fname) as varsfile: 218 for line in varsfile: 219 self._parse_line(line, image) 220 else: 221 print("Couldn't get bitbake variable from %s." % fname) 222 print("File %s doesn't exist." % fname) 223 return 224 else: 225 # Get bitbake -e output 226 cmd = "bitbake -e" 227 if image: 228 cmd += " %s" % image 229 230 log_level = logger.getEffectiveLevel() 231 logger.setLevel(logging.INFO) 232 ret, lines = _exec_cmd(cmd) 233 logger.setLevel(log_level) 234 235 if ret: 236 logger.error("Couldn't get '%s' output.", cmd) 237 logger.error("Bitbake failed with error:\n%s\n", lines) 238 return 239 240 # Parse bitbake -e output 241 for line in lines.split('\n'): 242 self._parse_line(line, image) 243 244 # Make first image a default set of variables 245 if cache: 246 images = [key for key in self if key] 247 if len(images) == 1: 248 self[None] = self[image] 249 250 result = self[image].get(var) 251 if not cache: 252 self.pop(image, None) 253 254 return result 255 256# Create BB_VARS singleton 257BB_VARS = BitbakeVars() 258 259def get_bitbake_var(var, image=None, cache=True): 260 """ 261 Provide old get_bitbake_var API by wrapping 262 get_var method of BB_VARS singleton. 263 """ 264 return BB_VARS.get_var(var, image, cache) 265