1# ex:ts=4:sw=4:sts=4:et 2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 3# 4# Copyright (c) 2013, Intel Corporation. 5# All rights reserved. 6# 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License version 2 as 9# published by the Free Software Foundation. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License along 17# with this program; if not, write to the Free Software Foundation, Inc., 18# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19# 20# DESCRIPTION 21 22# This module implements the image creation engine used by 'wic' to 23# create images. The engine parses through the OpenEmbedded kickstart 24# (wks) file specified and generates images that can then be directly 25# written onto media. 26# 27# AUTHORS 28# Tom Zanussi <tom.zanussi (at] linux.intel.com> 29# 30 31import logging 32import os 33import tempfile 34import json 35import subprocess 36 37from collections import namedtuple, OrderedDict 38from distutils.spawn import find_executable 39 40from wic import WicError 41from wic.filemap import sparse_copy 42from wic.pluginbase import PluginMgr 43from wic.misc import get_bitbake_var, exec_cmd 44 45logger = logging.getLogger('wic') 46 47def verify_build_env(): 48 """ 49 Verify that the build environment is sane. 50 51 Returns True if it is, false otherwise 52 """ 53 if not os.environ.get("BUILDDIR"): 54 raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") 55 56 return True 57 58 59CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts 60SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR 61WIC_DIR = "wic" 62 63def build_canned_image_list(path): 64 layers_path = get_bitbake_var("BBLAYERS") 65 canned_wks_layer_dirs = [] 66 67 if layers_path is not None: 68 for layer_path in layers_path.split(): 69 for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR): 70 cpath = os.path.join(layer_path, wks_path) 71 if os.path.isdir(cpath): 72 canned_wks_layer_dirs.append(cpath) 73 74 cpath = os.path.join(path, CANNED_IMAGE_DIR) 75 canned_wks_layer_dirs.append(cpath) 76 77 return canned_wks_layer_dirs 78 79def find_canned_image(scripts_path, wks_file): 80 """ 81 Find a .wks file with the given name in the canned files dir. 82 83 Return False if not found 84 """ 85 layers_canned_wks_dir = build_canned_image_list(scripts_path) 86 87 for canned_wks_dir in layers_canned_wks_dir: 88 for root, dirs, files in os.walk(canned_wks_dir): 89 for fname in files: 90 if fname.endswith("~") or fname.endswith("#"): 91 continue 92 if fname.endswith(".wks") and wks_file + ".wks" == fname: 93 fullpath = os.path.join(canned_wks_dir, fname) 94 return fullpath 95 return None 96 97 98def list_canned_images(scripts_path): 99 """ 100 List the .wks files in the canned image dir, minus the extension. 101 """ 102 layers_canned_wks_dir = build_canned_image_list(scripts_path) 103 104 for canned_wks_dir in layers_canned_wks_dir: 105 for root, dirs, files in os.walk(canned_wks_dir): 106 for fname in files: 107 if fname.endswith("~") or fname.endswith("#"): 108 continue 109 if fname.endswith(".wks"): 110 fullpath = os.path.join(canned_wks_dir, fname) 111 with open(fullpath) as wks: 112 for line in wks: 113 desc = "" 114 idx = line.find("short-description:") 115 if idx != -1: 116 desc = line[idx + len("short-description:"):].strip() 117 break 118 basename = os.path.splitext(fname)[0] 119 print(" %s\t\t%s" % (basename.ljust(30), desc)) 120 121 122def list_canned_image_help(scripts_path, fullpath): 123 """ 124 List the help and params in the specified canned image. 125 """ 126 found = False 127 with open(fullpath) as wks: 128 for line in wks: 129 if not found: 130 idx = line.find("long-description:") 131 if idx != -1: 132 print() 133 print(line[idx + len("long-description:"):].strip()) 134 found = True 135 continue 136 if not line.strip(): 137 break 138 idx = line.find("#") 139 if idx != -1: 140 print(line[idx + len("#:"):].rstrip()) 141 else: 142 break 143 144 145def list_source_plugins(): 146 """ 147 List the available source plugins i.e. plugins available for --source. 148 """ 149 plugins = PluginMgr.get_plugins('source') 150 151 for plugin in plugins: 152 print(" %s" % plugin) 153 154 155def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, 156 native_sysroot, options): 157 """ 158 Create image 159 160 wks_file - user-defined OE kickstart file 161 rootfs_dir - absolute path to the build's /rootfs dir 162 bootimg_dir - absolute path to the build's boot artifacts directory 163 kernel_dir - absolute path to the build's kernel directory 164 native_sysroot - absolute path to the build's native sysroots dir 165 image_output_dir - dirname to create for image 166 options - wic command line options (debug, bmap, etc) 167 168 Normally, the values for the build artifacts values are determined 169 by 'wic -e' from the output of the 'bitbake -e' command given an 170 image name e.g. 'core-image-minimal' and a given machine set in 171 local.conf. If that's the case, the variables get the following 172 values from the output of 'bitbake -e': 173 174 rootfs_dir: IMAGE_ROOTFS 175 kernel_dir: DEPLOY_DIR_IMAGE 176 native_sysroot: STAGING_DIR_NATIVE 177 178 In the above case, bootimg_dir remains unset and the 179 plugin-specific image creation code is responsible for finding the 180 bootimg artifacts. 181 182 In the case where the values are passed in explicitly i.e 'wic -e' 183 is not used but rather the individual 'wic' options are used to 184 explicitly specify these values. 185 """ 186 try: 187 oe_builddir = os.environ["BUILDDIR"] 188 except KeyError: 189 raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") 190 191 if not os.path.exists(options.outdir): 192 os.makedirs(options.outdir) 193 194 pname = options.imager 195 plugin_class = PluginMgr.get_plugins('imager').get(pname) 196 if not plugin_class: 197 raise WicError('Unknown plugin: %s' % pname) 198 199 plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir, 200 native_sysroot, oe_builddir, options) 201 202 plugin.do_create() 203 204 logger.info("The image(s) were created using OE kickstart file:\n %s", wks_file) 205 206 207def wic_list(args, scripts_path): 208 """ 209 Print the list of images or source plugins. 210 """ 211 if args.list_type is None: 212 return False 213 214 if args.list_type == "images": 215 216 list_canned_images(scripts_path) 217 return True 218 elif args.list_type == "source-plugins": 219 list_source_plugins() 220 return True 221 elif len(args.help_for) == 1 and args.help_for[0] == 'help': 222 wks_file = args.list_type 223 fullpath = find_canned_image(scripts_path, wks_file) 224 if not fullpath: 225 raise WicError("No image named %s found, exiting. " 226 "(Use 'wic list images' to list available images, " 227 "or specify a fully-qualified OE kickstart (.wks) " 228 "filename)" % wks_file) 229 230 list_canned_image_help(scripts_path, fullpath) 231 return True 232 233 return False 234 235 236class Disk: 237 def __init__(self, imagepath, native_sysroot, fstypes=('fat', 'ext')): 238 self.imagepath = imagepath 239 self.native_sysroot = native_sysroot 240 self.fstypes = fstypes 241 self._partitions = None 242 self._partimages = {} 243 self._lsector_size = None 244 self._psector_size = None 245 self._ptable_format = None 246 247 # find parted 248 # read paths from $PATH environment variable 249 # if it fails, use hardcoded paths 250 pathlist = "/bin:/usr/bin:/usr/sbin:/sbin/" 251 try: 252 self.paths = os.environ['PATH'] + ":" + pathlist 253 except KeyError: 254 self.paths = pathlist 255 256 if native_sysroot: 257 for path in pathlist.split(':'): 258 self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) 259 260 self.parted = find_executable("parted", self.paths) 261 if not self.parted: 262 raise WicError("Can't find executable parted") 263 264 self.partitions = self.get_partitions() 265 266 def __del__(self): 267 for path in self._partimages.values(): 268 os.unlink(path) 269 270 def get_partitions(self): 271 if self._partitions is None: 272 self._partitions = OrderedDict() 273 out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath)) 274 parttype = namedtuple("Part", "pnum start end size fstype") 275 splitted = out.splitlines() 276 # skip over possible errors in exec_cmd output 277 try: 278 idx =splitted.index("BYT;") 279 except ValueError: 280 raise WicError("Error getting partition information from %s" % (self.parted)) 281 lsector_size, psector_size, self._ptable_format = splitted[idx + 1].split(":")[3:6] 282 self._lsector_size = int(lsector_size) 283 self._psector_size = int(psector_size) 284 for line in splitted[idx + 2:]: 285 pnum, start, end, size, fstype = line.split(':')[:5] 286 partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]), 287 int(size[:-1]), fstype) 288 self._partitions[pnum] = partition 289 290 return self._partitions 291 292 def __getattr__(self, name): 293 """Get path to the executable in a lazy way.""" 294 if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck", 295 "resize2fs", "mkswap", "mkdosfs", "debugfs"): 296 aname = "_%s" % name 297 if aname not in self.__dict__: 298 setattr(self, aname, find_executable(name, self.paths)) 299 if aname not in self.__dict__ or self.__dict__[aname] is None: 300 raise WicError("Can't find executable '{}'".format(name)) 301 return self.__dict__[aname] 302 return self.__dict__[name] 303 304 def _get_part_image(self, pnum): 305 if pnum not in self.partitions: 306 raise WicError("Partition %s is not in the image") 307 part = self.partitions[pnum] 308 # check if fstype is supported 309 for fstype in self.fstypes: 310 if part.fstype.startswith(fstype): 311 break 312 else: 313 raise WicError("Not supported fstype: {}".format(part.fstype)) 314 if pnum not in self._partimages: 315 tmpf = tempfile.NamedTemporaryFile(prefix="wic-part") 316 dst_fname = tmpf.name 317 tmpf.close() 318 sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size) 319 self._partimages[pnum] = dst_fname 320 321 return self._partimages[pnum] 322 323 def _put_part_image(self, pnum): 324 """Put partition image into partitioned image.""" 325 sparse_copy(self._partimages[pnum], self.imagepath, 326 seek=self.partitions[pnum].start) 327 328 def dir(self, pnum, path): 329 if self.partitions[pnum].fstype.startswith('ext'): 330 return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs, 331 self._get_part_image(pnum), 332 path), as_shell=True) 333 else: # fat 334 return exec_cmd("{} -i {} ::{}".format(self.mdir, 335 self._get_part_image(pnum), 336 path)) 337 338 def copy(self, src, pnum, path): 339 """Copy partition image into wic image.""" 340 if self.partitions[pnum].fstype.startswith('ext'): 341 cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ 342 format(path, src, os.path.basename(src), 343 self.debugfs, self._get_part_image(pnum)) 344 else: # fat 345 cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, 346 self._get_part_image(pnum), 347 src, path) 348 exec_cmd(cmd, as_shell=True) 349 self._put_part_image(pnum) 350 351 def remove(self, pnum, path): 352 """Remove files/dirs from the partition.""" 353 partimg = self._get_part_image(pnum) 354 if self.partitions[pnum].fstype.startswith('ext'): 355 cmd = "{} {} -wR 'rm {}'".format(self.debugfs, 356 self._get_part_image(pnum), 357 path) 358 out = exec_cmd(cmd , as_shell=True) 359 for line in out.splitlines(): 360 if line.startswith("rm:"): 361 if "file is a directory" in line: 362 # Try rmdir to see if this is an empty directory. This won't delete 363 # any non empty directory so let user know about any error that this might 364 # generate. 365 print(exec_cmd("{} {} -wR 'rmdir {}'".format(self.debugfs, 366 self._get_part_image(pnum), 367 path), as_shell=True)) 368 else: 369 raise WicError("Could not complete operation: wic %s" % str(line)) 370 else: # fat 371 cmd = "{} -i {} ::{}".format(self.mdel, partimg, path) 372 try: 373 exec_cmd(cmd) 374 except WicError as err: 375 if "not found" in str(err) or "non empty" in str(err): 376 # mdel outputs 'File ... not found' or 'directory .. non empty" 377 # try to use mdeltree as path could be a directory 378 cmd = "{} -i {} ::{}".format(self.mdeltree, 379 partimg, path) 380 exec_cmd(cmd) 381 else: 382 raise err 383 self._put_part_image(pnum) 384 385 def write(self, target, expand): 386 """Write disk image to the media or file.""" 387 def write_sfdisk_script(outf, parts): 388 for key, val in parts['partitiontable'].items(): 389 if key in ("partitions", "device", "firstlba", "lastlba"): 390 continue 391 if key == "id": 392 key = "label-id" 393 outf.write("{}: {}\n".format(key, val)) 394 outf.write("\n") 395 for part in parts['partitiontable']['partitions']: 396 line = '' 397 for name in ('attrs', 'name', 'size', 'type', 'uuid'): 398 if name == 'size' and part['type'] == 'f': 399 # don't write size for extended partition 400 continue 401 val = part.get(name) 402 if val: 403 line += '{}={}, '.format(name, val) 404 if line: 405 line = line[:-2] # strip ', ' 406 if part.get('bootable'): 407 line += ' ,bootable' 408 outf.write("{}\n".format(line)) 409 outf.flush() 410 411 def read_ptable(path): 412 out = exec_cmd("{} -dJ {}".format(self.sfdisk, path)) 413 return json.loads(out) 414 415 def write_ptable(parts, target): 416 with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-", mode='w') as outf: 417 write_sfdisk_script(outf, parts) 418 cmd = "{} --no-reread {} < {} ".format(self.sfdisk, target, outf.name) 419 exec_cmd(cmd, as_shell=True) 420 421 if expand is None: 422 sparse_copy(self.imagepath, target) 423 else: 424 # copy first sectors that may contain bootloader 425 sparse_copy(self.imagepath, target, length=2048 * self._lsector_size) 426 427 # copy source partition table to the target 428 parts = read_ptable(self.imagepath) 429 write_ptable(parts, target) 430 431 # get size of unpartitioned space 432 free = None 433 for line in exec_cmd("{} -F {}".format(self.sfdisk, target)).splitlines(): 434 if line.startswith("Unpartitioned space ") and line.endswith("sectors"): 435 free = int(line.split()[-2]) 436 # Align free space to a 2048 sector boundary. YOCTO #12840. 437 free = free - (free % 2048) 438 if free is None: 439 raise WicError("Can't get size of unpartitioned space") 440 441 # calculate expanded partitions sizes 442 sizes = {} 443 num_auto_resize = 0 444 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 445 if num in expand: 446 if expand[num] != 0: # don't resize partition if size is set to 0 447 sectors = expand[num] // self._lsector_size 448 free -= sectors - part['size'] 449 part['size'] = sectors 450 sizes[num] = sectors 451 elif part['type'] != 'f': 452 sizes[num] = -1 453 num_auto_resize += 1 454 455 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 456 if sizes.get(num) == -1: 457 part['size'] += free // num_auto_resize 458 459 # write resized partition table to the target 460 write_ptable(parts, target) 461 462 # read resized partition table 463 parts = read_ptable(target) 464 465 # copy partitions content 466 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 467 pnum = str(num) 468 fstype = self.partitions[pnum].fstype 469 470 # copy unchanged partition 471 if part['size'] == self.partitions[pnum].size // self._lsector_size: 472 logger.info("copying unchanged partition {}".format(pnum)) 473 sparse_copy(self._get_part_image(pnum), target, seek=part['start'] * self._lsector_size) 474 continue 475 476 # resize or re-create partitions 477 if fstype.startswith('ext') or fstype.startswith('fat') or \ 478 fstype.startswith('linux-swap'): 479 480 partfname = None 481 with tempfile.NamedTemporaryFile(prefix="wic-part{}-".format(pnum)) as partf: 482 partfname = partf.name 483 484 if fstype.startswith('ext'): 485 logger.info("resizing ext partition {}".format(pnum)) 486 partimg = self._get_part_image(pnum) 487 sparse_copy(partimg, partfname) 488 exec_cmd("{} -pf {}".format(self.e2fsck, partfname)) 489 exec_cmd("{} {} {}s".format(\ 490 self.resize2fs, partfname, part['size'])) 491 elif fstype.startswith('fat'): 492 logger.info("copying content of the fat partition {}".format(pnum)) 493 with tempfile.TemporaryDirectory(prefix='wic-fatdir-') as tmpdir: 494 # copy content to the temporary directory 495 cmd = "{} -snompi {} :: {}".format(self.mcopy, 496 self._get_part_image(pnum), 497 tmpdir) 498 exec_cmd(cmd) 499 # create new msdos partition 500 label = part.get("name") 501 label_str = "-n {}".format(label) if label else '' 502 503 cmd = "{} {} -C {} {}".format(self.mkdosfs, label_str, partfname, 504 part['size']) 505 exec_cmd(cmd) 506 # copy content from the temporary directory to the new partition 507 cmd = "{} -snompi {} {}/* ::".format(self.mcopy, partfname, tmpdir) 508 exec_cmd(cmd, as_shell=True) 509 elif fstype.startswith('linux-swap'): 510 logger.info("creating swap partition {}".format(pnum)) 511 label = part.get("name") 512 label_str = "-L {}".format(label) if label else '' 513 uuid = part.get("uuid") 514 uuid_str = "-U {}".format(uuid) if uuid else '' 515 with open(partfname, 'w') as sparse: 516 os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size) 517 exec_cmd("{} {} {} {}".format(self.mkswap, label_str, uuid_str, partfname)) 518 sparse_copy(partfname, target, seek=part['start'] * self._lsector_size) 519 os.unlink(partfname) 520 elif part['type'] != 'f': 521 logger.warning("skipping partition {}: unsupported fstype {}".format(pnum, fstype)) 522 523def wic_ls(args, native_sysroot): 524 """List contents of partitioned image or vfat partition.""" 525 disk = Disk(args.path.image, native_sysroot) 526 if not args.path.part: 527 if disk.partitions: 528 print('Num Start End Size Fstype') 529 for part in disk.partitions.values(): 530 print("{:2d} {:12d} {:12d} {:12d} {}".format(\ 531 part.pnum, part.start, part.end, 532 part.size, part.fstype)) 533 else: 534 path = args.path.path or '/' 535 print(disk.dir(args.path.part, path)) 536 537def wic_cp(args, native_sysroot): 538 """ 539 Copy local file or directory to the vfat partition of 540 partitioned image. 541 """ 542 disk = Disk(args.dest.image, native_sysroot) 543 disk.copy(args.src, args.dest.part, args.dest.path) 544 545def wic_rm(args, native_sysroot): 546 """ 547 Remove files or directories from the vfat partition of 548 partitioned image. 549 """ 550 disk = Disk(args.path.image, native_sysroot) 551 disk.remove(args.path.part, args.path.path) 552 553def wic_write(args, native_sysroot): 554 """ 555 Write image to a target device. 556 """ 557 disk = Disk(args.image, native_sysroot, ('fat', 'ext', 'swap')) 558 disk.write(args.target, args.expand) 559 560def find_canned(scripts_path, file_name): 561 """ 562 Find a file either by its path or by name in the canned files dir. 563 564 Return None if not found 565 """ 566 if os.path.exists(file_name): 567 return file_name 568 569 layers_canned_wks_dir = build_canned_image_list(scripts_path) 570 for canned_wks_dir in layers_canned_wks_dir: 571 for root, dirs, files in os.walk(canned_wks_dir): 572 for fname in files: 573 if fname == file_name: 574 fullpath = os.path.join(canned_wks_dir, fname) 575 return fullpath 576 577def get_custom_config(boot_file): 578 """ 579 Get the custom configuration to be used for the bootloader. 580 581 Return None if the file can't be found. 582 """ 583 # Get the scripts path of poky 584 scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__)) 585 586 cfg_file = find_canned(scripts_path, boot_file) 587 if cfg_file: 588 with open(cfg_file, "r") as f: 589 config = f.read() 590 return config 591