1# ex:ts=4:sw=4:sts=4:et 2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 3# 4# Copyright (c) 2013, Intel Corporation. 5# All rights reserved. 6# 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License version 2 as 9# published by the Free Software Foundation. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License along 17# with this program; if not, write to the Free Software Foundation, Inc., 18# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19# 20# DESCRIPTION 21 22# This module implements the image creation engine used by 'wic' to 23# create images. The engine parses through the OpenEmbedded kickstart 24# (wks) file specified and generates images that can then be directly 25# written onto media. 26# 27# AUTHORS 28# Tom Zanussi <tom.zanussi (at] linux.intel.com> 29# 30 31import logging 32import os 33import tempfile 34import json 35import subprocess 36 37from collections import namedtuple, OrderedDict 38from distutils.spawn import find_executable 39 40from wic import WicError 41from wic.filemap import sparse_copy 42from wic.pluginbase import PluginMgr 43from wic.misc import get_bitbake_var, exec_cmd 44 45logger = logging.getLogger('wic') 46 47def verify_build_env(): 48 """ 49 Verify that the build environment is sane. 50 51 Returns True if it is, false otherwise 52 """ 53 if not os.environ.get("BUILDDIR"): 54 raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") 55 56 return True 57 58 59CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts 60SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR 61WIC_DIR = "wic" 62 63def build_canned_image_list(path): 64 layers_path = get_bitbake_var("BBLAYERS") 65 canned_wks_layer_dirs = [] 66 67 if layers_path is not None: 68 for layer_path in layers_path.split(): 69 for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR): 70 cpath = os.path.join(layer_path, wks_path) 71 if os.path.isdir(cpath): 72 canned_wks_layer_dirs.append(cpath) 73 74 cpath = os.path.join(path, CANNED_IMAGE_DIR) 75 canned_wks_layer_dirs.append(cpath) 76 77 return canned_wks_layer_dirs 78 79def find_canned_image(scripts_path, wks_file): 80 """ 81 Find a .wks file with the given name in the canned files dir. 82 83 Return False if not found 84 """ 85 layers_canned_wks_dir = build_canned_image_list(scripts_path) 86 87 for canned_wks_dir in layers_canned_wks_dir: 88 for root, dirs, files in os.walk(canned_wks_dir): 89 for fname in files: 90 if fname.endswith("~") or fname.endswith("#"): 91 continue 92 if fname.endswith(".wks") and wks_file + ".wks" == fname: 93 fullpath = os.path.join(canned_wks_dir, fname) 94 return fullpath 95 return None 96 97 98def list_canned_images(scripts_path): 99 """ 100 List the .wks files in the canned image dir, minus the extension. 101 """ 102 layers_canned_wks_dir = build_canned_image_list(scripts_path) 103 104 for canned_wks_dir in layers_canned_wks_dir: 105 for root, dirs, files in os.walk(canned_wks_dir): 106 for fname in files: 107 if fname.endswith("~") or fname.endswith("#"): 108 continue 109 if fname.endswith(".wks"): 110 fullpath = os.path.join(canned_wks_dir, fname) 111 with open(fullpath) as wks: 112 for line in wks: 113 desc = "" 114 idx = line.find("short-description:") 115 if idx != -1: 116 desc = line[idx + len("short-description:"):].strip() 117 break 118 basename = os.path.splitext(fname)[0] 119 print(" %s\t\t%s" % (basename.ljust(30), desc)) 120 121 122def list_canned_image_help(scripts_path, fullpath): 123 """ 124 List the help and params in the specified canned image. 125 """ 126 found = False 127 with open(fullpath) as wks: 128 for line in wks: 129 if not found: 130 idx = line.find("long-description:") 131 if idx != -1: 132 print() 133 print(line[idx + len("long-description:"):].strip()) 134 found = True 135 continue 136 if not line.strip(): 137 break 138 idx = line.find("#") 139 if idx != -1: 140 print(line[idx + len("#:"):].rstrip()) 141 else: 142 break 143 144 145def list_source_plugins(): 146 """ 147 List the available source plugins i.e. plugins available for --source. 148 """ 149 plugins = PluginMgr.get_plugins('source') 150 151 for plugin in plugins: 152 print(" %s" % plugin) 153 154 155def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, 156 native_sysroot, options): 157 """ 158 Create image 159 160 wks_file - user-defined OE kickstart file 161 rootfs_dir - absolute path to the build's /rootfs dir 162 bootimg_dir - absolute path to the build's boot artifacts directory 163 kernel_dir - absolute path to the build's kernel directory 164 native_sysroot - absolute path to the build's native sysroots dir 165 image_output_dir - dirname to create for image 166 options - wic command line options (debug, bmap, etc) 167 168 Normally, the values for the build artifacts values are determined 169 by 'wic -e' from the output of the 'bitbake -e' command given an 170 image name e.g. 'core-image-minimal' and a given machine set in 171 local.conf. If that's the case, the variables get the following 172 values from the output of 'bitbake -e': 173 174 rootfs_dir: IMAGE_ROOTFS 175 kernel_dir: DEPLOY_DIR_IMAGE 176 native_sysroot: STAGING_DIR_NATIVE 177 178 In the above case, bootimg_dir remains unset and the 179 plugin-specific image creation code is responsible for finding the 180 bootimg artifacts. 181 182 In the case where the values are passed in explicitly i.e 'wic -e' 183 is not used but rather the individual 'wic' options are used to 184 explicitly specify these values. 185 """ 186 try: 187 oe_builddir = os.environ["BUILDDIR"] 188 except KeyError: 189 raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") 190 191 if not os.path.exists(options.outdir): 192 os.makedirs(options.outdir) 193 194 pname = options.imager 195 plugin_class = PluginMgr.get_plugins('imager').get(pname) 196 if not plugin_class: 197 raise WicError('Unknown plugin: %s' % pname) 198 199 plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir, 200 native_sysroot, oe_builddir, options) 201 202 plugin.do_create() 203 204 logger.info("The image(s) were created using OE kickstart file:\n %s", wks_file) 205 206 207def wic_list(args, scripts_path): 208 """ 209 Print the list of images or source plugins. 210 """ 211 if args.list_type is None: 212 return False 213 214 if args.list_type == "images": 215 216 list_canned_images(scripts_path) 217 return True 218 elif args.list_type == "source-plugins": 219 list_source_plugins() 220 return True 221 elif len(args.help_for) == 1 and args.help_for[0] == 'help': 222 wks_file = args.list_type 223 fullpath = find_canned_image(scripts_path, wks_file) 224 if not fullpath: 225 raise WicError("No image named %s found, exiting. " 226 "(Use 'wic list images' to list available images, " 227 "or specify a fully-qualified OE kickstart (.wks) " 228 "filename)" % wks_file) 229 230 list_canned_image_help(scripts_path, fullpath) 231 return True 232 233 return False 234 235 236class Disk: 237 def __init__(self, imagepath, native_sysroot, fstypes=('fat', 'ext')): 238 self.imagepath = imagepath 239 self.native_sysroot = native_sysroot 240 self.fstypes = fstypes 241 self._partitions = None 242 self._partimages = {} 243 self._lsector_size = None 244 self._psector_size = None 245 self._ptable_format = None 246 247 # find parted 248 self.paths = "/bin:/usr/bin:/usr/sbin:/sbin/" 249 if native_sysroot: 250 for path in self.paths.split(':'): 251 self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) 252 253 self.parted = find_executable("parted", self.paths) 254 if not self.parted: 255 raise WicError("Can't find executable parted") 256 257 self.partitions = self.get_partitions() 258 259 def __del__(self): 260 for path in self._partimages.values(): 261 os.unlink(path) 262 263 def get_partitions(self): 264 if self._partitions is None: 265 self._partitions = OrderedDict() 266 out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath)) 267 parttype = namedtuple("Part", "pnum start end size fstype") 268 splitted = out.splitlines() 269 # skip over possible errors in exec_cmd output 270 try: 271 idx =splitted.index("BYT;") 272 except ValueError: 273 raise WicError("Error getting partition information from %s" % (self.parted)) 274 lsector_size, psector_size, self._ptable_format = splitted[idx + 1].split(":")[3:6] 275 self._lsector_size = int(lsector_size) 276 self._psector_size = int(psector_size) 277 for line in splitted[idx + 2:]: 278 pnum, start, end, size, fstype = line.split(':')[:5] 279 partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]), 280 int(size[:-1]), fstype) 281 self._partitions[pnum] = partition 282 283 return self._partitions 284 285 def __getattr__(self, name): 286 """Get path to the executable in a lazy way.""" 287 if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck", 288 "resize2fs", "mkswap", "mkdosfs", "debugfs"): 289 aname = "_%s" % name 290 if aname not in self.__dict__: 291 setattr(self, aname, find_executable(name, self.paths)) 292 if aname not in self.__dict__ or self.__dict__[aname] is None: 293 raise WicError("Can't find executable '{}'".format(name)) 294 return self.__dict__[aname] 295 return self.__dict__[name] 296 297 def _get_part_image(self, pnum): 298 if pnum not in self.partitions: 299 raise WicError("Partition %s is not in the image") 300 part = self.partitions[pnum] 301 # check if fstype is supported 302 for fstype in self.fstypes: 303 if part.fstype.startswith(fstype): 304 break 305 else: 306 raise WicError("Not supported fstype: {}".format(part.fstype)) 307 if pnum not in self._partimages: 308 tmpf = tempfile.NamedTemporaryFile(prefix="wic-part") 309 dst_fname = tmpf.name 310 tmpf.close() 311 sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size) 312 self._partimages[pnum] = dst_fname 313 314 return self._partimages[pnum] 315 316 def _put_part_image(self, pnum): 317 """Put partition image into partitioned image.""" 318 sparse_copy(self._partimages[pnum], self.imagepath, 319 seek=self.partitions[pnum].start) 320 321 def dir(self, pnum, path): 322 if self.partitions[pnum].fstype.startswith('ext'): 323 return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs, 324 self._get_part_image(pnum), 325 path), as_shell=True) 326 else: # fat 327 return exec_cmd("{} -i {} ::{}".format(self.mdir, 328 self._get_part_image(pnum), 329 path)) 330 331 def copy(self, src, pnum, path): 332 """Copy partition image into wic image.""" 333 if self.partitions[pnum].fstype.startswith('ext'): 334 cmd = "echo -e 'cd {}\nwrite {} {}' | {} -w {}".\ 335 format(path, src, os.path.basename(src), 336 self.debugfs, self._get_part_image(pnum)) 337 else: # fat 338 cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, 339 self._get_part_image(pnum), 340 src, path) 341 exec_cmd(cmd, as_shell=True) 342 self._put_part_image(pnum) 343 344 def remove(self, pnum, path): 345 """Remove files/dirs from the partition.""" 346 partimg = self._get_part_image(pnum) 347 if self.partitions[pnum].fstype.startswith('ext'): 348 cmd = "{} {} -wR 'rm {}'".format(self.debugfs, 349 self._get_part_image(pnum), 350 path) 351 out = exec_cmd(cmd , as_shell=True) 352 for line in out.splitlines(): 353 if line.startswith("rm:"): 354 if "file is a directory" in line: 355 # Try rmdir to see if this is an empty directory. This won't delete 356 # any non empty directory so let user know about any error that this might 357 # generate. 358 print(exec_cmd("{} {} -wR 'rmdir {}'".format(self.debugfs, 359 self._get_part_image(pnum), 360 path), as_shell=True)) 361 else: 362 raise WicError("Could not complete operation: wic %s" % str(line)) 363 else: # fat 364 cmd = "{} -i {} ::{}".format(self.mdel, partimg, path) 365 try: 366 exec_cmd(cmd) 367 except WicError as err: 368 if "not found" in str(err) or "non empty" in str(err): 369 # mdel outputs 'File ... not found' or 'directory .. non empty" 370 # try to use mdeltree as path could be a directory 371 cmd = "{} -i {} ::{}".format(self.mdeltree, 372 partimg, path) 373 exec_cmd(cmd) 374 else: 375 raise err 376 self._put_part_image(pnum) 377 378 def write(self, target, expand): 379 """Write disk image to the media or file.""" 380 def write_sfdisk_script(outf, parts): 381 for key, val in parts['partitiontable'].items(): 382 if key in ("partitions", "device", "firstlba", "lastlba"): 383 continue 384 if key == "id": 385 key = "label-id" 386 outf.write("{}: {}\n".format(key, val)) 387 outf.write("\n") 388 for part in parts['partitiontable']['partitions']: 389 line = '' 390 for name in ('attrs', 'name', 'size', 'type', 'uuid'): 391 if name == 'size' and part['type'] == 'f': 392 # don't write size for extended partition 393 continue 394 val = part.get(name) 395 if val: 396 line += '{}={}, '.format(name, val) 397 if line: 398 line = line[:-2] # strip ', ' 399 if part.get('bootable'): 400 line += ' ,bootable' 401 outf.write("{}\n".format(line)) 402 outf.flush() 403 404 def read_ptable(path): 405 out = exec_cmd("{} -dJ {}".format(self.sfdisk, path)) 406 return json.loads(out) 407 408 def write_ptable(parts, target): 409 with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-", mode='w') as outf: 410 write_sfdisk_script(outf, parts) 411 cmd = "{} --no-reread {} < {} ".format(self.sfdisk, target, outf.name) 412 exec_cmd(cmd, as_shell=True) 413 414 if expand is None: 415 sparse_copy(self.imagepath, target) 416 else: 417 # copy first sectors that may contain bootloader 418 sparse_copy(self.imagepath, target, length=2048 * self._lsector_size) 419 420 # copy source partition table to the target 421 parts = read_ptable(self.imagepath) 422 write_ptable(parts, target) 423 424 # get size of unpartitioned space 425 free = None 426 for line in exec_cmd("{} -F {}".format(self.sfdisk, target)).splitlines(): 427 if line.startswith("Unpartitioned space ") and line.endswith("sectors"): 428 free = int(line.split()[-2]) 429 # Align free space to a 2048 sector boundary. YOCTO #12840. 430 free = free - (free % 2048) 431 if free is None: 432 raise WicError("Can't get size of unpartitioned space") 433 434 # calculate expanded partitions sizes 435 sizes = {} 436 num_auto_resize = 0 437 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 438 if num in expand: 439 if expand[num] != 0: # don't resize partition if size is set to 0 440 sectors = expand[num] // self._lsector_size 441 free -= sectors - part['size'] 442 part['size'] = sectors 443 sizes[num] = sectors 444 elif part['type'] != 'f': 445 sizes[num] = -1 446 num_auto_resize += 1 447 448 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 449 if sizes.get(num) == -1: 450 part['size'] += free // num_auto_resize 451 452 # write resized partition table to the target 453 write_ptable(parts, target) 454 455 # read resized partition table 456 parts = read_ptable(target) 457 458 # copy partitions content 459 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 460 pnum = str(num) 461 fstype = self.partitions[pnum].fstype 462 463 # copy unchanged partition 464 if part['size'] == self.partitions[pnum].size // self._lsector_size: 465 logger.info("copying unchanged partition {}".format(pnum)) 466 sparse_copy(self._get_part_image(pnum), target, seek=part['start'] * self._lsector_size) 467 continue 468 469 # resize or re-create partitions 470 if fstype.startswith('ext') or fstype.startswith('fat') or \ 471 fstype.startswith('linux-swap'): 472 473 partfname = None 474 with tempfile.NamedTemporaryFile(prefix="wic-part{}-".format(pnum)) as partf: 475 partfname = partf.name 476 477 if fstype.startswith('ext'): 478 logger.info("resizing ext partition {}".format(pnum)) 479 partimg = self._get_part_image(pnum) 480 sparse_copy(partimg, partfname) 481 exec_cmd("{} -pf {}".format(self.e2fsck, partfname)) 482 exec_cmd("{} {} {}s".format(\ 483 self.resize2fs, partfname, part['size'])) 484 elif fstype.startswith('fat'): 485 logger.info("copying content of the fat partition {}".format(pnum)) 486 with tempfile.TemporaryDirectory(prefix='wic-fatdir-') as tmpdir: 487 # copy content to the temporary directory 488 cmd = "{} -snompi {} :: {}".format(self.mcopy, 489 self._get_part_image(pnum), 490 tmpdir) 491 exec_cmd(cmd) 492 # create new msdos partition 493 label = part.get("name") 494 label_str = "-n {}".format(label) if label else '' 495 496 cmd = "{} {} -C {} {}".format(self.mkdosfs, label_str, partfname, 497 part['size']) 498 exec_cmd(cmd) 499 # copy content from the temporary directory to the new partition 500 cmd = "{} -snompi {} {}/* ::".format(self.mcopy, partfname, tmpdir) 501 exec_cmd(cmd, as_shell=True) 502 elif fstype.startswith('linux-swap'): 503 logger.info("creating swap partition {}".format(pnum)) 504 label = part.get("name") 505 label_str = "-L {}".format(label) if label else '' 506 uuid = part.get("uuid") 507 uuid_str = "-U {}".format(uuid) if uuid else '' 508 with open(partfname, 'w') as sparse: 509 os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size) 510 exec_cmd("{} {} {} {}".format(self.mkswap, label_str, uuid_str, partfname)) 511 sparse_copy(partfname, target, seek=part['start'] * self._lsector_size) 512 os.unlink(partfname) 513 elif part['type'] != 'f': 514 logger.warning("skipping partition {}: unsupported fstype {}".format(pnum, fstype)) 515 516def wic_ls(args, native_sysroot): 517 """List contents of partitioned image or vfat partition.""" 518 disk = Disk(args.path.image, native_sysroot) 519 if not args.path.part: 520 if disk.partitions: 521 print('Num Start End Size Fstype') 522 for part in disk.partitions.values(): 523 print("{:2d} {:12d} {:12d} {:12d} {}".format(\ 524 part.pnum, part.start, part.end, 525 part.size, part.fstype)) 526 else: 527 path = args.path.path or '/' 528 print(disk.dir(args.path.part, path)) 529 530def wic_cp(args, native_sysroot): 531 """ 532 Copy local file or directory to the vfat partition of 533 partitioned image. 534 """ 535 disk = Disk(args.dest.image, native_sysroot) 536 disk.copy(args.src, args.dest.part, args.dest.path) 537 538def wic_rm(args, native_sysroot): 539 """ 540 Remove files or directories from the vfat partition of 541 partitioned image. 542 """ 543 disk = Disk(args.path.image, native_sysroot) 544 disk.remove(args.path.part, args.path.path) 545 546def wic_write(args, native_sysroot): 547 """ 548 Write image to a target device. 549 """ 550 disk = Disk(args.image, native_sysroot, ('fat', 'ext', 'swap')) 551 disk.write(args.target, args.expand) 552 553def find_canned(scripts_path, file_name): 554 """ 555 Find a file either by its path or by name in the canned files dir. 556 557 Return None if not found 558 """ 559 if os.path.exists(file_name): 560 return file_name 561 562 layers_canned_wks_dir = build_canned_image_list(scripts_path) 563 for canned_wks_dir in layers_canned_wks_dir: 564 for root, dirs, files in os.walk(canned_wks_dir): 565 for fname in files: 566 if fname == file_name: 567 fullpath = os.path.join(canned_wks_dir, fname) 568 return fullpath 569 570def get_custom_config(boot_file): 571 """ 572 Get the custom configuration to be used for the bootloader. 573 574 Return None if the file can't be found. 575 """ 576 # Get the scripts path of poky 577 scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__)) 578 579 cfg_file = find_canned(scripts_path, boot_file) 580 if cfg_file: 581 with open(cfg_file, "r") as f: 582 config = f.read() 583 return config 584