1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7 8# This module implements the image creation engine used by 'wic' to 9# create images. The engine parses through the OpenEmbedded kickstart 10# (wks) file specified and generates images that can then be directly 11# written onto media. 12# 13# AUTHORS 14# Tom Zanussi <tom.zanussi (at] linux.intel.com> 15# 16 17import logging 18import os 19import tempfile 20import json 21import subprocess 22import shutil 23import re 24 25from collections import namedtuple, OrderedDict 26 27from wic import WicError 28from wic.filemap import sparse_copy 29from wic.pluginbase import PluginMgr 30from wic.misc import get_bitbake_var, exec_cmd 31 32logger = logging.getLogger('wic') 33 34def verify_build_env(): 35 """ 36 Verify that the build environment is sane. 37 38 Returns True if it is, false otherwise 39 """ 40 if not os.environ.get("BUILDDIR"): 41 raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") 42 43 return True 44 45 46CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts 47SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR 48WIC_DIR = "wic" 49 50def build_canned_image_list(path): 51 layers_path = get_bitbake_var("BBLAYERS") 52 canned_wks_layer_dirs = [] 53 54 if layers_path is not None: 55 for layer_path in layers_path.split(): 56 for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR): 57 cpath = os.path.join(layer_path, wks_path) 58 if os.path.isdir(cpath): 59 canned_wks_layer_dirs.append(cpath) 60 61 cpath = os.path.join(path, CANNED_IMAGE_DIR) 62 canned_wks_layer_dirs.append(cpath) 63 64 return canned_wks_layer_dirs 65 66def find_canned_image(scripts_path, wks_file): 67 """ 68 Find a .wks file with the given name in the canned files dir. 69 70 Return False if not found 71 """ 72 layers_canned_wks_dir = build_canned_image_list(scripts_path) 73 74 for canned_wks_dir in layers_canned_wks_dir: 75 for root, dirs, files in os.walk(canned_wks_dir): 76 for fname in files: 77 if fname.endswith("~") or fname.endswith("#"): 78 continue 79 if ((fname.endswith(".wks") and wks_file + ".wks" == fname) or \ 80 (fname.endswith(".wks.in") and wks_file + ".wks.in" == fname)): 81 fullpath = os.path.join(canned_wks_dir, fname) 82 return fullpath 83 return None 84 85 86def list_canned_images(scripts_path): 87 """ 88 List the .wks files in the canned image dir, minus the extension. 89 """ 90 layers_canned_wks_dir = build_canned_image_list(scripts_path) 91 92 for canned_wks_dir in layers_canned_wks_dir: 93 for root, dirs, files in os.walk(canned_wks_dir): 94 for fname in files: 95 if fname.endswith("~") or fname.endswith("#"): 96 continue 97 if fname.endswith(".wks") or fname.endswith(".wks.in"): 98 fullpath = os.path.join(canned_wks_dir, fname) 99 with open(fullpath) as wks: 100 for line in wks: 101 desc = "" 102 idx = line.find("short-description:") 103 if idx != -1: 104 desc = line[idx + len("short-description:"):].strip() 105 break 106 basename = fname.split('.')[0] 107 print(" %s\t\t%s" % (basename.ljust(30), desc)) 108 109 110def list_canned_image_help(scripts_path, fullpath): 111 """ 112 List the help and params in the specified canned image. 113 """ 114 found = False 115 with open(fullpath) as wks: 116 for line in wks: 117 if not found: 118 idx = line.find("long-description:") 119 if idx != -1: 120 print() 121 print(line[idx + len("long-description:"):].strip()) 122 found = True 123 continue 124 if not line.strip(): 125 break 126 idx = line.find("#") 127 if idx != -1: 128 print(line[idx + len("#:"):].rstrip()) 129 else: 130 break 131 132 133def list_source_plugins(): 134 """ 135 List the available source plugins i.e. plugins available for --source. 136 """ 137 plugins = PluginMgr.get_plugins('source') 138 139 for plugin in plugins: 140 print(" %s" % plugin) 141 142 143def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, 144 native_sysroot, options): 145 """ 146 Create image 147 148 wks_file - user-defined OE kickstart file 149 rootfs_dir - absolute path to the build's /rootfs dir 150 bootimg_dir - absolute path to the build's boot artifacts directory 151 kernel_dir - absolute path to the build's kernel directory 152 native_sysroot - absolute path to the build's native sysroots dir 153 image_output_dir - dirname to create for image 154 options - wic command line options (debug, bmap, etc) 155 156 Normally, the values for the build artifacts values are determined 157 by 'wic -e' from the output of the 'bitbake -e' command given an 158 image name e.g. 'core-image-minimal' and a given machine set in 159 local.conf. If that's the case, the variables get the following 160 values from the output of 'bitbake -e': 161 162 rootfs_dir: IMAGE_ROOTFS 163 kernel_dir: DEPLOY_DIR_IMAGE 164 native_sysroot: STAGING_DIR_NATIVE 165 166 In the above case, bootimg_dir remains unset and the 167 plugin-specific image creation code is responsible for finding the 168 bootimg artifacts. 169 170 In the case where the values are passed in explicitly i.e 'wic -e' 171 is not used but rather the individual 'wic' options are used to 172 explicitly specify these values. 173 """ 174 try: 175 oe_builddir = os.environ["BUILDDIR"] 176 except KeyError: 177 raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") 178 179 if not os.path.exists(options.outdir): 180 os.makedirs(options.outdir) 181 182 pname = options.imager 183 plugin_class = PluginMgr.get_plugins('imager').get(pname) 184 if not plugin_class: 185 raise WicError('Unknown plugin: %s' % pname) 186 187 plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir, 188 native_sysroot, oe_builddir, options) 189 190 plugin.do_create() 191 192 logger.info("The image(s) were created using OE kickstart file:\n %s", wks_file) 193 194 195def wic_list(args, scripts_path): 196 """ 197 Print the list of images or source plugins. 198 """ 199 if args.list_type is None: 200 return False 201 202 if args.list_type == "images": 203 204 list_canned_images(scripts_path) 205 return True 206 elif args.list_type == "source-plugins": 207 list_source_plugins() 208 return True 209 elif len(args.help_for) == 1 and args.help_for[0] == 'help': 210 wks_file = args.list_type 211 fullpath = find_canned_image(scripts_path, wks_file) 212 if not fullpath: 213 raise WicError("No image named %s found, exiting. " 214 "(Use 'wic list images' to list available images, " 215 "or specify a fully-qualified OE kickstart (.wks) " 216 "filename)" % wks_file) 217 218 list_canned_image_help(scripts_path, fullpath) 219 return True 220 221 return False 222 223 224class Disk: 225 def __init__(self, imagepath, native_sysroot, fstypes=('fat', 'ext')): 226 self.imagepath = imagepath 227 self.native_sysroot = native_sysroot 228 self.fstypes = fstypes 229 self._partitions = None 230 self._partimages = {} 231 self._lsector_size = None 232 self._psector_size = None 233 self._ptable_format = None 234 235 # define sector size 236 sector_size_str = get_bitbake_var('WIC_SECTOR_SIZE') 237 if sector_size_str is not None: 238 try: 239 self.sector_size = int(sector_size_str) 240 except ValueError: 241 self.sector_size = None 242 else: 243 self.sector_size = None 244 245 # find parted 246 # read paths from $PATH environment variable 247 # if it fails, use hardcoded paths 248 pathlist = "/bin:/usr/bin:/usr/sbin:/sbin/" 249 try: 250 self.paths = os.environ['PATH'] + ":" + pathlist 251 except KeyError: 252 self.paths = pathlist 253 254 if native_sysroot: 255 for path in pathlist.split(':'): 256 self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) 257 258 self.parted = shutil.which("parted", path=self.paths) 259 if not self.parted: 260 raise WicError("Can't find executable parted") 261 262 self.partitions = self.get_partitions() 263 264 def __del__(self): 265 for path in self._partimages.values(): 266 os.unlink(path) 267 268 def get_partitions(self): 269 if self._partitions is None: 270 self._partitions = OrderedDict() 271 272 if self.sector_size is not None: 273 out = exec_cmd("export PARTED_SECTOR_SIZE=%d; %s -sm %s unit B print" % \ 274 (self.sector_size, self.parted, self.imagepath), True) 275 else: 276 out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath)) 277 278 parttype = namedtuple("Part", "pnum start end size fstype") 279 splitted = out.splitlines() 280 # skip over possible errors in exec_cmd output 281 try: 282 idx =splitted.index("BYT;") 283 except ValueError: 284 raise WicError("Error getting partition information from %s" % (self.parted)) 285 lsector_size, psector_size, self._ptable_format = splitted[idx + 1].split(":")[3:6] 286 self._lsector_size = int(lsector_size) 287 self._psector_size = int(psector_size) 288 for line in splitted[idx + 2:]: 289 pnum, start, end, size, fstype = line.split(':')[:5] 290 partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]), 291 int(size[:-1]), fstype) 292 self._partitions[pnum] = partition 293 294 return self._partitions 295 296 def __getattr__(self, name): 297 """Get path to the executable in a lazy way.""" 298 if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck", 299 "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"): 300 aname = "_%s" % name 301 if aname not in self.__dict__: 302 setattr(self, aname, shutil.which(name, path=self.paths)) 303 if aname not in self.__dict__ or self.__dict__[aname] is None: 304 raise WicError("Can't find executable '{}'".format(name)) 305 return self.__dict__[aname] 306 return self.__dict__[name] 307 308 def _get_part_image(self, pnum): 309 if pnum not in self.partitions: 310 raise WicError("Partition %s is not in the image" % pnum) 311 part = self.partitions[pnum] 312 # check if fstype is supported 313 for fstype in self.fstypes: 314 if part.fstype.startswith(fstype): 315 break 316 else: 317 raise WicError("Not supported fstype: {}".format(part.fstype)) 318 if pnum not in self._partimages: 319 tmpf = tempfile.NamedTemporaryFile(prefix="wic-part") 320 dst_fname = tmpf.name 321 tmpf.close() 322 sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size) 323 self._partimages[pnum] = dst_fname 324 325 return self._partimages[pnum] 326 327 def _put_part_image(self, pnum): 328 """Put partition image into partitioned image.""" 329 sparse_copy(self._partimages[pnum], self.imagepath, 330 seek=self.partitions[pnum].start) 331 332 def dir(self, pnum, path): 333 if pnum not in self.partitions: 334 raise WicError("Partition %s is not in the image" % pnum) 335 336 if self.partitions[pnum].fstype.startswith('ext'): 337 return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs, 338 self._get_part_image(pnum), 339 path), as_shell=True) 340 else: # fat 341 return exec_cmd("{} -i {} ::{}".format(self.mdir, 342 self._get_part_image(pnum), 343 path)) 344 345 def copy(self, src, dest): 346 """Copy partition image into wic image.""" 347 pnum = dest.part if isinstance(src, str) else src.part 348 349 if self.partitions[pnum].fstype.startswith('ext'): 350 if isinstance(src, str): 351 cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ 352 format(os.path.dirname(dest.path), src, os.path.basename(src), 353 self.debugfs, self._get_part_image(pnum)) 354 else: # copy from wic 355 # run both dump and rdump to support both files and directory 356 cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\ 357 format(os.path.dirname(src.path), src.path, 358 dest, src.path, dest, self.debugfs, 359 self._get_part_image(pnum)) 360 else: # fat 361 if isinstance(src, str): 362 cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, 363 self._get_part_image(pnum), 364 src, dest.path) 365 else: 366 cmd = "{} -i {} -snop ::{} {}".format(self.mcopy, 367 self._get_part_image(pnum), 368 src.path, dest) 369 370 exec_cmd(cmd, as_shell=True) 371 self._put_part_image(pnum) 372 373 def remove_ext(self, pnum, path, recursive): 374 """ 375 Remove files/dirs and their contents from the partition. 376 This only applies to ext* partition. 377 """ 378 abs_path = re.sub(r'\/\/+', '/', path) 379 cmd = "{} {} -wR 'rm \"{}\"'".format(self.debugfs, 380 self._get_part_image(pnum), 381 abs_path) 382 out = exec_cmd(cmd , as_shell=True) 383 for line in out.splitlines(): 384 if line.startswith("rm:"): 385 if "file is a directory" in line: 386 if recursive: 387 # loop through content and delete them one by one if 388 # flaged with -r 389 subdirs = iter(self.dir(pnum, abs_path).splitlines()) 390 next(subdirs) 391 for subdir in subdirs: 392 dir = subdir.split(':')[1].split(" ", 1)[1] 393 if not dir == "." and not dir == "..": 394 self.remove_ext(pnum, "%s/%s" % (abs_path, dir), recursive) 395 396 rmdir_out = exec_cmd("{} {} -wR 'rmdir \"{}\"'".format(self.debugfs, 397 self._get_part_image(pnum), 398 abs_path.rstrip('/')) 399 , as_shell=True) 400 401 for rmdir_line in rmdir_out.splitlines(): 402 if "directory not empty" in rmdir_line: 403 raise WicError("Could not complete operation: \n%s \n" 404 "use -r to remove non-empty directory" % rmdir_line) 405 if rmdir_line.startswith("rmdir:"): 406 raise WicError("Could not complete operation: \n%s " 407 "\n%s" % (str(line), rmdir_line)) 408 409 else: 410 raise WicError("Could not complete operation: \n%s " 411 "\nUnable to remove %s" % (str(line), abs_path)) 412 413 def remove(self, pnum, path, recursive): 414 """Remove files/dirs from the partition.""" 415 partimg = self._get_part_image(pnum) 416 if self.partitions[pnum].fstype.startswith('ext'): 417 self.remove_ext(pnum, path, recursive) 418 419 else: # fat 420 cmd = "{} -i {} ::{}".format(self.mdel, partimg, path) 421 try: 422 exec_cmd(cmd) 423 except WicError as err: 424 if "not found" in str(err) or "non empty" in str(err): 425 # mdel outputs 'File ... not found' or 'directory .. non empty" 426 # try to use mdeltree as path could be a directory 427 cmd = "{} -i {} ::{}".format(self.mdeltree, 428 partimg, path) 429 exec_cmd(cmd) 430 else: 431 raise err 432 self._put_part_image(pnum) 433 434 def write(self, target, expand): 435 """Write disk image to the media or file.""" 436 def write_sfdisk_script(outf, parts): 437 for key, val in parts['partitiontable'].items(): 438 if key in ("partitions", "device", "firstlba", "lastlba"): 439 continue 440 if key == "id": 441 key = "label-id" 442 outf.write("{}: {}\n".format(key, val)) 443 outf.write("\n") 444 for part in parts['partitiontable']['partitions']: 445 line = '' 446 for name in ('attrs', 'name', 'size', 'type', 'uuid'): 447 if name == 'size' and part['type'] == 'f': 448 # don't write size for extended partition 449 continue 450 val = part.get(name) 451 if val: 452 line += '{}={}, '.format(name, val) 453 if line: 454 line = line[:-2] # strip ', ' 455 if part.get('bootable'): 456 line += ' ,bootable' 457 outf.write("{}\n".format(line)) 458 outf.flush() 459 460 def read_ptable(path): 461 out = exec_cmd("{} -J {}".format(self.sfdisk, path)) 462 return json.loads(out) 463 464 def write_ptable(parts, target): 465 with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-", mode='w') as outf: 466 write_sfdisk_script(outf, parts) 467 cmd = "{} --no-reread {} < {} ".format(self.sfdisk, target, outf.name) 468 exec_cmd(cmd, as_shell=True) 469 470 if expand is None: 471 sparse_copy(self.imagepath, target) 472 else: 473 # copy first sectors that may contain bootloader 474 sparse_copy(self.imagepath, target, length=2048 * self._lsector_size) 475 476 # copy source partition table to the target 477 parts = read_ptable(self.imagepath) 478 write_ptable(parts, target) 479 480 # get size of unpartitioned space 481 free = None 482 for line in exec_cmd("{} -F {}".format(self.sfdisk, target)).splitlines(): 483 if line.startswith("Unpartitioned space ") and line.endswith("sectors"): 484 free = int(line.split()[-2]) 485 # Align free space to a 2048 sector boundary. YOCTO #12840. 486 free = free - (free % 2048) 487 if free is None: 488 raise WicError("Can't get size of unpartitioned space") 489 490 # calculate expanded partitions sizes 491 sizes = {} 492 num_auto_resize = 0 493 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 494 if num in expand: 495 if expand[num] != 0: # don't resize partition if size is set to 0 496 sectors = expand[num] // self._lsector_size 497 free -= sectors - part['size'] 498 part['size'] = sectors 499 sizes[num] = sectors 500 elif part['type'] != 'f': 501 sizes[num] = -1 502 num_auto_resize += 1 503 504 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 505 if sizes.get(num) == -1: 506 part['size'] += free // num_auto_resize 507 508 # write resized partition table to the target 509 write_ptable(parts, target) 510 511 # read resized partition table 512 parts = read_ptable(target) 513 514 # copy partitions content 515 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 516 pnum = str(num) 517 fstype = self.partitions[pnum].fstype 518 519 # copy unchanged partition 520 if part['size'] == self.partitions[pnum].size // self._lsector_size: 521 logger.info("copying unchanged partition {}".format(pnum)) 522 sparse_copy(self._get_part_image(pnum), target, seek=part['start'] * self._lsector_size) 523 continue 524 525 # resize or re-create partitions 526 if fstype.startswith('ext') or fstype.startswith('fat') or \ 527 fstype.startswith('linux-swap'): 528 529 partfname = None 530 with tempfile.NamedTemporaryFile(prefix="wic-part{}-".format(pnum)) as partf: 531 partfname = partf.name 532 533 if fstype.startswith('ext'): 534 logger.info("resizing ext partition {}".format(pnum)) 535 partimg = self._get_part_image(pnum) 536 sparse_copy(partimg, partfname) 537 exec_cmd("{} -pf {}".format(self.e2fsck, partfname)) 538 exec_cmd("{} {} {}s".format(\ 539 self.resize2fs, partfname, part['size'])) 540 elif fstype.startswith('fat'): 541 logger.info("copying content of the fat partition {}".format(pnum)) 542 with tempfile.TemporaryDirectory(prefix='wic-fatdir-') as tmpdir: 543 # copy content to the temporary directory 544 cmd = "{} -snompi {} :: {}".format(self.mcopy, 545 self._get_part_image(pnum), 546 tmpdir) 547 exec_cmd(cmd) 548 # create new msdos partition 549 label = part.get("name") 550 label_str = "-n {}".format(label) if label else '' 551 552 cmd = "{} {} -C {} {}".format(self.mkdosfs, label_str, partfname, 553 part['size']) 554 exec_cmd(cmd) 555 # copy content from the temporary directory to the new partition 556 cmd = "{} -snompi {} {}/* ::".format(self.mcopy, partfname, tmpdir) 557 exec_cmd(cmd, as_shell=True) 558 elif fstype.startswith('linux-swap'): 559 logger.info("creating swap partition {}".format(pnum)) 560 label = part.get("name") 561 label_str = "-L {}".format(label) if label else '' 562 out = exec_cmd("{} --probe {}".format(self.blkid, self._get_part_image(pnum))) 563 uuid = out[out.index("UUID=\"")+6:out.index("UUID=\"")+42] 564 uuid_str = "-U {}".format(uuid) if uuid else '' 565 with open(partfname, 'w') as sparse: 566 os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size) 567 exec_cmd("{} {} {} {}".format(self.mkswap, label_str, uuid_str, partfname)) 568 sparse_copy(partfname, target, seek=part['start'] * self._lsector_size) 569 os.unlink(partfname) 570 elif part['type'] != 'f': 571 logger.warning("skipping partition {}: unsupported fstype {}".format(pnum, fstype)) 572 573def wic_ls(args, native_sysroot): 574 """List contents of partitioned image or vfat partition.""" 575 disk = Disk(args.path.image, native_sysroot) 576 if not args.path.part: 577 if disk.partitions: 578 print('Num Start End Size Fstype') 579 for part in disk.partitions.values(): 580 print("{:2d} {:12d} {:12d} {:12d} {}".format(\ 581 part.pnum, part.start, part.end, 582 part.size, part.fstype)) 583 else: 584 path = args.path.path or '/' 585 print(disk.dir(args.path.part, path)) 586 587def wic_cp(args, native_sysroot): 588 """ 589 Copy file or directory to/from the vfat/ext partition of 590 partitioned image. 591 """ 592 if isinstance(args.dest, str): 593 disk = Disk(args.src.image, native_sysroot) 594 else: 595 disk = Disk(args.dest.image, native_sysroot) 596 disk.copy(args.src, args.dest) 597 598 599def wic_rm(args, native_sysroot): 600 """ 601 Remove files or directories from the vfat partition of 602 partitioned image. 603 """ 604 disk = Disk(args.path.image, native_sysroot) 605 disk.remove(args.path.part, args.path.path, args.recursive_delete) 606 607def wic_write(args, native_sysroot): 608 """ 609 Write image to a target device. 610 """ 611 disk = Disk(args.image, native_sysroot, ('fat', 'ext', 'linux-swap')) 612 disk.write(args.target, args.expand) 613 614def find_canned(scripts_path, file_name): 615 """ 616 Find a file either by its path or by name in the canned files dir. 617 618 Return None if not found 619 """ 620 if os.path.exists(file_name): 621 return file_name 622 623 layers_canned_wks_dir = build_canned_image_list(scripts_path) 624 for canned_wks_dir in layers_canned_wks_dir: 625 for root, dirs, files in os.walk(canned_wks_dir): 626 for fname in files: 627 if fname == file_name: 628 fullpath = os.path.join(canned_wks_dir, fname) 629 return fullpath 630 631def get_custom_config(boot_file): 632 """ 633 Get the custom configuration to be used for the bootloader. 634 635 Return None if the file can't be found. 636 """ 637 # Get the scripts path of poky 638 scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__)) 639 640 cfg_file = find_canned(scripts_path, boot_file) 641 if cfg_file: 642 with open(cfg_file, "r") as f: 643 config = f.read() 644 return config 645