1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7 8# This module implements the image creation engine used by 'wic' to 9# create images. The engine parses through the OpenEmbedded kickstart 10# (wks) file specified and generates images that can then be directly 11# written onto media. 12# 13# AUTHORS 14# Tom Zanussi <tom.zanussi (at] linux.intel.com> 15# 16 17import logging 18import os 19import tempfile 20import json 21import subprocess 22import shutil 23import re 24 25from collections import namedtuple, OrderedDict 26 27from wic import WicError 28from wic.filemap import sparse_copy 29from wic.pluginbase import PluginMgr 30from wic.misc import get_bitbake_var, exec_cmd 31 32logger = logging.getLogger('wic') 33 34def verify_build_env(): 35 """ 36 Verify that the build environment is sane. 37 38 Returns True if it is, false otherwise 39 """ 40 if not os.environ.get("BUILDDIR"): 41 raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") 42 43 return True 44 45 46CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts 47SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR 48WIC_DIR = "wic" 49 50def build_canned_image_list(path): 51 layers_path = get_bitbake_var("BBLAYERS") 52 canned_wks_layer_dirs = [] 53 54 if layers_path is not None: 55 for layer_path in layers_path.split(): 56 for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR): 57 cpath = os.path.join(layer_path, wks_path) 58 if os.path.isdir(cpath): 59 canned_wks_layer_dirs.append(cpath) 60 61 cpath = os.path.join(path, CANNED_IMAGE_DIR) 62 canned_wks_layer_dirs.append(cpath) 63 64 return canned_wks_layer_dirs 65 66def find_canned_image(scripts_path, wks_file): 67 """ 68 Find a .wks file with the given name in the canned files dir. 69 70 Return False if not found 71 """ 72 layers_canned_wks_dir = build_canned_image_list(scripts_path) 73 74 for canned_wks_dir in layers_canned_wks_dir: 75 for root, dirs, files in os.walk(canned_wks_dir): 76 for fname in files: 77 if fname.endswith("~") or fname.endswith("#"): 78 continue 79 if ((fname.endswith(".wks") and wks_file + ".wks" == fname) or \ 80 (fname.endswith(".wks.in") and wks_file + ".wks.in" == fname)): 81 fullpath = os.path.join(canned_wks_dir, fname) 82 return fullpath 83 return None 84 85 86def list_canned_images(scripts_path): 87 """ 88 List the .wks files in the canned image dir, minus the extension. 89 """ 90 layers_canned_wks_dir = build_canned_image_list(scripts_path) 91 92 for canned_wks_dir in layers_canned_wks_dir: 93 for root, dirs, files in os.walk(canned_wks_dir): 94 for fname in files: 95 if fname.endswith("~") or fname.endswith("#"): 96 continue 97 if fname.endswith(".wks") or fname.endswith(".wks.in"): 98 fullpath = os.path.join(canned_wks_dir, fname) 99 with open(fullpath) as wks: 100 for line in wks: 101 desc = "" 102 idx = line.find("short-description:") 103 if idx != -1: 104 desc = line[idx + len("short-description:"):].strip() 105 break 106 basename = fname.split('.')[0] 107 print(" %s\t\t%s" % (basename.ljust(30), desc)) 108 109 110def list_canned_image_help(scripts_path, fullpath): 111 """ 112 List the help and params in the specified canned image. 113 """ 114 found = False 115 with open(fullpath) as wks: 116 for line in wks: 117 if not found: 118 idx = line.find("long-description:") 119 if idx != -1: 120 print() 121 print(line[idx + len("long-description:"):].strip()) 122 found = True 123 continue 124 if not line.strip(): 125 break 126 idx = line.find("#") 127 if idx != -1: 128 print(line[idx + len("#:"):].rstrip()) 129 else: 130 break 131 132 133def list_source_plugins(): 134 """ 135 List the available source plugins i.e. plugins available for --source. 136 """ 137 plugins = PluginMgr.get_plugins('source') 138 139 for plugin in plugins: 140 print(" %s" % plugin) 141 142 143def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, 144 native_sysroot, options): 145 """ 146 Create image 147 148 wks_file - user-defined OE kickstart file 149 rootfs_dir - absolute path to the build's /rootfs dir 150 bootimg_dir - absolute path to the build's boot artifacts directory 151 kernel_dir - absolute path to the build's kernel directory 152 native_sysroot - absolute path to the build's native sysroots dir 153 image_output_dir - dirname to create for image 154 options - wic command line options (debug, bmap, etc) 155 156 Normally, the values for the build artifacts values are determined 157 by 'wic -e' from the output of the 'bitbake -e' command given an 158 image name e.g. 'core-image-minimal' and a given machine set in 159 local.conf. If that's the case, the variables get the following 160 values from the output of 'bitbake -e': 161 162 rootfs_dir: IMAGE_ROOTFS 163 kernel_dir: DEPLOY_DIR_IMAGE 164 native_sysroot: STAGING_DIR_NATIVE 165 166 In the above case, bootimg_dir remains unset and the 167 plugin-specific image creation code is responsible for finding the 168 bootimg artifacts. 169 170 In the case where the values are passed in explicitly i.e 'wic -e' 171 is not used but rather the individual 'wic' options are used to 172 explicitly specify these values. 173 """ 174 try: 175 oe_builddir = os.environ["BUILDDIR"] 176 except KeyError: 177 raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") 178 179 if not os.path.exists(options.outdir): 180 os.makedirs(options.outdir) 181 182 pname = options.imager 183 plugin_class = PluginMgr.get_plugins('imager').get(pname) 184 if not plugin_class: 185 raise WicError('Unknown plugin: %s' % pname) 186 187 plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir, 188 native_sysroot, oe_builddir, options) 189 190 plugin.do_create() 191 192 logger.info("The image(s) were created using OE kickstart file:\n %s", wks_file) 193 194 195def wic_list(args, scripts_path): 196 """ 197 Print the list of images or source plugins. 198 """ 199 if args.list_type is None: 200 return False 201 202 if args.list_type == "images": 203 204 list_canned_images(scripts_path) 205 return True 206 elif args.list_type == "source-plugins": 207 list_source_plugins() 208 return True 209 elif len(args.help_for) == 1 and args.help_for[0] == 'help': 210 wks_file = args.list_type 211 fullpath = find_canned_image(scripts_path, wks_file) 212 if not fullpath: 213 raise WicError("No image named %s found, exiting. " 214 "(Use 'wic list images' to list available images, " 215 "or specify a fully-qualified OE kickstart (.wks) " 216 "filename)" % wks_file) 217 218 list_canned_image_help(scripts_path, fullpath) 219 return True 220 221 return False 222 223 224class Disk: 225 def __init__(self, imagepath, native_sysroot, fstypes=('fat', 'ext')): 226 self.imagepath = imagepath 227 self.native_sysroot = native_sysroot 228 self.fstypes = fstypes 229 self._partitions = None 230 self._partimages = {} 231 self._lsector_size = None 232 self._psector_size = None 233 self._ptable_format = None 234 235 # find parted 236 # read paths from $PATH environment variable 237 # if it fails, use hardcoded paths 238 pathlist = "/bin:/usr/bin:/usr/sbin:/sbin/" 239 try: 240 self.paths = os.environ['PATH'] + ":" + pathlist 241 except KeyError: 242 self.paths = pathlist 243 244 if native_sysroot: 245 for path in pathlist.split(':'): 246 self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) 247 248 self.parted = shutil.which("parted", path=self.paths) 249 if not self.parted: 250 raise WicError("Can't find executable parted") 251 252 self.partitions = self.get_partitions() 253 254 def __del__(self): 255 for path in self._partimages.values(): 256 os.unlink(path) 257 258 def get_partitions(self): 259 if self._partitions is None: 260 self._partitions = OrderedDict() 261 out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath)) 262 parttype = namedtuple("Part", "pnum start end size fstype") 263 splitted = out.splitlines() 264 # skip over possible errors in exec_cmd output 265 try: 266 idx =splitted.index("BYT;") 267 except ValueError: 268 raise WicError("Error getting partition information from %s" % (self.parted)) 269 lsector_size, psector_size, self._ptable_format = splitted[idx + 1].split(":")[3:6] 270 self._lsector_size = int(lsector_size) 271 self._psector_size = int(psector_size) 272 for line in splitted[idx + 2:]: 273 pnum, start, end, size, fstype = line.split(':')[:5] 274 partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]), 275 int(size[:-1]), fstype) 276 self._partitions[pnum] = partition 277 278 return self._partitions 279 280 def __getattr__(self, name): 281 """Get path to the executable in a lazy way.""" 282 if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck", 283 "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"): 284 aname = "_%s" % name 285 if aname not in self.__dict__: 286 setattr(self, aname, shutil.which(name, path=self.paths)) 287 if aname not in self.__dict__ or self.__dict__[aname] is None: 288 raise WicError("Can't find executable '{}'".format(name)) 289 return self.__dict__[aname] 290 return self.__dict__[name] 291 292 def _get_part_image(self, pnum): 293 if pnum not in self.partitions: 294 raise WicError("Partition %s is not in the image" % pnum) 295 part = self.partitions[pnum] 296 # check if fstype is supported 297 for fstype in self.fstypes: 298 if part.fstype.startswith(fstype): 299 break 300 else: 301 raise WicError("Not supported fstype: {}".format(part.fstype)) 302 if pnum not in self._partimages: 303 tmpf = tempfile.NamedTemporaryFile(prefix="wic-part") 304 dst_fname = tmpf.name 305 tmpf.close() 306 sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size) 307 self._partimages[pnum] = dst_fname 308 309 return self._partimages[pnum] 310 311 def _put_part_image(self, pnum): 312 """Put partition image into partitioned image.""" 313 sparse_copy(self._partimages[pnum], self.imagepath, 314 seek=self.partitions[pnum].start) 315 316 def dir(self, pnum, path): 317 if pnum not in self.partitions: 318 raise WicError("Partition %s is not in the image" % pnum) 319 320 if self.partitions[pnum].fstype.startswith('ext'): 321 return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs, 322 self._get_part_image(pnum), 323 path), as_shell=True) 324 else: # fat 325 return exec_cmd("{} -i {} ::{}".format(self.mdir, 326 self._get_part_image(pnum), 327 path)) 328 329 def copy(self, src, dest): 330 """Copy partition image into wic image.""" 331 pnum = dest.part if isinstance(src, str) else src.part 332 333 if self.partitions[pnum].fstype.startswith('ext'): 334 if isinstance(src, str): 335 cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ 336 format(os.path.dirname(dest.path), src, os.path.basename(src), 337 self.debugfs, self._get_part_image(pnum)) 338 else: # copy from wic 339 # run both dump and rdump to support both files and directory 340 cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\ 341 format(os.path.dirname(src.path), src.path, 342 dest, src.path, dest, self.debugfs, 343 self._get_part_image(pnum)) 344 else: # fat 345 if isinstance(src, str): 346 cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, 347 self._get_part_image(pnum), 348 src, dest.path) 349 else: 350 cmd = "{} -i {} -snop ::{} {}".format(self.mcopy, 351 self._get_part_image(pnum), 352 src.path, dest) 353 354 exec_cmd(cmd, as_shell=True) 355 self._put_part_image(pnum) 356 357 def remove_ext(self, pnum, path, recursive): 358 """ 359 Remove files/dirs and their contents from the partition. 360 This only applies to ext* partition. 361 """ 362 abs_path = re.sub('\/\/+', '/', path) 363 cmd = "{} {} -wR 'rm \"{}\"'".format(self.debugfs, 364 self._get_part_image(pnum), 365 abs_path) 366 out = exec_cmd(cmd , as_shell=True) 367 for line in out.splitlines(): 368 if line.startswith("rm:"): 369 if "file is a directory" in line: 370 if recursive: 371 # loop through content and delete them one by one if 372 # flaged with -r 373 subdirs = iter(self.dir(pnum, abs_path).splitlines()) 374 next(subdirs) 375 for subdir in subdirs: 376 dir = subdir.split(':')[1].split(" ", 1)[1] 377 if not dir == "." and not dir == "..": 378 self.remove_ext(pnum, "%s/%s" % (abs_path, dir), recursive) 379 380 rmdir_out = exec_cmd("{} {} -wR 'rmdir \"{}\"'".format(self.debugfs, 381 self._get_part_image(pnum), 382 abs_path.rstrip('/')) 383 , as_shell=True) 384 385 for rmdir_line in rmdir_out.splitlines(): 386 if "directory not empty" in rmdir_line: 387 raise WicError("Could not complete operation: \n%s \n" 388 "use -r to remove non-empty directory" % rmdir_line) 389 if rmdir_line.startswith("rmdir:"): 390 raise WicError("Could not complete operation: \n%s " 391 "\n%s" % (str(line), rmdir_line)) 392 393 else: 394 raise WicError("Could not complete operation: \n%s " 395 "\nUnable to remove %s" % (str(line), abs_path)) 396 397 def remove(self, pnum, path, recursive): 398 """Remove files/dirs from the partition.""" 399 partimg = self._get_part_image(pnum) 400 if self.partitions[pnum].fstype.startswith('ext'): 401 self.remove_ext(pnum, path, recursive) 402 403 else: # fat 404 cmd = "{} -i {} ::{}".format(self.mdel, partimg, path) 405 try: 406 exec_cmd(cmd) 407 except WicError as err: 408 if "not found" in str(err) or "non empty" in str(err): 409 # mdel outputs 'File ... not found' or 'directory .. non empty" 410 # try to use mdeltree as path could be a directory 411 cmd = "{} -i {} ::{}".format(self.mdeltree, 412 partimg, path) 413 exec_cmd(cmd) 414 else: 415 raise err 416 self._put_part_image(pnum) 417 418 def write(self, target, expand): 419 """Write disk image to the media or file.""" 420 def write_sfdisk_script(outf, parts): 421 for key, val in parts['partitiontable'].items(): 422 if key in ("partitions", "device", "firstlba", "lastlba"): 423 continue 424 if key == "id": 425 key = "label-id" 426 outf.write("{}: {}\n".format(key, val)) 427 outf.write("\n") 428 for part in parts['partitiontable']['partitions']: 429 line = '' 430 for name in ('attrs', 'name', 'size', 'type', 'uuid'): 431 if name == 'size' and part['type'] == 'f': 432 # don't write size for extended partition 433 continue 434 val = part.get(name) 435 if val: 436 line += '{}={}, '.format(name, val) 437 if line: 438 line = line[:-2] # strip ', ' 439 if part.get('bootable'): 440 line += ' ,bootable' 441 outf.write("{}\n".format(line)) 442 outf.flush() 443 444 def read_ptable(path): 445 out = exec_cmd("{} -J {}".format(self.sfdisk, path)) 446 return json.loads(out) 447 448 def write_ptable(parts, target): 449 with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-", mode='w') as outf: 450 write_sfdisk_script(outf, parts) 451 cmd = "{} --no-reread {} < {} ".format(self.sfdisk, target, outf.name) 452 exec_cmd(cmd, as_shell=True) 453 454 if expand is None: 455 sparse_copy(self.imagepath, target) 456 else: 457 # copy first sectors that may contain bootloader 458 sparse_copy(self.imagepath, target, length=2048 * self._lsector_size) 459 460 # copy source partition table to the target 461 parts = read_ptable(self.imagepath) 462 write_ptable(parts, target) 463 464 # get size of unpartitioned space 465 free = None 466 for line in exec_cmd("{} -F {}".format(self.sfdisk, target)).splitlines(): 467 if line.startswith("Unpartitioned space ") and line.endswith("sectors"): 468 free = int(line.split()[-2]) 469 # Align free space to a 2048 sector boundary. YOCTO #12840. 470 free = free - (free % 2048) 471 if free is None: 472 raise WicError("Can't get size of unpartitioned space") 473 474 # calculate expanded partitions sizes 475 sizes = {} 476 num_auto_resize = 0 477 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 478 if num in expand: 479 if expand[num] != 0: # don't resize partition if size is set to 0 480 sectors = expand[num] // self._lsector_size 481 free -= sectors - part['size'] 482 part['size'] = sectors 483 sizes[num] = sectors 484 elif part['type'] != 'f': 485 sizes[num] = -1 486 num_auto_resize += 1 487 488 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 489 if sizes.get(num) == -1: 490 part['size'] += free // num_auto_resize 491 492 # write resized partition table to the target 493 write_ptable(parts, target) 494 495 # read resized partition table 496 parts = read_ptable(target) 497 498 # copy partitions content 499 for num, part in enumerate(parts['partitiontable']['partitions'], 1): 500 pnum = str(num) 501 fstype = self.partitions[pnum].fstype 502 503 # copy unchanged partition 504 if part['size'] == self.partitions[pnum].size // self._lsector_size: 505 logger.info("copying unchanged partition {}".format(pnum)) 506 sparse_copy(self._get_part_image(pnum), target, seek=part['start'] * self._lsector_size) 507 continue 508 509 # resize or re-create partitions 510 if fstype.startswith('ext') or fstype.startswith('fat') or \ 511 fstype.startswith('linux-swap'): 512 513 partfname = None 514 with tempfile.NamedTemporaryFile(prefix="wic-part{}-".format(pnum)) as partf: 515 partfname = partf.name 516 517 if fstype.startswith('ext'): 518 logger.info("resizing ext partition {}".format(pnum)) 519 partimg = self._get_part_image(pnum) 520 sparse_copy(partimg, partfname) 521 exec_cmd("{} -pf {}".format(self.e2fsck, partfname)) 522 exec_cmd("{} {} {}s".format(\ 523 self.resize2fs, partfname, part['size'])) 524 elif fstype.startswith('fat'): 525 logger.info("copying content of the fat partition {}".format(pnum)) 526 with tempfile.TemporaryDirectory(prefix='wic-fatdir-') as tmpdir: 527 # copy content to the temporary directory 528 cmd = "{} -snompi {} :: {}".format(self.mcopy, 529 self._get_part_image(pnum), 530 tmpdir) 531 exec_cmd(cmd) 532 # create new msdos partition 533 label = part.get("name") 534 label_str = "-n {}".format(label) if label else '' 535 536 cmd = "{} {} -C {} {}".format(self.mkdosfs, label_str, partfname, 537 part['size']) 538 exec_cmd(cmd) 539 # copy content from the temporary directory to the new partition 540 cmd = "{} -snompi {} {}/* ::".format(self.mcopy, partfname, tmpdir) 541 exec_cmd(cmd, as_shell=True) 542 elif fstype.startswith('linux-swap'): 543 logger.info("creating swap partition {}".format(pnum)) 544 label = part.get("name") 545 label_str = "-L {}".format(label) if label else '' 546 out = exec_cmd("{} --probe {}".format(self.blkid, self._get_part_image(pnum))) 547 uuid = out[out.index("UUID=\"")+6:out.index("UUID=\"")+42] 548 uuid_str = "-U {}".format(uuid) if uuid else '' 549 with open(partfname, 'w') as sparse: 550 os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size) 551 exec_cmd("{} {} {} {}".format(self.mkswap, label_str, uuid_str, partfname)) 552 sparse_copy(partfname, target, seek=part['start'] * self._lsector_size) 553 os.unlink(partfname) 554 elif part['type'] != 'f': 555 logger.warning("skipping partition {}: unsupported fstype {}".format(pnum, fstype)) 556 557def wic_ls(args, native_sysroot): 558 """List contents of partitioned image or vfat partition.""" 559 disk = Disk(args.path.image, native_sysroot) 560 if not args.path.part: 561 if disk.partitions: 562 print('Num Start End Size Fstype') 563 for part in disk.partitions.values(): 564 print("{:2d} {:12d} {:12d} {:12d} {}".format(\ 565 part.pnum, part.start, part.end, 566 part.size, part.fstype)) 567 else: 568 path = args.path.path or '/' 569 print(disk.dir(args.path.part, path)) 570 571def wic_cp(args, native_sysroot): 572 """ 573 Copy file or directory to/from the vfat/ext partition of 574 partitioned image. 575 """ 576 if isinstance(args.dest, str): 577 disk = Disk(args.src.image, native_sysroot) 578 else: 579 disk = Disk(args.dest.image, native_sysroot) 580 disk.copy(args.src, args.dest) 581 582 583def wic_rm(args, native_sysroot): 584 """ 585 Remove files or directories from the vfat partition of 586 partitioned image. 587 """ 588 disk = Disk(args.path.image, native_sysroot) 589 disk.remove(args.path.part, args.path.path, args.recursive_delete) 590 591def wic_write(args, native_sysroot): 592 """ 593 Write image to a target device. 594 """ 595 disk = Disk(args.image, native_sysroot, ('fat', 'ext', 'linux-swap')) 596 disk.write(args.target, args.expand) 597 598def find_canned(scripts_path, file_name): 599 """ 600 Find a file either by its path or by name in the canned files dir. 601 602 Return None if not found 603 """ 604 if os.path.exists(file_name): 605 return file_name 606 607 layers_canned_wks_dir = build_canned_image_list(scripts_path) 608 for canned_wks_dir in layers_canned_wks_dir: 609 for root, dirs, files in os.walk(canned_wks_dir): 610 for fname in files: 611 if fname == file_name: 612 fullpath = os.path.join(canned_wks_dir, fname) 613 return fullpath 614 615def get_custom_config(boot_file): 616 """ 617 Get the custom configuration to be used for the bootloader. 618 619 Return None if the file can't be found. 620 """ 621 # Get the scripts path of poky 622 scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__)) 623 624 cfg_file = find_canned(scripts_path, boot_file) 625 if cfg_file: 626 with open(cfg_file, "r") as f: 627 config = f.read() 628 return config 629