1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7# This implements the 'direct' imager plugin class for 'wic' 8# 9# AUTHORS 10# Tom Zanussi <tom.zanussi (at] linux.intel.com> 11# 12 13import logging 14import os 15import random 16import shutil 17import tempfile 18import uuid 19 20from time import strftime 21 22from oe.path import copyhardlinktree 23 24from wic import WicError 25from wic.filemap import sparse_copy 26from wic.ksparser import KickStart, KickStartError 27from wic.pluginbase import PluginMgr, ImagerPlugin 28from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd 29 30logger = logging.getLogger('wic') 31 32class DirectPlugin(ImagerPlugin): 33 """ 34 Install a system into a file containing a partitioned disk image. 35 36 An image file is formatted with a partition table, each partition 37 created from a rootfs or other OpenEmbedded build artifact and dd'ed 38 into the virtual disk. The disk image can subsequently be dd'ed onto 39 media and used on actual hardware. 40 """ 41 name = 'direct' 42 43 def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir, 44 native_sysroot, oe_builddir, options): 45 try: 46 self.ks = KickStart(wks_file) 47 except KickStartError as err: 48 raise WicError(str(err)) 49 50 # parse possible 'rootfs=name' items 51 self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' ')) 52 self.bootimg_dir = bootimg_dir 53 self.kernel_dir = kernel_dir 54 self.native_sysroot = native_sysroot 55 self.oe_builddir = oe_builddir 56 57 self.debug = options.debug 58 self.outdir = options.outdir 59 self.compressor = options.compressor 60 self.bmap = options.bmap 61 self.no_fstab_update = options.no_fstab_update 62 self.updated_fstab_path = None 63 64 self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0], 65 strftime("%Y%m%d%H%M")) 66 self.workdir = self.setup_workdir(options.workdir) 67 self._image = None 68 self.ptable_format = self.ks.bootloader.ptable 69 self.parts = self.ks.partitions 70 71 # as a convenience, set source to the boot partition source 72 # instead of forcing it to be set via bootloader --source 73 for part in self.parts: 74 if not self.ks.bootloader.source and part.mountpoint == "/boot": 75 self.ks.bootloader.source = part.source 76 break 77 78 image_path = self._full_path(self.workdir, self.parts[0].disk, "direct") 79 self._image = PartitionedImage(image_path, self.ptable_format, 80 self.parts, self.native_sysroot, 81 options.extra_space) 82 83 def setup_workdir(self, workdir): 84 if workdir: 85 if os.path.exists(workdir): 86 raise WicError("Internal workdir '%s' specified in wic arguments already exists!" % (workdir)) 87 88 os.makedirs(workdir) 89 return workdir 90 else: 91 return tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.') 92 93 def do_create(self): 94 """ 95 Plugin entry point. 96 """ 97 try: 98 self.create() 99 self.assemble() 100 self.finalize() 101 self.print_info() 102 finally: 103 self.cleanup() 104 105 def update_fstab(self, image_rootfs): 106 """Assume partition order same as in wks""" 107 if not image_rootfs: 108 return 109 110 fstab_path = image_rootfs + "/etc/fstab" 111 if not os.path.isfile(fstab_path): 112 return 113 114 with open(fstab_path) as fstab: 115 fstab_lines = fstab.readlines() 116 117 updated = False 118 for part in self.parts: 119 if not part.realnum or not part.mountpoint \ 120 or not part.mountpoint.startswith('/'): 121 continue 122 123 if part.use_uuid: 124 if part.fsuuid: 125 # FAT UUID is different from others 126 if len(part.fsuuid) == 10: 127 device_name = "UUID=%s-%s" % \ 128 (part.fsuuid[2:6], part.fsuuid[6:]) 129 else: 130 device_name = "UUID=%s" % part.fsuuid 131 else: 132 device_name = "PARTUUID=%s" % part.uuid 133 elif part.use_label: 134 device_name = "LABEL=%s" % part.label 135 else: 136 # mmc device partitions are named mmcblk0p1, mmcblk0p2.. 137 prefix = 'p' if part.disk.startswith('mmcblk') else '' 138 device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum) 139 140 opts = part.fsopts if part.fsopts else "defaults" 141 passno = part.fspassno if part.fspassno else "0" 142 line = "\t".join([device_name, part.mountpoint, part.fstype, 143 opts, "0", passno]) + "\n" 144 145 fstab_lines.append(line) 146 updated = True 147 148 if updated: 149 self.updated_fstab_path = os.path.join(self.workdir, "fstab") 150 with open(self.updated_fstab_path, "w") as f: 151 f.writelines(fstab_lines) 152 153 def _full_path(self, path, name, extention): 154 """ Construct full file path to a file we generate. """ 155 return os.path.join(path, "%s-%s.%s" % (self.name, name, extention)) 156 157 # 158 # Actual implemention 159 # 160 def create(self): 161 """ 162 For 'wic', we already have our build artifacts - we just create 163 filesystems from the artifacts directly and combine them into 164 a partitioned image. 165 """ 166 if not self.no_fstab_update: 167 self.update_fstab(self.rootfs_dir.get("ROOTFS_DIR")) 168 169 for part in self.parts: 170 # get rootfs size from bitbake variable if it's not set in .ks file 171 if not part.size: 172 # and if rootfs name is specified for the partition 173 image_name = self.rootfs_dir.get(part.rootfs_dir) 174 if image_name and os.path.sep not in image_name: 175 # Bitbake variable ROOTFS_SIZE is calculated in 176 # Image._get_rootfs_size method from meta/lib/oe/image.py 177 # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, 178 # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE 179 rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name) 180 if rsize_bb: 181 part.size = int(round(float(rsize_bb))) 182 183 self._image.prepare(self) 184 self._image.layout_partitions() 185 self._image.create() 186 187 def assemble(self): 188 """ 189 Assemble partitions into disk image 190 """ 191 self._image.assemble() 192 193 def finalize(self): 194 """ 195 Finalize the disk image. 196 197 For example, prepare the image to be bootable by e.g. 198 creating and installing a bootloader configuration. 199 """ 200 source_plugin = self.ks.bootloader.source 201 disk_name = self.parts[0].disk 202 if source_plugin: 203 plugin = PluginMgr.get_plugins('source')[source_plugin] 204 plugin.do_install_disk(self._image, disk_name, self, self.workdir, 205 self.oe_builddir, self.bootimg_dir, 206 self.kernel_dir, self.native_sysroot) 207 208 full_path = self._image.path 209 # Generate .bmap 210 if self.bmap: 211 logger.debug("Generating bmap file for %s", disk_name) 212 python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3') 213 bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool') 214 exec_native_cmd("%s %s create %s -o %s.bmap" % \ 215 (python, bmaptool, full_path, full_path), self.native_sysroot) 216 # Compress the image 217 if self.compressor: 218 logger.debug("Compressing disk %s with %s", disk_name, self.compressor) 219 exec_cmd("%s %s" % (self.compressor, full_path)) 220 221 def print_info(self): 222 """ 223 Print the image(s) and artifacts used, for the user. 224 """ 225 msg = "The new image(s) can be found here:\n" 226 227 extension = "direct" + {"gzip": ".gz", 228 "bzip2": ".bz2", 229 "xz": ".xz", 230 None: ""}.get(self.compressor) 231 full_path = self._full_path(self.outdir, self.parts[0].disk, extension) 232 msg += ' %s\n\n' % full_path 233 234 msg += 'The following build artifacts were used to create the image(s):\n' 235 for part in self.parts: 236 if part.rootfs_dir is None: 237 continue 238 if part.mountpoint == '/': 239 suffix = ':' 240 else: 241 suffix = '["%s"]:' % (part.mountpoint or part.label) 242 rootdir = part.rootfs_dir 243 msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), rootdir) 244 245 msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir 246 msg += ' KERNEL_DIR: %s\n' % self.kernel_dir 247 msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot 248 249 logger.info(msg) 250 251 @property 252 def rootdev(self): 253 """ 254 Get root device name to use as a 'root' parameter 255 in kernel command line. 256 257 Assume partition order same as in wks 258 """ 259 for part in self.parts: 260 if part.mountpoint == "/": 261 if part.uuid: 262 return "PARTUUID=%s" % part.uuid 263 elif part.label and self.ptable_format != 'msdos': 264 return "PARTLABEL=%s" % part.label 265 else: 266 suffix = 'p' if part.disk.startswith('mmcblk') else '' 267 return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) 268 269 def cleanup(self): 270 if self._image: 271 self._image.cleanup() 272 273 # Move results to the output dir 274 if not os.path.exists(self.outdir): 275 os.makedirs(self.outdir) 276 277 for fname in os.listdir(self.workdir): 278 path = os.path.join(self.workdir, fname) 279 if os.path.isfile(path): 280 shutil.move(path, os.path.join(self.outdir, fname)) 281 282 # remove work directory when it is not in debugging mode 283 if not self.debug: 284 shutil.rmtree(self.workdir, ignore_errors=True) 285 286# Overhead of the MBR partitioning scheme (just one sector) 287MBR_OVERHEAD = 1 288 289# Overhead of the GPT partitioning scheme 290GPT_OVERHEAD = 34 291 292# Size of a sector in bytes 293SECTOR_SIZE = 512 294 295class PartitionedImage(): 296 """ 297 Partitioned image in a file. 298 """ 299 300 def __init__(self, path, ptable_format, partitions, native_sysroot=None, extra_space=0): 301 self.path = path # Path to the image file 302 self.numpart = 0 # Number of allocated partitions 303 self.realpart = 0 # Number of partitions in the partition table 304 self.primary_part_num = 0 # Number of primary partitions (msdos) 305 self.extendedpart = 0 # Create extended partition before this logical partition (msdos) 306 self.extended_size_sec = 0 # Size of exteded partition (msdos) 307 self.logical_part_cnt = 0 # Number of total logical paritions (msdos) 308 self.offset = 0 # Offset of next partition (in sectors) 309 self.min_size = 0 # Minimum required disk size to fit 310 # all partitions (in bytes) 311 self.ptable_format = ptable_format # Partition table format 312 # Disk system identifier 313 self.identifier = random.SystemRandom().randint(1, 0xffffffff) 314 315 self.partitions = partitions 316 self.partimages = [] 317 # Size of a sector used in calculations 318 self.sector_size = SECTOR_SIZE 319 self.native_sysroot = native_sysroot 320 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 321 self.extra_space = extra_space 322 323 # calculate the real partition number, accounting for partitions not 324 # in the partition table and logical partitions 325 realnum = 0 326 for part in self.partitions: 327 if part.no_table: 328 part.realnum = 0 329 else: 330 realnum += 1 331 if self.ptable_format == 'msdos' and realnum > 3 and num_real_partitions > 4: 332 part.realnum = realnum + 1 333 continue 334 part.realnum = realnum 335 336 # generate parition and filesystem UUIDs 337 for part in self.partitions: 338 if not part.uuid and part.use_uuid: 339 if self.ptable_format == 'gpt': 340 part.uuid = str(uuid.uuid4()) 341 else: # msdos partition table 342 part.uuid = '%08x-%02d' % (self.identifier, part.realnum) 343 if not part.fsuuid: 344 if part.fstype == 'vfat' or part.fstype == 'msdos': 345 part.fsuuid = '0x' + str(uuid.uuid4())[:8].upper() 346 else: 347 part.fsuuid = str(uuid.uuid4()) 348 else: 349 #make sure the fsuuid for vfat/msdos align with format 0xYYYYYYYY 350 if part.fstype == 'vfat' or part.fstype == 'msdos': 351 if part.fsuuid.upper().startswith("0X"): 352 part.fsuuid = '0x' + part.fsuuid.upper()[2:].rjust(8,"0") 353 else: 354 part.fsuuid = '0x' + part.fsuuid.upper().rjust(8,"0") 355 356 def prepare(self, imager): 357 """Prepare an image. Call prepare method of all image partitions.""" 358 for part in self.partitions: 359 # need to create the filesystems in order to get their 360 # sizes before we can add them and do the layout. 361 part.prepare(imager, imager.workdir, imager.oe_builddir, 362 imager.rootfs_dir, imager.bootimg_dir, 363 imager.kernel_dir, imager.native_sysroot, 364 imager.updated_fstab_path) 365 366 # Converting kB to sectors for parted 367 part.size_sec = part.disk_size * 1024 // self.sector_size 368 369 def layout_partitions(self): 370 """ Layout the partitions, meaning calculate the position of every 371 partition on the disk. The 'ptable_format' parameter defines the 372 partition table format and may be "msdos". """ 373 374 logger.debug("Assigning %s partitions to disks", self.ptable_format) 375 376 # The number of primary and logical partitions. Extended partition and 377 # partitions not listed in the table are not included. 378 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 379 380 # Go through partitions in the order they are added in .ks file 381 for num in range(len(self.partitions)): 382 part = self.partitions[num] 383 384 if self.ptable_format == 'msdos' and part.part_name: 385 raise WicError("setting custom partition name is not " \ 386 "implemented for msdos partitions") 387 388 if self.ptable_format == 'msdos' and part.part_type: 389 # The --part-type can also be implemented for MBR partitions, 390 # in which case it would map to the 1-byte "partition type" 391 # filed at offset 3 of the partition entry. 392 raise WicError("setting custom partition type is not " \ 393 "implemented for msdos partitions") 394 395 # Get the disk where the partition is located 396 self.numpart += 1 397 if not part.no_table: 398 self.realpart += 1 399 400 if self.numpart == 1: 401 if self.ptable_format == "msdos": 402 overhead = MBR_OVERHEAD 403 elif self.ptable_format == "gpt": 404 overhead = GPT_OVERHEAD 405 406 # Skip one sector required for the partitioning scheme overhead 407 self.offset += overhead 408 409 if self.ptable_format == "msdos": 410 if self.primary_part_num > 3 or \ 411 (self.extendedpart == 0 and self.primary_part_num >= 3 and num_real_partitions > 4): 412 part.type = 'logical' 413 # Reserve a sector for EBR for every logical partition 414 # before alignment is performed. 415 if part.type == 'logical': 416 self.offset += 2 417 418 align_sectors = 0 419 if part.align: 420 # If not first partition and we do have alignment set we need 421 # to align the partition. 422 # FIXME: This leaves a empty spaces to the disk. To fill the 423 # gaps we could enlargea the previous partition? 424 425 # Calc how much the alignment is off. 426 align_sectors = self.offset % (part.align * 1024 // self.sector_size) 427 428 if align_sectors: 429 # If partition is not aligned as required, we need 430 # to move forward to the next alignment point 431 align_sectors = (part.align * 1024 // self.sector_size) - align_sectors 432 433 logger.debug("Realignment for %s%s with %s sectors, original" 434 " offset %s, target alignment is %sK.", 435 part.disk, self.numpart, align_sectors, 436 self.offset, part.align) 437 438 # increase the offset so we actually start the partition on right alignment 439 self.offset += align_sectors 440 441 if part.offset is not None: 442 offset = part.offset // self.sector_size 443 444 if offset * self.sector_size != part.offset: 445 raise WicError("Could not place %s%s at offset %d with sector size %d" % (part.disk, self.numpart, part.offset, self.sector_size)) 446 447 delta = offset - self.offset 448 if delta < 0: 449 raise WicError("Could not place %s%s at offset %d: next free sector is %d (delta: %d)" % (part.disk, self.numpart, part.offset, self.offset, delta)) 450 451 logger.debug("Skipping %d sectors to place %s%s at offset %dK", 452 delta, part.disk, self.numpart, part.offset) 453 454 self.offset = offset 455 456 part.start = self.offset 457 self.offset += part.size_sec 458 459 if not part.no_table: 460 part.num = self.realpart 461 else: 462 part.num = 0 463 464 if self.ptable_format == "msdos" and not part.no_table: 465 if part.type == 'logical': 466 self.logical_part_cnt += 1 467 part.num = self.logical_part_cnt + 4 468 if self.extendedpart == 0: 469 # Create extended partition as a primary partition 470 self.primary_part_num += 1 471 self.extendedpart = part.num 472 else: 473 self.extended_size_sec += align_sectors 474 self.extended_size_sec += part.size_sec + 2 475 else: 476 self.primary_part_num += 1 477 part.num = self.primary_part_num 478 479 logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d " 480 "sectors (%d bytes).", part.mountpoint, part.disk, 481 part.num, part.start, self.offset - 1, part.size_sec, 482 part.size_sec * self.sector_size) 483 484 # Once all the partitions have been layed out, we can calculate the 485 # minumim disk size 486 self.min_size = self.offset 487 if self.ptable_format == "gpt": 488 self.min_size += GPT_OVERHEAD 489 490 self.min_size *= self.sector_size 491 self.min_size += self.extra_space 492 493 def _create_partition(self, device, parttype, fstype, start, size): 494 """ Create a partition on an image described by the 'device' object. """ 495 496 # Start is included to the size so we need to substract one from the end. 497 end = start + size - 1 498 logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors", 499 parttype, start, end, size) 500 501 cmd = "parted -s %s unit s mkpart %s" % (device, parttype) 502 if fstype: 503 cmd += " %s" % fstype 504 cmd += " %d %d" % (start, end) 505 506 return exec_native_cmd(cmd, self.native_sysroot) 507 508 def create(self): 509 logger.debug("Creating sparse file %s", self.path) 510 with open(self.path, 'w') as sparse: 511 os.ftruncate(sparse.fileno(), self.min_size) 512 513 logger.debug("Initializing partition table for %s", self.path) 514 exec_native_cmd("parted -s %s mklabel %s" % 515 (self.path, self.ptable_format), self.native_sysroot) 516 517 logger.debug("Set disk identifier %x", self.identifier) 518 with open(self.path, 'r+b') as img: 519 img.seek(0x1B8) 520 img.write(self.identifier.to_bytes(4, 'little')) 521 522 logger.debug("Creating partitions") 523 524 for part in self.partitions: 525 if part.num == 0: 526 continue 527 528 if self.ptable_format == "msdos" and part.num == self.extendedpart: 529 # Create an extended partition (note: extended 530 # partition is described in MBR and contains all 531 # logical partitions). The logical partitions save a 532 # sector for an EBR just before the start of a 533 # partition. The extended partition must start one 534 # sector before the start of the first logical 535 # partition. This way the first EBR is inside of the 536 # extended partition. Since the extended partitions 537 # starts a sector before the first logical partition, 538 # add a sector at the back, so that there is enough 539 # room for all logical partitions. 540 self._create_partition(self.path, "extended", 541 None, part.start - 2, 542 self.extended_size_sec) 543 544 if part.fstype == "swap": 545 parted_fs_type = "linux-swap" 546 elif part.fstype == "vfat": 547 parted_fs_type = "fat32" 548 elif part.fstype == "msdos": 549 parted_fs_type = "fat16" 550 if not part.system_id: 551 part.system_id = '0x6' # FAT16 552 else: 553 # Type for ext2/ext3/ext4/btrfs 554 parted_fs_type = "ext2" 555 556 # Boot ROM of OMAP boards require vfat boot partition to have an 557 # even number of sectors. 558 if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \ 559 and part.size_sec % 2: 560 logger.debug("Subtracting one sector from '%s' partition to " 561 "get even number of sectors for the partition", 562 part.mountpoint) 563 part.size_sec -= 1 564 565 self._create_partition(self.path, part.type, 566 parted_fs_type, part.start, part.size_sec) 567 568 if part.part_name: 569 logger.debug("partition %d: set name to %s", 570 part.num, part.part_name) 571 exec_native_cmd("sgdisk --change-name=%d:%s %s" % \ 572 (part.num, part.part_name, 573 self.path), self.native_sysroot) 574 575 if part.part_type: 576 logger.debug("partition %d: set type UID to %s", 577 part.num, part.part_type) 578 exec_native_cmd("sgdisk --typecode=%d:%s %s" % \ 579 (part.num, part.part_type, 580 self.path), self.native_sysroot) 581 582 if part.uuid and self.ptable_format == "gpt": 583 logger.debug("partition %d: set UUID to %s", 584 part.num, part.uuid) 585 exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ 586 (part.num, part.uuid, self.path), 587 self.native_sysroot) 588 589 if part.label and self.ptable_format == "gpt": 590 logger.debug("partition %d: set name to %s", 591 part.num, part.label) 592 exec_native_cmd("parted -s %s name %d %s" % \ 593 (self.path, part.num, part.label), 594 self.native_sysroot) 595 596 if part.active: 597 flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot" 598 logger.debug("Set '%s' flag for partition '%s' on disk '%s'", 599 flag_name, part.num, self.path) 600 exec_native_cmd("parted -s %s set %d %s on" % \ 601 (self.path, part.num, flag_name), 602 self.native_sysroot) 603 if part.system_id: 604 exec_native_cmd("sfdisk --part-type %s %s %s" % \ 605 (self.path, part.num, part.system_id), 606 self.native_sysroot) 607 608 def cleanup(self): 609 pass 610 611 def assemble(self): 612 logger.debug("Installing partitions") 613 614 for part in self.partitions: 615 source = part.source_file 616 if source: 617 # install source_file contents into a partition 618 sparse_copy(source, self.path, seek=part.start * self.sector_size) 619 620 logger.debug("Installed %s in partition %d, sectors %d-%d, " 621 "size %d sectors", source, part.num, part.start, 622 part.start + part.size_sec - 1, part.size_sec) 623 624 partimage = self.path + '.p%d' % part.num 625 os.rename(source, partimage) 626 self.partimages.append(partimage) 627