1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7# This implements the 'direct' imager plugin class for 'wic' 8# 9# AUTHORS 10# Tom Zanussi <tom.zanussi (at] linux.intel.com> 11# 12 13import logging 14import os 15import random 16import shutil 17import tempfile 18import uuid 19 20from time import strftime 21 22from oe.path import copyhardlinktree 23 24from wic import WicError 25from wic.filemap import sparse_copy 26from wic.ksparser import KickStart, KickStartError 27from wic.pluginbase import PluginMgr, ImagerPlugin 28from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd 29 30logger = logging.getLogger('wic') 31 32class DirectPlugin(ImagerPlugin): 33 """ 34 Install a system into a file containing a partitioned disk image. 35 36 An image file is formatted with a partition table, each partition 37 created from a rootfs or other OpenEmbedded build artifact and dd'ed 38 into the virtual disk. The disk image can subsequently be dd'ed onto 39 media and used on actual hardware. 40 """ 41 name = 'direct' 42 43 def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir, 44 native_sysroot, oe_builddir, options): 45 try: 46 self.ks = KickStart(wks_file) 47 except KickStartError as err: 48 raise WicError(str(err)) 49 50 # parse possible 'rootfs=name' items 51 self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' ')) 52 self.bootimg_dir = bootimg_dir 53 self.kernel_dir = kernel_dir 54 self.native_sysroot = native_sysroot 55 self.oe_builddir = oe_builddir 56 57 self.debug = options.debug 58 self.outdir = options.outdir 59 self.compressor = options.compressor 60 self.bmap = options.bmap 61 self.no_fstab_update = options.no_fstab_update 62 self.updated_fstab_path = None 63 64 self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0], 65 strftime("%Y%m%d%H%M")) 66 self.workdir = self.setup_workdir(options.workdir) 67 self._image = None 68 self.ptable_format = self.ks.bootloader.ptable 69 self.parts = self.ks.partitions 70 71 # as a convenience, set source to the boot partition source 72 # instead of forcing it to be set via bootloader --source 73 for part in self.parts: 74 if not self.ks.bootloader.source and part.mountpoint == "/boot": 75 self.ks.bootloader.source = part.source 76 break 77 78 image_path = self._full_path(self.workdir, self.parts[0].disk, "direct") 79 self._image = PartitionedImage(image_path, self.ptable_format, 80 self.parts, self.native_sysroot) 81 82 def setup_workdir(self, workdir): 83 if workdir: 84 if os.path.exists(workdir): 85 raise WicError("Internal workdir '%s' specified in wic arguments already exists!" % (workdir)) 86 87 os.makedirs(workdir) 88 return workdir 89 else: 90 return tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.') 91 92 def do_create(self): 93 """ 94 Plugin entry point. 95 """ 96 try: 97 self.create() 98 self.assemble() 99 self.finalize() 100 self.print_info() 101 finally: 102 self.cleanup() 103 104 def update_fstab(self, image_rootfs): 105 """Assume partition order same as in wks""" 106 if not image_rootfs: 107 return 108 109 fstab_path = image_rootfs + "/etc/fstab" 110 if not os.path.isfile(fstab_path): 111 return 112 113 with open(fstab_path) as fstab: 114 fstab_lines = fstab.readlines() 115 116 updated = False 117 for part in self.parts: 118 if not part.realnum or not part.mountpoint \ 119 or part.mountpoint == "/": 120 continue 121 122 if part.use_uuid: 123 if part.fsuuid: 124 # FAT UUID is different from others 125 if len(part.fsuuid) == 10: 126 device_name = "UUID=%s-%s" % \ 127 (part.fsuuid[2:6], part.fsuuid[6:]) 128 else: 129 device_name = "UUID=%s" % part.fsuuid 130 else: 131 device_name = "PARTUUID=%s" % part.uuid 132 elif part.use_label: 133 device_name = "LABEL=%s" % part.label 134 else: 135 # mmc device partitions are named mmcblk0p1, mmcblk0p2.. 136 prefix = 'p' if part.disk.startswith('mmcblk') else '' 137 device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum) 138 139 opts = part.fsopts if part.fsopts else "defaults" 140 line = "\t".join([device_name, part.mountpoint, part.fstype, 141 opts, "0", "0"]) + "\n" 142 143 fstab_lines.append(line) 144 updated = True 145 146 if updated: 147 self.updated_fstab_path = os.path.join(self.workdir, "fstab") 148 with open(self.updated_fstab_path, "w") as f: 149 f.writelines(fstab_lines) 150 151 def _full_path(self, path, name, extention): 152 """ Construct full file path to a file we generate. """ 153 return os.path.join(path, "%s-%s.%s" % (self.name, name, extention)) 154 155 # 156 # Actual implemention 157 # 158 def create(self): 159 """ 160 For 'wic', we already have our build artifacts - we just create 161 filesystems from the artifacts directly and combine them into 162 a partitioned image. 163 """ 164 if not self.no_fstab_update: 165 self.update_fstab(self.rootfs_dir.get("ROOTFS_DIR")) 166 167 for part in self.parts: 168 # get rootfs size from bitbake variable if it's not set in .ks file 169 if not part.size: 170 # and if rootfs name is specified for the partition 171 image_name = self.rootfs_dir.get(part.rootfs_dir) 172 if image_name and os.path.sep not in image_name: 173 # Bitbake variable ROOTFS_SIZE is calculated in 174 # Image._get_rootfs_size method from meta/lib/oe/image.py 175 # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, 176 # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE 177 rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name) 178 if rsize_bb: 179 part.size = int(round(float(rsize_bb))) 180 181 self._image.prepare(self) 182 self._image.layout_partitions() 183 self._image.create() 184 185 def assemble(self): 186 """ 187 Assemble partitions into disk image 188 """ 189 self._image.assemble() 190 191 def finalize(self): 192 """ 193 Finalize the disk image. 194 195 For example, prepare the image to be bootable by e.g. 196 creating and installing a bootloader configuration. 197 """ 198 source_plugin = self.ks.bootloader.source 199 disk_name = self.parts[0].disk 200 if source_plugin: 201 plugin = PluginMgr.get_plugins('source')[source_plugin] 202 plugin.do_install_disk(self._image, disk_name, self, self.workdir, 203 self.oe_builddir, self.bootimg_dir, 204 self.kernel_dir, self.native_sysroot) 205 206 full_path = self._image.path 207 # Generate .bmap 208 if self.bmap: 209 logger.debug("Generating bmap file for %s", disk_name) 210 python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3') 211 bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool') 212 exec_native_cmd("%s %s create %s -o %s.bmap" % \ 213 (python, bmaptool, full_path, full_path), self.native_sysroot) 214 # Compress the image 215 if self.compressor: 216 logger.debug("Compressing disk %s with %s", disk_name, self.compressor) 217 exec_cmd("%s %s" % (self.compressor, full_path)) 218 219 def print_info(self): 220 """ 221 Print the image(s) and artifacts used, for the user. 222 """ 223 msg = "The new image(s) can be found here:\n" 224 225 extension = "direct" + {"gzip": ".gz", 226 "bzip2": ".bz2", 227 "xz": ".xz", 228 None: ""}.get(self.compressor) 229 full_path = self._full_path(self.outdir, self.parts[0].disk, extension) 230 msg += ' %s\n\n' % full_path 231 232 msg += 'The following build artifacts were used to create the image(s):\n' 233 for part in self.parts: 234 if part.rootfs_dir is None: 235 continue 236 if part.mountpoint == '/': 237 suffix = ':' 238 else: 239 suffix = '["%s"]:' % (part.mountpoint or part.label) 240 rootdir = part.rootfs_dir 241 msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), rootdir) 242 243 msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir 244 msg += ' KERNEL_DIR: %s\n' % self.kernel_dir 245 msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot 246 247 logger.info(msg) 248 249 @property 250 def rootdev(self): 251 """ 252 Get root device name to use as a 'root' parameter 253 in kernel command line. 254 255 Assume partition order same as in wks 256 """ 257 for part in self.parts: 258 if part.mountpoint == "/": 259 if part.uuid: 260 return "PARTUUID=%s" % part.uuid 261 else: 262 suffix = 'p' if part.disk.startswith('mmcblk') else '' 263 return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) 264 265 def cleanup(self): 266 if self._image: 267 self._image.cleanup() 268 269 # Move results to the output dir 270 if not os.path.exists(self.outdir): 271 os.makedirs(self.outdir) 272 273 for fname in os.listdir(self.workdir): 274 path = os.path.join(self.workdir, fname) 275 if os.path.isfile(path): 276 shutil.move(path, os.path.join(self.outdir, fname)) 277 278 # remove work directory when it is not in debugging mode 279 if not self.debug: 280 shutil.rmtree(self.workdir, ignore_errors=True) 281 282# Overhead of the MBR partitioning scheme (just one sector) 283MBR_OVERHEAD = 1 284 285# Overhead of the GPT partitioning scheme 286GPT_OVERHEAD = 34 287 288# Size of a sector in bytes 289SECTOR_SIZE = 512 290 291class PartitionedImage(): 292 """ 293 Partitioned image in a file. 294 """ 295 296 def __init__(self, path, ptable_format, partitions, native_sysroot=None): 297 self.path = path # Path to the image file 298 self.numpart = 0 # Number of allocated partitions 299 self.realpart = 0 # Number of partitions in the partition table 300 self.primary_part_num = 0 # Number of primary partitions (msdos) 301 self.extendedpart = 0 # Create extended partition before this logical partition (msdos) 302 self.extended_size_sec = 0 # Size of exteded partition (msdos) 303 self.logical_part_cnt = 0 # Number of total logical paritions (msdos) 304 self.offset = 0 # Offset of next partition (in sectors) 305 self.min_size = 0 # Minimum required disk size to fit 306 # all partitions (in bytes) 307 self.ptable_format = ptable_format # Partition table format 308 # Disk system identifier 309 self.identifier = random.SystemRandom().randint(1, 0xffffffff) 310 311 self.partitions = partitions 312 self.partimages = [] 313 # Size of a sector used in calculations 314 self.sector_size = SECTOR_SIZE 315 self.native_sysroot = native_sysroot 316 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 317 318 # calculate the real partition number, accounting for partitions not 319 # in the partition table and logical partitions 320 realnum = 0 321 for part in self.partitions: 322 if part.no_table: 323 part.realnum = 0 324 else: 325 realnum += 1 326 if self.ptable_format == 'msdos' and realnum > 3 and num_real_partitions > 4: 327 part.realnum = realnum + 1 328 continue 329 part.realnum = realnum 330 331 # generate parition and filesystem UUIDs 332 for part in self.partitions: 333 if not part.uuid and part.use_uuid: 334 if self.ptable_format == 'gpt': 335 part.uuid = str(uuid.uuid4()) 336 else: # msdos partition table 337 part.uuid = '%08x-%02d' % (self.identifier, part.realnum) 338 if not part.fsuuid: 339 if part.fstype == 'vfat' or part.fstype == 'msdos': 340 part.fsuuid = '0x' + str(uuid.uuid4())[:8].upper() 341 else: 342 part.fsuuid = str(uuid.uuid4()) 343 else: 344 #make sure the fsuuid for vfat/msdos align with format 0xYYYYYYYY 345 if part.fstype == 'vfat' or part.fstype == 'msdos': 346 if part.fsuuid.upper().startswith("0X"): 347 part.fsuuid = '0x' + part.fsuuid.upper()[2:].rjust(8,"0") 348 else: 349 part.fsuuid = '0x' + part.fsuuid.upper().rjust(8,"0") 350 351 def prepare(self, imager): 352 """Prepare an image. Call prepare method of all image partitions.""" 353 for part in self.partitions: 354 # need to create the filesystems in order to get their 355 # sizes before we can add them and do the layout. 356 part.prepare(imager, imager.workdir, imager.oe_builddir, 357 imager.rootfs_dir, imager.bootimg_dir, 358 imager.kernel_dir, imager.native_sysroot, 359 imager.updated_fstab_path) 360 361 # Converting kB to sectors for parted 362 part.size_sec = part.disk_size * 1024 // self.sector_size 363 364 def layout_partitions(self): 365 """ Layout the partitions, meaning calculate the position of every 366 partition on the disk. The 'ptable_format' parameter defines the 367 partition table format and may be "msdos". """ 368 369 logger.debug("Assigning %s partitions to disks", self.ptable_format) 370 371 # The number of primary and logical partitions. Extended partition and 372 # partitions not listed in the table are not included. 373 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 374 375 # Go through partitions in the order they are added in .ks file 376 for num in range(len(self.partitions)): 377 part = self.partitions[num] 378 379 if self.ptable_format == 'msdos' and part.part_name: 380 raise WicError("setting custom partition name is not " \ 381 "implemented for msdos partitions") 382 383 if self.ptable_format == 'msdos' and part.part_type: 384 # The --part-type can also be implemented for MBR partitions, 385 # in which case it would map to the 1-byte "partition type" 386 # filed at offset 3 of the partition entry. 387 raise WicError("setting custom partition type is not " \ 388 "implemented for msdos partitions") 389 390 # Get the disk where the partition is located 391 self.numpart += 1 392 if not part.no_table: 393 self.realpart += 1 394 395 if self.numpart == 1: 396 if self.ptable_format == "msdos": 397 overhead = MBR_OVERHEAD 398 elif self.ptable_format == "gpt": 399 overhead = GPT_OVERHEAD 400 401 # Skip one sector required for the partitioning scheme overhead 402 self.offset += overhead 403 404 if self.ptable_format == "msdos": 405 if self.primary_part_num > 3 or \ 406 (self.extendedpart == 0 and self.primary_part_num >= 3 and num_real_partitions > 4): 407 part.type = 'logical' 408 # Reserve a sector for EBR for every logical partition 409 # before alignment is performed. 410 if part.type == 'logical': 411 self.offset += 2 412 413 align_sectors = 0 414 if part.align: 415 # If not first partition and we do have alignment set we need 416 # to align the partition. 417 # FIXME: This leaves a empty spaces to the disk. To fill the 418 # gaps we could enlargea the previous partition? 419 420 # Calc how much the alignment is off. 421 align_sectors = self.offset % (part.align * 1024 // self.sector_size) 422 423 if align_sectors: 424 # If partition is not aligned as required, we need 425 # to move forward to the next alignment point 426 align_sectors = (part.align * 1024 // self.sector_size) - align_sectors 427 428 logger.debug("Realignment for %s%s with %s sectors, original" 429 " offset %s, target alignment is %sK.", 430 part.disk, self.numpart, align_sectors, 431 self.offset, part.align) 432 433 # increase the offset so we actually start the partition on right alignment 434 self.offset += align_sectors 435 436 if part.offset is not None: 437 offset = part.offset // self.sector_size 438 439 if offset * self.sector_size != part.offset: 440 raise WicError("Could not place %s%s at offset %d with sector size %d" % (part.disk, self.numpart, part.offset, self.sector_size)) 441 442 delta = offset - self.offset 443 if delta < 0: 444 raise WicError("Could not place %s%s at offset %d: next free sector is %d (delta: %d)" % (part.disk, self.numpart, part.offset, self.offset, delta)) 445 446 logger.debug("Skipping %d sectors to place %s%s at offset %dK", 447 delta, part.disk, self.numpart, part.offset) 448 449 self.offset = offset 450 451 part.start = self.offset 452 self.offset += part.size_sec 453 454 if not part.no_table: 455 part.num = self.realpart 456 else: 457 part.num = 0 458 459 if self.ptable_format == "msdos" and not part.no_table: 460 if part.type == 'logical': 461 self.logical_part_cnt += 1 462 part.num = self.logical_part_cnt + 4 463 if self.extendedpart == 0: 464 # Create extended partition as a primary partition 465 self.primary_part_num += 1 466 self.extendedpart = part.num 467 else: 468 self.extended_size_sec += align_sectors 469 self.extended_size_sec += part.size_sec + 2 470 else: 471 self.primary_part_num += 1 472 part.num = self.primary_part_num 473 474 logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d " 475 "sectors (%d bytes).", part.mountpoint, part.disk, 476 part.num, part.start, self.offset - 1, part.size_sec, 477 part.size_sec * self.sector_size) 478 479 # Once all the partitions have been layed out, we can calculate the 480 # minumim disk size 481 self.min_size = self.offset 482 if self.ptable_format == "gpt": 483 self.min_size += GPT_OVERHEAD 484 485 self.min_size *= self.sector_size 486 487 def _create_partition(self, device, parttype, fstype, start, size): 488 """ Create a partition on an image described by the 'device' object. """ 489 490 # Start is included to the size so we need to substract one from the end. 491 end = start + size - 1 492 logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors", 493 parttype, start, end, size) 494 495 cmd = "parted -s %s unit s mkpart %s" % (device, parttype) 496 if fstype: 497 cmd += " %s" % fstype 498 cmd += " %d %d" % (start, end) 499 500 return exec_native_cmd(cmd, self.native_sysroot) 501 502 def create(self): 503 logger.debug("Creating sparse file %s", self.path) 504 with open(self.path, 'w') as sparse: 505 os.ftruncate(sparse.fileno(), self.min_size) 506 507 logger.debug("Initializing partition table for %s", self.path) 508 exec_native_cmd("parted -s %s mklabel %s" % 509 (self.path, self.ptable_format), self.native_sysroot) 510 511 logger.debug("Set disk identifier %x", self.identifier) 512 with open(self.path, 'r+b') as img: 513 img.seek(0x1B8) 514 img.write(self.identifier.to_bytes(4, 'little')) 515 516 logger.debug("Creating partitions") 517 518 for part in self.partitions: 519 if part.num == 0: 520 continue 521 522 if self.ptable_format == "msdos" and part.num == self.extendedpart: 523 # Create an extended partition (note: extended 524 # partition is described in MBR and contains all 525 # logical partitions). The logical partitions save a 526 # sector for an EBR just before the start of a 527 # partition. The extended partition must start one 528 # sector before the start of the first logical 529 # partition. This way the first EBR is inside of the 530 # extended partition. Since the extended partitions 531 # starts a sector before the first logical partition, 532 # add a sector at the back, so that there is enough 533 # room for all logical partitions. 534 self._create_partition(self.path, "extended", 535 None, part.start - 2, 536 self.extended_size_sec) 537 538 if part.fstype == "swap": 539 parted_fs_type = "linux-swap" 540 elif part.fstype == "vfat": 541 parted_fs_type = "fat32" 542 elif part.fstype == "msdos": 543 parted_fs_type = "fat16" 544 if not part.system_id: 545 part.system_id = '0x6' # FAT16 546 else: 547 # Type for ext2/ext3/ext4/btrfs 548 parted_fs_type = "ext2" 549 550 # Boot ROM of OMAP boards require vfat boot partition to have an 551 # even number of sectors. 552 if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \ 553 and part.size_sec % 2: 554 logger.debug("Subtracting one sector from '%s' partition to " 555 "get even number of sectors for the partition", 556 part.mountpoint) 557 part.size_sec -= 1 558 559 self._create_partition(self.path, part.type, 560 parted_fs_type, part.start, part.size_sec) 561 562 if part.part_name: 563 logger.debug("partition %d: set name to %s", 564 part.num, part.part_name) 565 exec_native_cmd("sgdisk --change-name=%d:%s %s" % \ 566 (part.num, part.part_name, 567 self.path), self.native_sysroot) 568 569 if part.part_type: 570 logger.debug("partition %d: set type UID to %s", 571 part.num, part.part_type) 572 exec_native_cmd("sgdisk --typecode=%d:%s %s" % \ 573 (part.num, part.part_type, 574 self.path), self.native_sysroot) 575 576 if part.uuid and self.ptable_format == "gpt": 577 logger.debug("partition %d: set UUID to %s", 578 part.num, part.uuid) 579 exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ 580 (part.num, part.uuid, self.path), 581 self.native_sysroot) 582 583 if part.label and self.ptable_format == "gpt": 584 logger.debug("partition %d: set name to %s", 585 part.num, part.label) 586 exec_native_cmd("parted -s %s name %d %s" % \ 587 (self.path, part.num, part.label), 588 self.native_sysroot) 589 590 if part.active: 591 flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot" 592 logger.debug("Set '%s' flag for partition '%s' on disk '%s'", 593 flag_name, part.num, self.path) 594 exec_native_cmd("parted -s %s set %d %s on" % \ 595 (self.path, part.num, flag_name), 596 self.native_sysroot) 597 if part.system_id: 598 exec_native_cmd("sfdisk --part-type %s %s %s" % \ 599 (self.path, part.num, part.system_id), 600 self.native_sysroot) 601 602 def cleanup(self): 603 pass 604 605 def assemble(self): 606 logger.debug("Installing partitions") 607 608 for part in self.partitions: 609 source = part.source_file 610 if source: 611 # install source_file contents into a partition 612 sparse_copy(source, self.path, seek=part.start * self.sector_size) 613 614 logger.debug("Installed %s in partition %d, sectors %d-%d, " 615 "size %d sectors", source, part.num, part.start, 616 part.start + part.size_sec - 1, part.size_sec) 617 618 partimage = self.path + '.p%d' % part.num 619 os.rename(source, partimage) 620 self.partimages.append(partimage) 621