1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7# This implements the 'direct' imager plugin class for 'wic' 8# 9# AUTHORS 10# Tom Zanussi <tom.zanussi (at] linux.intel.com> 11# 12 13import logging 14import os 15import random 16import shutil 17import tempfile 18import uuid 19 20from time import strftime 21 22from oe.path import copyhardlinktree 23 24from wic import WicError 25from wic.filemap import sparse_copy 26from wic.ksparser import KickStart, KickStartError 27from wic.pluginbase import PluginMgr, ImagerPlugin 28from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd 29 30logger = logging.getLogger('wic') 31 32class DirectPlugin(ImagerPlugin): 33 """ 34 Install a system into a file containing a partitioned disk image. 35 36 An image file is formatted with a partition table, each partition 37 created from a rootfs or other OpenEmbedded build artifact and dd'ed 38 into the virtual disk. The disk image can subsequently be dd'ed onto 39 media and used on actual hardware. 40 """ 41 name = 'direct' 42 43 def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir, 44 native_sysroot, oe_builddir, options): 45 try: 46 self.ks = KickStart(wks_file) 47 except KickStartError as err: 48 raise WicError(str(err)) 49 50 # parse possible 'rootfs=name' items 51 self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' ')) 52 self.bootimg_dir = bootimg_dir 53 self.kernel_dir = kernel_dir 54 self.native_sysroot = native_sysroot 55 self.oe_builddir = oe_builddir 56 57 self.outdir = options.outdir 58 self.compressor = options.compressor 59 self.bmap = options.bmap 60 self.no_fstab_update = options.no_fstab_update 61 self.original_fstab = None 62 63 self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0], 64 strftime("%Y%m%d%H%M")) 65 self.workdir = tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.') 66 self._image = None 67 self.ptable_format = self.ks.bootloader.ptable 68 self.parts = self.ks.partitions 69 70 # as a convenience, set source to the boot partition source 71 # instead of forcing it to be set via bootloader --source 72 for part in self.parts: 73 if not self.ks.bootloader.source and part.mountpoint == "/boot": 74 self.ks.bootloader.source = part.source 75 break 76 77 image_path = self._full_path(self.workdir, self.parts[0].disk, "direct") 78 self._image = PartitionedImage(image_path, self.ptable_format, 79 self.parts, self.native_sysroot) 80 81 def do_create(self): 82 """ 83 Plugin entry point. 84 """ 85 try: 86 self.create() 87 self.assemble() 88 self.finalize() 89 self.print_info() 90 finally: 91 self.cleanup() 92 93 def _write_fstab(self, image_rootfs): 94 """overriden to generate fstab (temporarily) in rootfs. This is called 95 from _create, make sure it doesn't get called from 96 BaseImage.create() 97 """ 98 if not image_rootfs: 99 return 100 101 fstab_path = image_rootfs + "/etc/fstab" 102 if not os.path.isfile(fstab_path): 103 return 104 105 with open(fstab_path) as fstab: 106 fstab_lines = fstab.readlines() 107 self.original_fstab = fstab_lines.copy() 108 109 if self._update_fstab(fstab_lines, self.parts): 110 with open(fstab_path, "w") as fstab: 111 fstab.writelines(fstab_lines) 112 else: 113 self.original_fstab = None 114 115 def _update_fstab(self, fstab_lines, parts): 116 """Assume partition order same as in wks""" 117 updated = False 118 for part in parts: 119 if not part.realnum or not part.mountpoint \ 120 or part.mountpoint == "/": 121 continue 122 123 if part.use_uuid: 124 if part.fsuuid: 125 # FAT UUID is different from others 126 if len(part.fsuuid) == 10: 127 device_name = "UUID=%s-%s" % \ 128 (part.fsuuid[2:6], part.fsuuid[6:]) 129 else: 130 device_name = "UUID=%s" % part.fsuuid 131 else: 132 device_name = "PARTUUID=%s" % part.uuid 133 elif part.use_label: 134 device_name = "LABEL=%s" % part.label 135 else: 136 # mmc device partitions are named mmcblk0p1, mmcblk0p2.. 137 prefix = 'p' if part.disk.startswith('mmcblk') else '' 138 device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum) 139 140 opts = part.fsopts if part.fsopts else "defaults" 141 line = "\t".join([device_name, part.mountpoint, part.fstype, 142 opts, "0", "0"]) + "\n" 143 144 fstab_lines.append(line) 145 updated = True 146 147 return updated 148 149 def _full_path(self, path, name, extention): 150 """ Construct full file path to a file we generate. """ 151 return os.path.join(path, "%s-%s.%s" % (self.name, name, extention)) 152 153 # 154 # Actual implemention 155 # 156 def create(self): 157 """ 158 For 'wic', we already have our build artifacts - we just create 159 filesystems from the artifacts directly and combine them into 160 a partitioned image. 161 """ 162 if not self.no_fstab_update: 163 self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR")) 164 165 for part in self.parts: 166 # get rootfs size from bitbake variable if it's not set in .ks file 167 if not part.size: 168 # and if rootfs name is specified for the partition 169 image_name = self.rootfs_dir.get(part.rootfs_dir) 170 if image_name and os.path.sep not in image_name: 171 # Bitbake variable ROOTFS_SIZE is calculated in 172 # Image._get_rootfs_size method from meta/lib/oe/image.py 173 # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, 174 # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE 175 rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name) 176 if rsize_bb: 177 part.size = int(round(float(rsize_bb))) 178 179 self._image.prepare(self) 180 self._image.layout_partitions() 181 self._image.create() 182 183 def assemble(self): 184 """ 185 Assemble partitions into disk image 186 """ 187 self._image.assemble() 188 189 def finalize(self): 190 """ 191 Finalize the disk image. 192 193 For example, prepare the image to be bootable by e.g. 194 creating and installing a bootloader configuration. 195 """ 196 source_plugin = self.ks.bootloader.source 197 disk_name = self.parts[0].disk 198 if source_plugin: 199 plugin = PluginMgr.get_plugins('source')[source_plugin] 200 plugin.do_install_disk(self._image, disk_name, self, self.workdir, 201 self.oe_builddir, self.bootimg_dir, 202 self.kernel_dir, self.native_sysroot) 203 204 full_path = self._image.path 205 # Generate .bmap 206 if self.bmap: 207 logger.debug("Generating bmap file for %s", disk_name) 208 python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3') 209 bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool') 210 exec_native_cmd("%s %s create %s -o %s.bmap" % \ 211 (python, bmaptool, full_path, full_path), self.native_sysroot) 212 # Compress the image 213 if self.compressor: 214 logger.debug("Compressing disk %s with %s", disk_name, self.compressor) 215 exec_cmd("%s %s" % (self.compressor, full_path)) 216 217 def print_info(self): 218 """ 219 Print the image(s) and artifacts used, for the user. 220 """ 221 msg = "The new image(s) can be found here:\n" 222 223 extension = "direct" + {"gzip": ".gz", 224 "bzip2": ".bz2", 225 "xz": ".xz", 226 None: ""}.get(self.compressor) 227 full_path = self._full_path(self.outdir, self.parts[0].disk, extension) 228 msg += ' %s\n\n' % full_path 229 230 msg += 'The following build artifacts were used to create the image(s):\n' 231 for part in self.parts: 232 if part.rootfs_dir is None: 233 continue 234 if part.mountpoint == '/': 235 suffix = ':' 236 else: 237 suffix = '["%s"]:' % (part.mountpoint or part.label) 238 rootdir = part.rootfs_dir 239 msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), rootdir) 240 241 msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir 242 msg += ' KERNEL_DIR: %s\n' % self.kernel_dir 243 msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot 244 245 logger.info(msg) 246 247 @property 248 def rootdev(self): 249 """ 250 Get root device name to use as a 'root' parameter 251 in kernel command line. 252 253 Assume partition order same as in wks 254 """ 255 for part in self.parts: 256 if part.mountpoint == "/": 257 if part.uuid: 258 return "PARTUUID=%s" % part.uuid 259 else: 260 suffix = 'p' if part.disk.startswith('mmcblk') else '' 261 return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) 262 263 def cleanup(self): 264 if self._image: 265 self._image.cleanup() 266 267 # Move results to the output dir 268 if not os.path.exists(self.outdir): 269 os.makedirs(self.outdir) 270 271 for fname in os.listdir(self.workdir): 272 path = os.path.join(self.workdir, fname) 273 if os.path.isfile(path): 274 shutil.move(path, os.path.join(self.outdir, fname)) 275 276 #Restore original fstab 277 if self.original_fstab: 278 fstab_path = self.rootfs_dir.get("ROOTFS_DIR") + "/etc/fstab" 279 with open(fstab_path, "w") as fstab: 280 fstab.writelines(self.original_fstab) 281 282 # remove work directory 283 shutil.rmtree(self.workdir, ignore_errors=True) 284 285# Overhead of the MBR partitioning scheme (just one sector) 286MBR_OVERHEAD = 1 287 288# Overhead of the GPT partitioning scheme 289GPT_OVERHEAD = 34 290 291# Size of a sector in bytes 292SECTOR_SIZE = 512 293 294class PartitionedImage(): 295 """ 296 Partitioned image in a file. 297 """ 298 299 def __init__(self, path, ptable_format, partitions, native_sysroot=None): 300 self.path = path # Path to the image file 301 self.numpart = 0 # Number of allocated partitions 302 self.realpart = 0 # Number of partitions in the partition table 303 self.primary_part_num = 0 # Number of primary partitions (msdos) 304 self.extendedpart = 0 # Create extended partition before this logical partition (msdos) 305 self.extended_size_sec = 0 # Size of exteded partition (msdos) 306 self.logical_part_cnt = 0 # Number of total logical paritions (msdos) 307 self.offset = 0 # Offset of next partition (in sectors) 308 self.min_size = 0 # Minimum required disk size to fit 309 # all partitions (in bytes) 310 self.ptable_format = ptable_format # Partition table format 311 # Disk system identifier 312 self.identifier = random.SystemRandom().randint(1, 0xffffffff) 313 314 self.partitions = partitions 315 self.partimages = [] 316 # Size of a sector used in calculations 317 self.sector_size = SECTOR_SIZE 318 self.native_sysroot = native_sysroot 319 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 320 321 # calculate the real partition number, accounting for partitions not 322 # in the partition table and logical partitions 323 realnum = 0 324 for part in self.partitions: 325 if part.no_table: 326 part.realnum = 0 327 else: 328 realnum += 1 329 if self.ptable_format == 'msdos' and realnum > 3 and num_real_partitions > 4: 330 part.realnum = realnum + 1 331 continue 332 part.realnum = realnum 333 334 # generate parition and filesystem UUIDs 335 for part in self.partitions: 336 if not part.uuid and part.use_uuid: 337 if self.ptable_format == 'gpt': 338 part.uuid = str(uuid.uuid4()) 339 else: # msdos partition table 340 part.uuid = '%08x-%02d' % (self.identifier, part.realnum) 341 if not part.fsuuid: 342 if part.fstype == 'vfat' or part.fstype == 'msdos': 343 part.fsuuid = '0x' + str(uuid.uuid4())[:8].upper() 344 else: 345 part.fsuuid = str(uuid.uuid4()) 346 347 def prepare(self, imager): 348 """Prepare an image. Call prepare method of all image partitions.""" 349 for part in self.partitions: 350 # need to create the filesystems in order to get their 351 # sizes before we can add them and do the layout. 352 part.prepare(imager, imager.workdir, imager.oe_builddir, 353 imager.rootfs_dir, imager.bootimg_dir, 354 imager.kernel_dir, imager.native_sysroot) 355 356 # Converting kB to sectors for parted 357 part.size_sec = part.disk_size * 1024 // self.sector_size 358 359 def layout_partitions(self): 360 """ Layout the partitions, meaning calculate the position of every 361 partition on the disk. The 'ptable_format' parameter defines the 362 partition table format and may be "msdos". """ 363 364 logger.debug("Assigning %s partitions to disks", self.ptable_format) 365 366 # The number of primary and logical partitions. Extended partition and 367 # partitions not listed in the table are not included. 368 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 369 370 # Go through partitions in the order they are added in .ks file 371 for num in range(len(self.partitions)): 372 part = self.partitions[num] 373 374 if self.ptable_format == 'msdos' and part.part_name: 375 raise WicError("setting custom partition name is not " \ 376 "implemented for msdos partitions") 377 378 if self.ptable_format == 'msdos' and part.part_type: 379 # The --part-type can also be implemented for MBR partitions, 380 # in which case it would map to the 1-byte "partition type" 381 # filed at offset 3 of the partition entry. 382 raise WicError("setting custom partition type is not " \ 383 "implemented for msdos partitions") 384 385 # Get the disk where the partition is located 386 self.numpart += 1 387 if not part.no_table: 388 self.realpart += 1 389 390 if self.numpart == 1: 391 if self.ptable_format == "msdos": 392 overhead = MBR_OVERHEAD 393 elif self.ptable_format == "gpt": 394 overhead = GPT_OVERHEAD 395 396 # Skip one sector required for the partitioning scheme overhead 397 self.offset += overhead 398 399 if self.ptable_format == "msdos": 400 if self.primary_part_num > 3 or \ 401 (self.extendedpart == 0 and self.primary_part_num >= 3 and num_real_partitions > 4): 402 part.type = 'logical' 403 # Reserve a sector for EBR for every logical partition 404 # before alignment is performed. 405 if part.type == 'logical': 406 self.offset += 2 407 408 align_sectors = 0 409 if part.align: 410 # If not first partition and we do have alignment set we need 411 # to align the partition. 412 # FIXME: This leaves a empty spaces to the disk. To fill the 413 # gaps we could enlargea the previous partition? 414 415 # Calc how much the alignment is off. 416 align_sectors = self.offset % (part.align * 1024 // self.sector_size) 417 418 if align_sectors: 419 # If partition is not aligned as required, we need 420 # to move forward to the next alignment point 421 align_sectors = (part.align * 1024 // self.sector_size) - align_sectors 422 423 logger.debug("Realignment for %s%s with %s sectors, original" 424 " offset %s, target alignment is %sK.", 425 part.disk, self.numpart, align_sectors, 426 self.offset, part.align) 427 428 # increase the offset so we actually start the partition on right alignment 429 self.offset += align_sectors 430 431 if part.offset is not None: 432 offset = part.offset // self.sector_size 433 434 if offset * self.sector_size != part.offset: 435 raise WicError("Could not place %s%s at offset %d with sector size %d" % (part.disk, self.numpart, part.offset, self.sector_size)) 436 437 delta = offset - self.offset 438 if delta < 0: 439 raise WicError("Could not place %s%s at offset %d: next free sector is %d (delta: %d)" % (part.disk, self.numpart, part.offset, self.offset, delta)) 440 441 logger.debug("Skipping %d sectors to place %s%s at offset %dK", 442 delta, part.disk, self.numpart, part.offset) 443 444 self.offset = offset 445 446 part.start = self.offset 447 self.offset += part.size_sec 448 449 if not part.no_table: 450 part.num = self.realpart 451 else: 452 part.num = 0 453 454 if self.ptable_format == "msdos" and not part.no_table: 455 if part.type == 'logical': 456 self.logical_part_cnt += 1 457 part.num = self.logical_part_cnt + 4 458 if self.extendedpart == 0: 459 # Create extended partition as a primary partition 460 self.primary_part_num += 1 461 self.extendedpart = part.num 462 else: 463 self.extended_size_sec += align_sectors 464 self.extended_size_sec += part.size_sec + 2 465 else: 466 self.primary_part_num += 1 467 part.num = self.primary_part_num 468 469 logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d " 470 "sectors (%d bytes).", part.mountpoint, part.disk, 471 part.num, part.start, self.offset - 1, part.size_sec, 472 part.size_sec * self.sector_size) 473 474 # Once all the partitions have been layed out, we can calculate the 475 # minumim disk size 476 self.min_size = self.offset 477 if self.ptable_format == "gpt": 478 self.min_size += GPT_OVERHEAD 479 480 self.min_size *= self.sector_size 481 482 def _create_partition(self, device, parttype, fstype, start, size): 483 """ Create a partition on an image described by the 'device' object. """ 484 485 # Start is included to the size so we need to substract one from the end. 486 end = start + size - 1 487 logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors", 488 parttype, start, end, size) 489 490 cmd = "parted -s %s unit s mkpart %s" % (device, parttype) 491 if fstype: 492 cmd += " %s" % fstype 493 cmd += " %d %d" % (start, end) 494 495 return exec_native_cmd(cmd, self.native_sysroot) 496 497 def create(self): 498 logger.debug("Creating sparse file %s", self.path) 499 with open(self.path, 'w') as sparse: 500 os.ftruncate(sparse.fileno(), self.min_size) 501 502 logger.debug("Initializing partition table for %s", self.path) 503 exec_native_cmd("parted -s %s mklabel %s" % 504 (self.path, self.ptable_format), self.native_sysroot) 505 506 logger.debug("Set disk identifier %x", self.identifier) 507 with open(self.path, 'r+b') as img: 508 img.seek(0x1B8) 509 img.write(self.identifier.to_bytes(4, 'little')) 510 511 logger.debug("Creating partitions") 512 513 for part in self.partitions: 514 if part.num == 0: 515 continue 516 517 if self.ptable_format == "msdos" and part.num == self.extendedpart: 518 # Create an extended partition (note: extended 519 # partition is described in MBR and contains all 520 # logical partitions). The logical partitions save a 521 # sector for an EBR just before the start of a 522 # partition. The extended partition must start one 523 # sector before the start of the first logical 524 # partition. This way the first EBR is inside of the 525 # extended partition. Since the extended partitions 526 # starts a sector before the first logical partition, 527 # add a sector at the back, so that there is enough 528 # room for all logical partitions. 529 self._create_partition(self.path, "extended", 530 None, part.start - 2, 531 self.extended_size_sec) 532 533 if part.fstype == "swap": 534 parted_fs_type = "linux-swap" 535 elif part.fstype == "vfat": 536 parted_fs_type = "fat32" 537 elif part.fstype == "msdos": 538 parted_fs_type = "fat16" 539 if not part.system_id: 540 part.system_id = '0x6' # FAT16 541 else: 542 # Type for ext2/ext3/ext4/btrfs 543 parted_fs_type = "ext2" 544 545 # Boot ROM of OMAP boards require vfat boot partition to have an 546 # even number of sectors. 547 if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \ 548 and part.size_sec % 2: 549 logger.debug("Subtracting one sector from '%s' partition to " 550 "get even number of sectors for the partition", 551 part.mountpoint) 552 part.size_sec -= 1 553 554 self._create_partition(self.path, part.type, 555 parted_fs_type, part.start, part.size_sec) 556 557 if part.part_name: 558 logger.debug("partition %d: set name to %s", 559 part.num, part.part_name) 560 exec_native_cmd("sgdisk --change-name=%d:%s %s" % \ 561 (part.num, part.part_name, 562 self.path), self.native_sysroot) 563 564 if part.part_type: 565 logger.debug("partition %d: set type UID to %s", 566 part.num, part.part_type) 567 exec_native_cmd("sgdisk --typecode=%d:%s %s" % \ 568 (part.num, part.part_type, 569 self.path), self.native_sysroot) 570 571 if part.uuid and self.ptable_format == "gpt": 572 logger.debug("partition %d: set UUID to %s", 573 part.num, part.uuid) 574 exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ 575 (part.num, part.uuid, self.path), 576 self.native_sysroot) 577 578 if part.label and self.ptable_format == "gpt": 579 logger.debug("partition %d: set name to %s", 580 part.num, part.label) 581 exec_native_cmd("parted -s %s name %d %s" % \ 582 (self.path, part.num, part.label), 583 self.native_sysroot) 584 585 if part.active: 586 flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot" 587 logger.debug("Set '%s' flag for partition '%s' on disk '%s'", 588 flag_name, part.num, self.path) 589 exec_native_cmd("parted -s %s set %d %s on" % \ 590 (self.path, part.num, flag_name), 591 self.native_sysroot) 592 if part.system_id: 593 exec_native_cmd("sfdisk --part-type %s %s %s" % \ 594 (self.path, part.num, part.system_id), 595 self.native_sysroot) 596 597 def cleanup(self): 598 pass 599 600 def assemble(self): 601 logger.debug("Installing partitions") 602 603 for part in self.partitions: 604 source = part.source_file 605 if source: 606 # install source_file contents into a partition 607 sparse_copy(source, self.path, seek=part.start * self.sector_size) 608 609 logger.debug("Installed %s in partition %d, sectors %d-%d, " 610 "size %d sectors", source, part.num, part.start, 611 part.start + part.size_sec - 1, part.size_sec) 612 613 partimage = self.path + '.p%d' % part.num 614 os.rename(source, partimage) 615 self.partimages.append(partimage) 616