1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7# This implements the 'direct' imager plugin class for 'wic' 8# 9# AUTHORS 10# Tom Zanussi <tom.zanussi (at] linux.intel.com> 11# 12 13import logging 14import os 15import random 16import shutil 17import tempfile 18import uuid 19 20from time import strftime 21 22from oe.path import copyhardlinktree 23 24from wic import WicError 25from wic.filemap import sparse_copy 26from wic.ksparser import KickStart, KickStartError 27from wic.pluginbase import PluginMgr, ImagerPlugin 28from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd 29 30logger = logging.getLogger('wic') 31 32class DirectPlugin(ImagerPlugin): 33 """ 34 Install a system into a file containing a partitioned disk image. 35 36 An image file is formatted with a partition table, each partition 37 created from a rootfs or other OpenEmbedded build artifact and dd'ed 38 into the virtual disk. The disk image can subsequently be dd'ed onto 39 media and used on actual hardware. 40 """ 41 name = 'direct' 42 43 def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir, 44 native_sysroot, oe_builddir, options): 45 try: 46 self.ks = KickStart(wks_file) 47 except KickStartError as err: 48 raise WicError(str(err)) 49 50 # parse possible 'rootfs=name' items 51 self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' ')) 52 self.bootimg_dir = bootimg_dir 53 self.kernel_dir = kernel_dir 54 self.native_sysroot = native_sysroot 55 self.oe_builddir = oe_builddir 56 57 self.debug = options.debug 58 self.outdir = options.outdir 59 self.compressor = options.compressor 60 self.bmap = options.bmap 61 self.no_fstab_update = options.no_fstab_update 62 self.updated_fstab_path = None 63 64 self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0], 65 strftime("%Y%m%d%H%M")) 66 self.workdir = self.setup_workdir(options.workdir) 67 self._image = None 68 self.ptable_format = self.ks.bootloader.ptable 69 self.parts = self.ks.partitions 70 71 # as a convenience, set source to the boot partition source 72 # instead of forcing it to be set via bootloader --source 73 for part in self.parts: 74 if not self.ks.bootloader.source and part.mountpoint == "/boot": 75 self.ks.bootloader.source = part.source 76 break 77 78 image_path = self._full_path(self.workdir, self.parts[0].disk, "direct") 79 self._image = PartitionedImage(image_path, self.ptable_format, 80 self.parts, self.native_sysroot, 81 options.extra_space) 82 83 def setup_workdir(self, workdir): 84 if workdir: 85 if os.path.exists(workdir): 86 raise WicError("Internal workdir '%s' specified in wic arguments already exists!" % (workdir)) 87 88 os.makedirs(workdir) 89 return workdir 90 else: 91 return tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.') 92 93 def do_create(self): 94 """ 95 Plugin entry point. 96 """ 97 try: 98 self.create() 99 self.assemble() 100 self.finalize() 101 self.print_info() 102 finally: 103 self.cleanup() 104 105 def update_fstab(self, image_rootfs): 106 """Assume partition order same as in wks""" 107 if not image_rootfs: 108 return 109 110 fstab_path = image_rootfs + "/etc/fstab" 111 if not os.path.isfile(fstab_path): 112 return 113 114 with open(fstab_path) as fstab: 115 fstab_lines = fstab.readlines() 116 117 updated = False 118 for part in self.parts: 119 if not part.realnum or not part.mountpoint \ 120 or part.mountpoint == "/" or not (part.mountpoint.startswith('/') or part.mountpoint == "swap"): 121 continue 122 123 if part.use_uuid: 124 if part.fsuuid: 125 # FAT UUID is different from others 126 if len(part.fsuuid) == 10: 127 device_name = "UUID=%s-%s" % \ 128 (part.fsuuid[2:6], part.fsuuid[6:]) 129 else: 130 device_name = "UUID=%s" % part.fsuuid 131 else: 132 device_name = "PARTUUID=%s" % part.uuid 133 elif part.use_label: 134 device_name = "LABEL=%s" % part.label 135 else: 136 # mmc device partitions are named mmcblk0p1, mmcblk0p2.. 137 prefix = 'p' if part.disk.startswith('mmcblk') else '' 138 device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum) 139 140 opts = part.fsopts if part.fsopts else "defaults" 141 passno = part.fspassno if part.fspassno else "0" 142 line = "\t".join([device_name, part.mountpoint, part.fstype, 143 opts, "0", passno]) + "\n" 144 145 fstab_lines.append(line) 146 updated = True 147 148 if updated: 149 self.updated_fstab_path = os.path.join(self.workdir, "fstab") 150 with open(self.updated_fstab_path, "w") as f: 151 f.writelines(fstab_lines) 152 if os.getenv('SOURCE_DATE_EPOCH'): 153 fstab_time = int(os.getenv('SOURCE_DATE_EPOCH')) 154 os.utime(self.updated_fstab_path, (fstab_time, fstab_time)) 155 156 def _full_path(self, path, name, extention): 157 """ Construct full file path to a file we generate. """ 158 return os.path.join(path, "%s-%s.%s" % (self.name, name, extention)) 159 160 # 161 # Actual implemention 162 # 163 def create(self): 164 """ 165 For 'wic', we already have our build artifacts - we just create 166 filesystems from the artifacts directly and combine them into 167 a partitioned image. 168 """ 169 if not self.no_fstab_update: 170 self.update_fstab(self.rootfs_dir.get("ROOTFS_DIR")) 171 172 for part in self.parts: 173 # get rootfs size from bitbake variable if it's not set in .ks file 174 if not part.size: 175 # and if rootfs name is specified for the partition 176 image_name = self.rootfs_dir.get(part.rootfs_dir) 177 if image_name and os.path.sep not in image_name: 178 # Bitbake variable ROOTFS_SIZE is calculated in 179 # Image._get_rootfs_size method from meta/lib/oe/image.py 180 # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, 181 # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE 182 rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name) 183 if rsize_bb: 184 part.size = int(round(float(rsize_bb))) 185 186 self._image.prepare(self) 187 self._image.layout_partitions() 188 self._image.create() 189 190 def assemble(self): 191 """ 192 Assemble partitions into disk image 193 """ 194 self._image.assemble() 195 196 def finalize(self): 197 """ 198 Finalize the disk image. 199 200 For example, prepare the image to be bootable by e.g. 201 creating and installing a bootloader configuration. 202 """ 203 source_plugin = self.ks.bootloader.source 204 disk_name = self.parts[0].disk 205 if source_plugin: 206 plugin = PluginMgr.get_plugins('source')[source_plugin] 207 plugin.do_install_disk(self._image, disk_name, self, self.workdir, 208 self.oe_builddir, self.bootimg_dir, 209 self.kernel_dir, self.native_sysroot) 210 211 full_path = self._image.path 212 # Generate .bmap 213 if self.bmap: 214 logger.debug("Generating bmap file for %s", disk_name) 215 python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3') 216 bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool') 217 exec_native_cmd("%s %s create %s -o %s.bmap" % \ 218 (python, bmaptool, full_path, full_path), self.native_sysroot) 219 # Compress the image 220 if self.compressor: 221 logger.debug("Compressing disk %s with %s", disk_name, self.compressor) 222 exec_cmd("%s %s" % (self.compressor, full_path)) 223 224 def print_info(self): 225 """ 226 Print the image(s) and artifacts used, for the user. 227 """ 228 msg = "The new image(s) can be found here:\n" 229 230 extension = "direct" + {"gzip": ".gz", 231 "bzip2": ".bz2", 232 "xz": ".xz", 233 None: ""}.get(self.compressor) 234 full_path = self._full_path(self.outdir, self.parts[0].disk, extension) 235 msg += ' %s\n\n' % full_path 236 237 msg += 'The following build artifacts were used to create the image(s):\n' 238 for part in self.parts: 239 if part.rootfs_dir is None: 240 continue 241 if part.mountpoint == '/': 242 suffix = ':' 243 else: 244 suffix = '["%s"]:' % (part.mountpoint or part.label) 245 rootdir = part.rootfs_dir 246 msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), rootdir) 247 248 msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir 249 msg += ' KERNEL_DIR: %s\n' % self.kernel_dir 250 msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot 251 252 logger.info(msg) 253 254 @property 255 def rootdev(self): 256 """ 257 Get root device name to use as a 'root' parameter 258 in kernel command line. 259 260 Assume partition order same as in wks 261 """ 262 for part in self.parts: 263 if part.mountpoint == "/": 264 if part.uuid: 265 return "PARTUUID=%s" % part.uuid 266 elif part.label and self.ptable_format != 'msdos': 267 return "PARTLABEL=%s" % part.label 268 else: 269 suffix = 'p' if part.disk.startswith('mmcblk') else '' 270 return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) 271 272 def cleanup(self): 273 if self._image: 274 self._image.cleanup() 275 276 # Move results to the output dir 277 if not os.path.exists(self.outdir): 278 os.makedirs(self.outdir) 279 280 for fname in os.listdir(self.workdir): 281 path = os.path.join(self.workdir, fname) 282 if os.path.isfile(path): 283 shutil.move(path, os.path.join(self.outdir, fname)) 284 285 # remove work directory when it is not in debugging mode 286 if not self.debug: 287 shutil.rmtree(self.workdir, ignore_errors=True) 288 289# Overhead of the MBR partitioning scheme (just one sector) 290MBR_OVERHEAD = 1 291 292# Overhead of the GPT partitioning scheme 293GPT_OVERHEAD = 34 294 295# Size of a sector in bytes 296SECTOR_SIZE = 512 297 298class PartitionedImage(): 299 """ 300 Partitioned image in a file. 301 """ 302 303 def __init__(self, path, ptable_format, partitions, native_sysroot=None, extra_space=0): 304 self.path = path # Path to the image file 305 self.numpart = 0 # Number of allocated partitions 306 self.realpart = 0 # Number of partitions in the partition table 307 self.primary_part_num = 0 # Number of primary partitions (msdos) 308 self.extendedpart = 0 # Create extended partition before this logical partition (msdos) 309 self.extended_size_sec = 0 # Size of exteded partition (msdos) 310 self.logical_part_cnt = 0 # Number of total logical paritions (msdos) 311 self.offset = 0 # Offset of next partition (in sectors) 312 self.min_size = 0 # Minimum required disk size to fit 313 # all partitions (in bytes) 314 self.ptable_format = ptable_format # Partition table format 315 # Disk system identifier 316 if os.getenv('SOURCE_DATE_EPOCH'): 317 self.identifier = random.Random(int(os.getenv('SOURCE_DATE_EPOCH'))).randint(1, 0xffffffff) 318 else: 319 self.identifier = random.SystemRandom().randint(1, 0xffffffff) 320 321 self.partitions = partitions 322 self.partimages = [] 323 # Size of a sector used in calculations 324 self.sector_size = SECTOR_SIZE 325 self.native_sysroot = native_sysroot 326 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 327 self.extra_space = extra_space 328 329 # calculate the real partition number, accounting for partitions not 330 # in the partition table and logical partitions 331 realnum = 0 332 for part in self.partitions: 333 if part.no_table: 334 part.realnum = 0 335 else: 336 realnum += 1 337 if self.ptable_format == 'msdos' and realnum > 3 and num_real_partitions > 4: 338 part.realnum = realnum + 1 339 continue 340 part.realnum = realnum 341 342 # generate parition and filesystem UUIDs 343 for part in self.partitions: 344 if not part.uuid and part.use_uuid: 345 if self.ptable_format == 'gpt': 346 part.uuid = str(uuid.uuid4()) 347 else: # msdos partition table 348 part.uuid = '%08x-%02d' % (self.identifier, part.realnum) 349 if not part.fsuuid: 350 if part.fstype == 'vfat' or part.fstype == 'msdos': 351 part.fsuuid = '0x' + str(uuid.uuid4())[:8].upper() 352 else: 353 part.fsuuid = str(uuid.uuid4()) 354 else: 355 #make sure the fsuuid for vfat/msdos align with format 0xYYYYYYYY 356 if part.fstype == 'vfat' or part.fstype == 'msdos': 357 if part.fsuuid.upper().startswith("0X"): 358 part.fsuuid = '0x' + part.fsuuid.upper()[2:].rjust(8,"0") 359 else: 360 part.fsuuid = '0x' + part.fsuuid.upper().rjust(8,"0") 361 362 def prepare(self, imager): 363 """Prepare an image. Call prepare method of all image partitions.""" 364 for part in self.partitions: 365 # need to create the filesystems in order to get their 366 # sizes before we can add them and do the layout. 367 part.prepare(imager, imager.workdir, imager.oe_builddir, 368 imager.rootfs_dir, imager.bootimg_dir, 369 imager.kernel_dir, imager.native_sysroot, 370 imager.updated_fstab_path) 371 372 # Converting kB to sectors for parted 373 part.size_sec = part.disk_size * 1024 // self.sector_size 374 375 def layout_partitions(self): 376 """ Layout the partitions, meaning calculate the position of every 377 partition on the disk. The 'ptable_format' parameter defines the 378 partition table format and may be "msdos". """ 379 380 logger.debug("Assigning %s partitions to disks", self.ptable_format) 381 382 # The number of primary and logical partitions. Extended partition and 383 # partitions not listed in the table are not included. 384 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 385 386 # Go through partitions in the order they are added in .ks file 387 for num in range(len(self.partitions)): 388 part = self.partitions[num] 389 390 if self.ptable_format == 'msdos' and part.part_name: 391 raise WicError("setting custom partition name is not " \ 392 "implemented for msdos partitions") 393 394 if self.ptable_format == 'msdos' and part.part_type: 395 # The --part-type can also be implemented for MBR partitions, 396 # in which case it would map to the 1-byte "partition type" 397 # filed at offset 3 of the partition entry. 398 raise WicError("setting custom partition type is not " \ 399 "implemented for msdos partitions") 400 401 # Get the disk where the partition is located 402 self.numpart += 1 403 if not part.no_table: 404 self.realpart += 1 405 406 if self.numpart == 1: 407 if self.ptable_format == "msdos": 408 overhead = MBR_OVERHEAD 409 elif self.ptable_format == "gpt": 410 overhead = GPT_OVERHEAD 411 412 # Skip one sector required for the partitioning scheme overhead 413 self.offset += overhead 414 415 if self.ptable_format == "msdos": 416 if self.primary_part_num > 3 or \ 417 (self.extendedpart == 0 and self.primary_part_num >= 3 and num_real_partitions > 4): 418 part.type = 'logical' 419 # Reserve a sector for EBR for every logical partition 420 # before alignment is performed. 421 if part.type == 'logical': 422 self.offset += 2 423 424 align_sectors = 0 425 if part.align: 426 # If not first partition and we do have alignment set we need 427 # to align the partition. 428 # FIXME: This leaves a empty spaces to the disk. To fill the 429 # gaps we could enlargea the previous partition? 430 431 # Calc how much the alignment is off. 432 align_sectors = self.offset % (part.align * 1024 // self.sector_size) 433 434 if align_sectors: 435 # If partition is not aligned as required, we need 436 # to move forward to the next alignment point 437 align_sectors = (part.align * 1024 // self.sector_size) - align_sectors 438 439 logger.debug("Realignment for %s%s with %s sectors, original" 440 " offset %s, target alignment is %sK.", 441 part.disk, self.numpart, align_sectors, 442 self.offset, part.align) 443 444 # increase the offset so we actually start the partition on right alignment 445 self.offset += align_sectors 446 447 if part.offset is not None: 448 offset = part.offset // self.sector_size 449 450 if offset * self.sector_size != part.offset: 451 raise WicError("Could not place %s%s at offset %d with sector size %d" % (part.disk, self.numpart, part.offset, self.sector_size)) 452 453 delta = offset - self.offset 454 if delta < 0: 455 raise WicError("Could not place %s%s at offset %d: next free sector is %d (delta: %d)" % (part.disk, self.numpart, part.offset, self.offset, delta)) 456 457 logger.debug("Skipping %d sectors to place %s%s at offset %dK", 458 delta, part.disk, self.numpart, part.offset) 459 460 self.offset = offset 461 462 part.start = self.offset 463 self.offset += part.size_sec 464 465 if not part.no_table: 466 part.num = self.realpart 467 else: 468 part.num = 0 469 470 if self.ptable_format == "msdos" and not part.no_table: 471 if part.type == 'logical': 472 self.logical_part_cnt += 1 473 part.num = self.logical_part_cnt + 4 474 if self.extendedpart == 0: 475 # Create extended partition as a primary partition 476 self.primary_part_num += 1 477 self.extendedpart = part.num 478 else: 479 self.extended_size_sec += align_sectors 480 self.extended_size_sec += part.size_sec + 2 481 else: 482 self.primary_part_num += 1 483 part.num = self.primary_part_num 484 485 logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d " 486 "sectors (%d bytes).", part.mountpoint, part.disk, 487 part.num, part.start, self.offset - 1, part.size_sec, 488 part.size_sec * self.sector_size) 489 490 # Once all the partitions have been layed out, we can calculate the 491 # minumim disk size 492 self.min_size = self.offset 493 if self.ptable_format == "gpt": 494 self.min_size += GPT_OVERHEAD 495 496 self.min_size *= self.sector_size 497 self.min_size += self.extra_space 498 499 def _create_partition(self, device, parttype, fstype, start, size): 500 """ Create a partition on an image described by the 'device' object. """ 501 502 # Start is included to the size so we need to substract one from the end. 503 end = start + size - 1 504 logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors", 505 parttype, start, end, size) 506 507 cmd = "parted -s %s unit s mkpart %s" % (device, parttype) 508 if fstype: 509 cmd += " %s" % fstype 510 cmd += " %d %d" % (start, end) 511 512 return exec_native_cmd(cmd, self.native_sysroot) 513 514 def create(self): 515 logger.debug("Creating sparse file %s", self.path) 516 with open(self.path, 'w') as sparse: 517 os.ftruncate(sparse.fileno(), self.min_size) 518 519 logger.debug("Initializing partition table for %s", self.path) 520 exec_native_cmd("parted -s %s mklabel %s" % 521 (self.path, self.ptable_format), self.native_sysroot) 522 523 logger.debug("Set disk identifier %x", self.identifier) 524 with open(self.path, 'r+b') as img: 525 img.seek(0x1B8) 526 img.write(self.identifier.to_bytes(4, 'little')) 527 528 logger.debug("Creating partitions") 529 530 for part in self.partitions: 531 if part.num == 0: 532 continue 533 534 if self.ptable_format == "msdos" and part.num == self.extendedpart: 535 # Create an extended partition (note: extended 536 # partition is described in MBR and contains all 537 # logical partitions). The logical partitions save a 538 # sector for an EBR just before the start of a 539 # partition. The extended partition must start one 540 # sector before the start of the first logical 541 # partition. This way the first EBR is inside of the 542 # extended partition. Since the extended partitions 543 # starts a sector before the first logical partition, 544 # add a sector at the back, so that there is enough 545 # room for all logical partitions. 546 self._create_partition(self.path, "extended", 547 None, part.start - 2, 548 self.extended_size_sec) 549 550 if part.fstype == "swap": 551 parted_fs_type = "linux-swap" 552 elif part.fstype == "vfat": 553 parted_fs_type = "fat32" 554 elif part.fstype == "msdos": 555 parted_fs_type = "fat16" 556 if not part.system_id: 557 part.system_id = '0x6' # FAT16 558 else: 559 # Type for ext2/ext3/ext4/btrfs 560 parted_fs_type = "ext2" 561 562 # Boot ROM of OMAP boards require vfat boot partition to have an 563 # even number of sectors. 564 if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \ 565 and part.size_sec % 2: 566 logger.debug("Subtracting one sector from '%s' partition to " 567 "get even number of sectors for the partition", 568 part.mountpoint) 569 part.size_sec -= 1 570 571 self._create_partition(self.path, part.type, 572 parted_fs_type, part.start, part.size_sec) 573 574 if part.part_name: 575 logger.debug("partition %d: set name to %s", 576 part.num, part.part_name) 577 exec_native_cmd("sgdisk --change-name=%d:%s %s" % \ 578 (part.num, part.part_name, 579 self.path), self.native_sysroot) 580 581 if part.part_type: 582 logger.debug("partition %d: set type UID to %s", 583 part.num, part.part_type) 584 exec_native_cmd("sgdisk --typecode=%d:%s %s" % \ 585 (part.num, part.part_type, 586 self.path), self.native_sysroot) 587 588 if part.uuid and self.ptable_format == "gpt": 589 logger.debug("partition %d: set UUID to %s", 590 part.num, part.uuid) 591 exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ 592 (part.num, part.uuid, self.path), 593 self.native_sysroot) 594 595 if part.label and self.ptable_format == "gpt": 596 logger.debug("partition %d: set name to %s", 597 part.num, part.label) 598 exec_native_cmd("parted -s %s name %d %s" % \ 599 (self.path, part.num, part.label), 600 self.native_sysroot) 601 602 if part.active: 603 flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot" 604 logger.debug("Set '%s' flag for partition '%s' on disk '%s'", 605 flag_name, part.num, self.path) 606 exec_native_cmd("parted -s %s set %d %s on" % \ 607 (self.path, part.num, flag_name), 608 self.native_sysroot) 609 if part.system_id: 610 exec_native_cmd("sfdisk --part-type %s %s %s" % \ 611 (self.path, part.num, part.system_id), 612 self.native_sysroot) 613 614 def cleanup(self): 615 pass 616 617 def assemble(self): 618 logger.debug("Installing partitions") 619 620 for part in self.partitions: 621 source = part.source_file 622 if source: 623 # install source_file contents into a partition 624 sparse_copy(source, self.path, seek=part.start * self.sector_size) 625 626 logger.debug("Installed %s in partition %d, sectors %d-%d, " 627 "size %d sectors", source, part.num, part.start, 628 part.start + part.size_sec - 1, part.size_sec) 629 630 partimage = self.path + '.p%d' % part.num 631 os.rename(source, partimage) 632 self.partimages.append(partimage) 633