1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7# This implements the 'direct' imager plugin class for 'wic' 8# 9# AUTHORS 10# Tom Zanussi <tom.zanussi (at] linux.intel.com> 11# 12 13import logging 14import os 15import random 16import shutil 17import tempfile 18import uuid 19 20from time import strftime 21 22from oe.path import copyhardlinktree 23 24from wic import WicError 25from wic.filemap import sparse_copy 26from wic.ksparser import KickStart, KickStartError 27from wic.pluginbase import PluginMgr, ImagerPlugin 28from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd 29 30logger = logging.getLogger('wic') 31 32class DirectPlugin(ImagerPlugin): 33 """ 34 Install a system into a file containing a partitioned disk image. 35 36 An image file is formatted with a partition table, each partition 37 created from a rootfs or other OpenEmbedded build artifact and dd'ed 38 into the virtual disk. The disk image can subsequently be dd'ed onto 39 media and used on actual hardware. 40 """ 41 name = 'direct' 42 43 def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir, 44 native_sysroot, oe_builddir, options): 45 try: 46 self.ks = KickStart(wks_file) 47 except KickStartError as err: 48 raise WicError(str(err)) 49 50 # parse possible 'rootfs=name' items 51 self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' ')) 52 self.bootimg_dir = bootimg_dir 53 self.kernel_dir = kernel_dir 54 self.native_sysroot = native_sysroot 55 self.oe_builddir = oe_builddir 56 57 self.outdir = options.outdir 58 self.compressor = options.compressor 59 self.bmap = options.bmap 60 self.no_fstab_update = options.no_fstab_update 61 self.original_fstab = None 62 63 self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0], 64 strftime("%Y%m%d%H%M")) 65 self.workdir = tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.') 66 self._image = None 67 self.ptable_format = self.ks.bootloader.ptable 68 self.parts = self.ks.partitions 69 70 # as a convenience, set source to the boot partition source 71 # instead of forcing it to be set via bootloader --source 72 for part in self.parts: 73 if not self.ks.bootloader.source and part.mountpoint == "/boot": 74 self.ks.bootloader.source = part.source 75 break 76 77 image_path = self._full_path(self.workdir, self.parts[0].disk, "direct") 78 self._image = PartitionedImage(image_path, self.ptable_format, 79 self.parts, self.native_sysroot) 80 81 def do_create(self): 82 """ 83 Plugin entry point. 84 """ 85 try: 86 self.create() 87 self.assemble() 88 self.finalize() 89 self.print_info() 90 finally: 91 self.cleanup() 92 93 def _write_fstab(self, image_rootfs): 94 """overriden to generate fstab (temporarily) in rootfs. This is called 95 from _create, make sure it doesn't get called from 96 BaseImage.create() 97 """ 98 if not image_rootfs: 99 return 100 101 fstab_path = image_rootfs + "/etc/fstab" 102 if not os.path.isfile(fstab_path): 103 return 104 105 with open(fstab_path) as fstab: 106 fstab_lines = fstab.readlines() 107 self.original_fstab = fstab_lines.copy() 108 109 if self._update_fstab(fstab_lines, self.parts): 110 with open(fstab_path, "w") as fstab: 111 fstab.writelines(fstab_lines) 112 else: 113 self.original_fstab = None 114 115 def _update_fstab(self, fstab_lines, parts): 116 """Assume partition order same as in wks""" 117 updated = False 118 for part in parts: 119 if not part.realnum or not part.mountpoint \ 120 or part.mountpoint == "/": 121 continue 122 123 if part.use_uuid: 124 if part.fsuuid: 125 # FAT UUID is different from others 126 if len(part.fsuuid) == 10: 127 device_name = "UUID=%s-%s" % \ 128 (part.fsuuid[2:6], part.fsuuid[6:]) 129 else: 130 device_name = "UUID=%s" % part.fsuuid 131 else: 132 device_name = "PARTUUID=%s" % part.uuid 133 elif part.use_label: 134 device_name = "LABEL=%s" % part.label 135 else: 136 # mmc device partitions are named mmcblk0p1, mmcblk0p2.. 137 prefix = 'p' if part.disk.startswith('mmcblk') else '' 138 device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum) 139 140 opts = part.fsopts if part.fsopts else "defaults" 141 line = "\t".join([device_name, part.mountpoint, part.fstype, 142 opts, "0", "0"]) + "\n" 143 144 fstab_lines.append(line) 145 updated = True 146 147 return updated 148 149 def _full_path(self, path, name, extention): 150 """ Construct full file path to a file we generate. """ 151 return os.path.join(path, "%s-%s.%s" % (self.name, name, extention)) 152 153 # 154 # Actual implemention 155 # 156 def create(self): 157 """ 158 For 'wic', we already have our build artifacts - we just create 159 filesystems from the artifacts directly and combine them into 160 a partitioned image. 161 """ 162 if not self.no_fstab_update: 163 self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR")) 164 165 for part in self.parts: 166 # get rootfs size from bitbake variable if it's not set in .ks file 167 if not part.size: 168 # and if rootfs name is specified for the partition 169 image_name = self.rootfs_dir.get(part.rootfs_dir) 170 if image_name and os.path.sep not in image_name: 171 # Bitbake variable ROOTFS_SIZE is calculated in 172 # Image._get_rootfs_size method from meta/lib/oe/image.py 173 # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, 174 # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE 175 rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name) 176 if rsize_bb: 177 part.size = int(round(float(rsize_bb))) 178 179 self._image.prepare(self) 180 self._image.layout_partitions() 181 self._image.create() 182 183 def assemble(self): 184 """ 185 Assemble partitions into disk image 186 """ 187 self._image.assemble() 188 189 def finalize(self): 190 """ 191 Finalize the disk image. 192 193 For example, prepare the image to be bootable by e.g. 194 creating and installing a bootloader configuration. 195 """ 196 source_plugin = self.ks.bootloader.source 197 disk_name = self.parts[0].disk 198 if source_plugin: 199 plugin = PluginMgr.get_plugins('source')[source_plugin] 200 plugin.do_install_disk(self._image, disk_name, self, self.workdir, 201 self.oe_builddir, self.bootimg_dir, 202 self.kernel_dir, self.native_sysroot) 203 204 full_path = self._image.path 205 # Generate .bmap 206 if self.bmap: 207 logger.debug("Generating bmap file for %s", disk_name) 208 python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3') 209 bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool') 210 exec_native_cmd("%s %s create %s -o %s.bmap" % \ 211 (python, bmaptool, full_path, full_path), self.native_sysroot) 212 # Compress the image 213 if self.compressor: 214 logger.debug("Compressing disk %s with %s", disk_name, self.compressor) 215 exec_cmd("%s %s" % (self.compressor, full_path)) 216 217 def print_info(self): 218 """ 219 Print the image(s) and artifacts used, for the user. 220 """ 221 msg = "The new image(s) can be found here:\n" 222 223 extension = "direct" + {"gzip": ".gz", 224 "bzip2": ".bz2", 225 "xz": ".xz", 226 None: ""}.get(self.compressor) 227 full_path = self._full_path(self.outdir, self.parts[0].disk, extension) 228 msg += ' %s\n\n' % full_path 229 230 msg += 'The following build artifacts were used to create the image(s):\n' 231 for part in self.parts: 232 if part.rootfs_dir is None: 233 continue 234 if part.mountpoint == '/': 235 suffix = ':' 236 else: 237 suffix = '["%s"]:' % (part.mountpoint or part.label) 238 rootdir = part.rootfs_dir 239 msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), rootdir) 240 241 msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir 242 msg += ' KERNEL_DIR: %s\n' % self.kernel_dir 243 msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot 244 245 logger.info(msg) 246 247 @property 248 def rootdev(self): 249 """ 250 Get root device name to use as a 'root' parameter 251 in kernel command line. 252 253 Assume partition order same as in wks 254 """ 255 for part in self.parts: 256 if part.mountpoint == "/": 257 if part.uuid: 258 return "PARTUUID=%s" % part.uuid 259 else: 260 suffix = 'p' if part.disk.startswith('mmcblk') else '' 261 return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) 262 263 def cleanup(self): 264 if self._image: 265 self._image.cleanup() 266 267 # Move results to the output dir 268 if not os.path.exists(self.outdir): 269 os.makedirs(self.outdir) 270 271 for fname in os.listdir(self.workdir): 272 path = os.path.join(self.workdir, fname) 273 if os.path.isfile(path): 274 shutil.move(path, os.path.join(self.outdir, fname)) 275 276 #Restore original fstab 277 if self.original_fstab: 278 fstab_path = self.rootfs_dir.get("ROOTFS_DIR") + "/etc/fstab" 279 with open(fstab_path, "w") as fstab: 280 fstab.writelines(self.original_fstab) 281 282 # remove work directory 283 shutil.rmtree(self.workdir, ignore_errors=True) 284 285# Overhead of the MBR partitioning scheme (just one sector) 286MBR_OVERHEAD = 1 287 288# Overhead of the GPT partitioning scheme 289GPT_OVERHEAD = 34 290 291# Size of a sector in bytes 292SECTOR_SIZE = 512 293 294class PartitionedImage(): 295 """ 296 Partitioned image in a file. 297 """ 298 299 def __init__(self, path, ptable_format, partitions, native_sysroot=None): 300 self.path = path # Path to the image file 301 self.numpart = 0 # Number of allocated partitions 302 self.realpart = 0 # Number of partitions in the partition table 303 self.primary_part_num = 0 # Number of primary partitions (msdos) 304 self.extendedpart = 0 # Create extended partition before this logical partition (msdos) 305 self.extended_size_sec = 0 # Size of exteded partition (msdos) 306 self.logical_part_cnt = 0 # Number of total logical paritions (msdos) 307 self.offset = 0 # Offset of next partition (in sectors) 308 self.min_size = 0 # Minimum required disk size to fit 309 # all partitions (in bytes) 310 self.ptable_format = ptable_format # Partition table format 311 # Disk system identifier 312 self.identifier = random.SystemRandom().randint(1, 0xffffffff) 313 314 self.partitions = partitions 315 self.partimages = [] 316 # Size of a sector used in calculations 317 self.sector_size = SECTOR_SIZE 318 self.native_sysroot = native_sysroot 319 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 320 321 # calculate the real partition number, accounting for partitions not 322 # in the partition table and logical partitions 323 realnum = 0 324 for part in self.partitions: 325 if part.no_table: 326 part.realnum = 0 327 else: 328 realnum += 1 329 if self.ptable_format == 'msdos' and realnum > 3 and num_real_partitions > 4: 330 part.realnum = realnum + 1 331 continue 332 part.realnum = realnum 333 334 # generate parition and filesystem UUIDs 335 for part in self.partitions: 336 if not part.uuid and part.use_uuid: 337 if self.ptable_format == 'gpt': 338 part.uuid = str(uuid.uuid4()) 339 else: # msdos partition table 340 part.uuid = '%08x-%02d' % (self.identifier, part.realnum) 341 if not part.fsuuid: 342 if part.fstype == 'vfat' or part.fstype == 'msdos': 343 part.fsuuid = '0x' + str(uuid.uuid4())[:8].upper() 344 else: 345 part.fsuuid = str(uuid.uuid4()) 346 347 def prepare(self, imager): 348 """Prepare an image. Call prepare method of all image partitions.""" 349 for part in self.partitions: 350 # need to create the filesystems in order to get their 351 # sizes before we can add them and do the layout. 352 part.prepare(imager, imager.workdir, imager.oe_builddir, 353 imager.rootfs_dir, imager.bootimg_dir, 354 imager.kernel_dir, imager.native_sysroot) 355 356 # Converting kB to sectors for parted 357 part.size_sec = part.disk_size * 1024 // self.sector_size 358 359 def layout_partitions(self): 360 """ Layout the partitions, meaning calculate the position of every 361 partition on the disk. The 'ptable_format' parameter defines the 362 partition table format and may be "msdos". """ 363 364 logger.debug("Assigning %s partitions to disks", self.ptable_format) 365 366 # The number of primary and logical partitions. Extended partition and 367 # partitions not listed in the table are not included. 368 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 369 370 # Go through partitions in the order they are added in .ks file 371 for num in range(len(self.partitions)): 372 part = self.partitions[num] 373 374 if self.ptable_format == 'msdos' and part.part_name: 375 raise WicError("setting custom partition name is not " \ 376 "implemented for msdos partitions") 377 378 if self.ptable_format == 'msdos' and part.part_type: 379 # The --part-type can also be implemented for MBR partitions, 380 # in which case it would map to the 1-byte "partition type" 381 # filed at offset 3 of the partition entry. 382 raise WicError("setting custom partition type is not " \ 383 "implemented for msdos partitions") 384 385 # Get the disk where the partition is located 386 self.numpart += 1 387 if not part.no_table: 388 self.realpart += 1 389 390 if self.numpart == 1: 391 if self.ptable_format == "msdos": 392 overhead = MBR_OVERHEAD 393 elif self.ptable_format == "gpt": 394 overhead = GPT_OVERHEAD 395 396 # Skip one sector required for the partitioning scheme overhead 397 self.offset += overhead 398 399 if self.ptable_format == "msdos": 400 if self.primary_part_num > 3 or \ 401 (self.extendedpart == 0 and self.primary_part_num >= 3 and num_real_partitions > 4): 402 part.type = 'logical' 403 # Reserve a sector for EBR for every logical partition 404 # before alignment is performed. 405 if part.type == 'logical': 406 self.offset += 2 407 408 align_sectors = 0 409 if part.align: 410 # If not first partition and we do have alignment set we need 411 # to align the partition. 412 # FIXME: This leaves a empty spaces to the disk. To fill the 413 # gaps we could enlargea the previous partition? 414 415 # Calc how much the alignment is off. 416 align_sectors = self.offset % (part.align * 1024 // self.sector_size) 417 418 if align_sectors: 419 # If partition is not aligned as required, we need 420 # to move forward to the next alignment point 421 align_sectors = (part.align * 1024 // self.sector_size) - align_sectors 422 423 logger.debug("Realignment for %s%s with %s sectors, original" 424 " offset %s, target alignment is %sK.", 425 part.disk, self.numpart, align_sectors, 426 self.offset, part.align) 427 428 # increase the offset so we actually start the partition on right alignment 429 self.offset += align_sectors 430 431 part.start = self.offset 432 self.offset += part.size_sec 433 434 if not part.no_table: 435 part.num = self.realpart 436 else: 437 part.num = 0 438 439 if self.ptable_format == "msdos" and not part.no_table: 440 if part.type == 'logical': 441 self.logical_part_cnt += 1 442 part.num = self.logical_part_cnt + 4 443 if self.extendedpart == 0: 444 # Create extended partition as a primary partition 445 self.primary_part_num += 1 446 self.extendedpart = part.num 447 else: 448 self.extended_size_sec += align_sectors 449 self.extended_size_sec += part.size_sec + 2 450 else: 451 self.primary_part_num += 1 452 part.num = self.primary_part_num 453 454 logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d " 455 "sectors (%d bytes).", part.mountpoint, part.disk, 456 part.num, part.start, self.offset - 1, part.size_sec, 457 part.size_sec * self.sector_size) 458 459 # Once all the partitions have been layed out, we can calculate the 460 # minumim disk size 461 self.min_size = self.offset 462 if self.ptable_format == "gpt": 463 self.min_size += GPT_OVERHEAD 464 465 self.min_size *= self.sector_size 466 467 def _create_partition(self, device, parttype, fstype, start, size): 468 """ Create a partition on an image described by the 'device' object. """ 469 470 # Start is included to the size so we need to substract one from the end. 471 end = start + size - 1 472 logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors", 473 parttype, start, end, size) 474 475 cmd = "parted -s %s unit s mkpart %s" % (device, parttype) 476 if fstype: 477 cmd += " %s" % fstype 478 cmd += " %d %d" % (start, end) 479 480 return exec_native_cmd(cmd, self.native_sysroot) 481 482 def create(self): 483 logger.debug("Creating sparse file %s", self.path) 484 with open(self.path, 'w') as sparse: 485 os.ftruncate(sparse.fileno(), self.min_size) 486 487 logger.debug("Initializing partition table for %s", self.path) 488 exec_native_cmd("parted -s %s mklabel %s" % 489 (self.path, self.ptable_format), self.native_sysroot) 490 491 logger.debug("Set disk identifier %x", self.identifier) 492 with open(self.path, 'r+b') as img: 493 img.seek(0x1B8) 494 img.write(self.identifier.to_bytes(4, 'little')) 495 496 logger.debug("Creating partitions") 497 498 for part in self.partitions: 499 if part.num == 0: 500 continue 501 502 if self.ptable_format == "msdos" and part.num == self.extendedpart: 503 # Create an extended partition (note: extended 504 # partition is described in MBR and contains all 505 # logical partitions). The logical partitions save a 506 # sector for an EBR just before the start of a 507 # partition. The extended partition must start one 508 # sector before the start of the first logical 509 # partition. This way the first EBR is inside of the 510 # extended partition. Since the extended partitions 511 # starts a sector before the first logical partition, 512 # add a sector at the back, so that there is enough 513 # room for all logical partitions. 514 self._create_partition(self.path, "extended", 515 None, part.start - 2, 516 self.extended_size_sec) 517 518 if part.fstype == "swap": 519 parted_fs_type = "linux-swap" 520 elif part.fstype == "vfat": 521 parted_fs_type = "fat32" 522 elif part.fstype == "msdos": 523 parted_fs_type = "fat16" 524 if not part.system_id: 525 part.system_id = '0x6' # FAT16 526 else: 527 # Type for ext2/ext3/ext4/btrfs 528 parted_fs_type = "ext2" 529 530 # Boot ROM of OMAP boards require vfat boot partition to have an 531 # even number of sectors. 532 if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \ 533 and part.size_sec % 2: 534 logger.debug("Subtracting one sector from '%s' partition to " 535 "get even number of sectors for the partition", 536 part.mountpoint) 537 part.size_sec -= 1 538 539 self._create_partition(self.path, part.type, 540 parted_fs_type, part.start, part.size_sec) 541 542 if part.part_name: 543 logger.debug("partition %d: set name to %s", 544 part.num, part.part_name) 545 exec_native_cmd("sgdisk --change-name=%d:%s %s" % \ 546 (part.num, part.part_name, 547 self.path), self.native_sysroot) 548 549 if part.part_type: 550 logger.debug("partition %d: set type UID to %s", 551 part.num, part.part_type) 552 exec_native_cmd("sgdisk --typecode=%d:%s %s" % \ 553 (part.num, part.part_type, 554 self.path), self.native_sysroot) 555 556 if part.uuid and self.ptable_format == "gpt": 557 logger.debug("partition %d: set UUID to %s", 558 part.num, part.uuid) 559 exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ 560 (part.num, part.uuid, self.path), 561 self.native_sysroot) 562 563 if part.label and self.ptable_format == "gpt": 564 logger.debug("partition %d: set name to %s", 565 part.num, part.label) 566 exec_native_cmd("parted -s %s name %d %s" % \ 567 (self.path, part.num, part.label), 568 self.native_sysroot) 569 570 if part.active: 571 flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot" 572 logger.debug("Set '%s' flag for partition '%s' on disk '%s'", 573 flag_name, part.num, self.path) 574 exec_native_cmd("parted -s %s set %d %s on" % \ 575 (self.path, part.num, flag_name), 576 self.native_sysroot) 577 if part.system_id: 578 exec_native_cmd("sfdisk --part-type %s %s %s" % \ 579 (self.path, part.num, part.system_id), 580 self.native_sysroot) 581 582 def cleanup(self): 583 pass 584 585 def assemble(self): 586 logger.debug("Installing partitions") 587 588 for part in self.partitions: 589 source = part.source_file 590 if source: 591 # install source_file contents into a partition 592 sparse_copy(source, self.path, seek=part.start * self.sector_size) 593 594 logger.debug("Installed %s in partition %d, sectors %d-%d, " 595 "size %d sectors", source, part.num, part.start, 596 part.start + part.size_sec - 1, part.size_sec) 597 598 partimage = self.path + '.p%d' % part.num 599 os.rename(source, partimage) 600 self.partimages.append(partimage) 601