1# 2# Copyright (c) 2013-2016 Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7# This module provides the OpenEmbedded partition object definitions. 8# 9# AUTHORS 10# Tom Zanussi <tom.zanussi (at] linux.intel.com> 11# Ed Bartosh <ed.bartosh> (at] linux.intel.com> 12 13import logging 14import os 15import uuid 16 17from wic import WicError 18from wic.misc import exec_cmd, exec_native_cmd, get_bitbake_var 19from wic.pluginbase import PluginMgr 20 21logger = logging.getLogger('wic') 22 23class Partition(): 24 25 def __init__(self, args, lineno): 26 self.args = args 27 self.active = args.active 28 self.align = args.align 29 self.disk = args.disk 30 self.device = None 31 self.extra_space = args.extra_space 32 self.exclude_path = args.exclude_path 33 self.include_path = args.include_path 34 self.change_directory = args.change_directory 35 self.fsopts = args.fsopts 36 self.fspassno = args.fspassno 37 self.fstype = args.fstype 38 self.label = args.label 39 self.use_label = args.use_label 40 self.mkfs_extraopts = args.mkfs_extraopts 41 self.mountpoint = args.mountpoint 42 self.no_table = args.no_table 43 self.num = None 44 self.offset = args.offset 45 self.overhead_factor = args.overhead_factor 46 self.part_name = args.part_name 47 self.part_type = args.part_type 48 self.rootfs_dir = args.rootfs_dir 49 self.size = args.size 50 self.fixed_size = args.fixed_size 51 self.source = args.source 52 self.sourceparams = args.sourceparams 53 self.system_id = args.system_id 54 self.use_uuid = args.use_uuid 55 self.uuid = args.uuid 56 self.fsuuid = args.fsuuid 57 self.type = args.type 58 self.no_fstab_update = args.no_fstab_update 59 self.updated_fstab_path = None 60 self.has_fstab = False 61 self.update_fstab_in_rootfs = False 62 self.hidden = args.hidden 63 self.mbr = args.mbr 64 65 self.lineno = lineno 66 self.source_file = "" 67 68 def get_extra_block_count(self, current_blocks): 69 """ 70 The --size param is reflected in self.size (in kB), and we already 71 have current_blocks (1k) blocks, calculate and return the 72 number of (1k) blocks we need to add to get to --size, 0 if 73 we're already there or beyond. 74 """ 75 logger.debug("Requested partition size for %s: %d", 76 self.mountpoint, self.size) 77 78 if not self.size: 79 return 0 80 81 requested_blocks = self.size 82 83 logger.debug("Requested blocks %d, current_blocks %d", 84 requested_blocks, current_blocks) 85 86 if requested_blocks > current_blocks: 87 return requested_blocks - current_blocks 88 else: 89 return 0 90 91 def get_rootfs_size(self, actual_rootfs_size=0): 92 """ 93 Calculate the required size of rootfs taking into consideration 94 --size/--fixed-size flags as well as overhead and extra space, as 95 specified in kickstart file. Raises an error if the 96 `actual_rootfs_size` is larger than fixed-size rootfs. 97 98 """ 99 if self.fixed_size: 100 rootfs_size = self.fixed_size 101 if actual_rootfs_size > rootfs_size: 102 raise WicError("Actual rootfs size (%d kB) is larger than " 103 "allowed size %d kB" % 104 (actual_rootfs_size, rootfs_size)) 105 else: 106 extra_blocks = self.get_extra_block_count(actual_rootfs_size) 107 if extra_blocks < self.extra_space: 108 extra_blocks = self.extra_space 109 110 rootfs_size = actual_rootfs_size + extra_blocks 111 rootfs_size = int(rootfs_size * self.overhead_factor) 112 113 logger.debug("Added %d extra blocks to %s to get to %d total blocks", 114 extra_blocks, self.mountpoint, rootfs_size) 115 116 return rootfs_size 117 118 @property 119 def disk_size(self): 120 """ 121 Obtain on-disk size of partition taking into consideration 122 --size/--fixed-size options. 123 124 """ 125 return self.fixed_size if self.fixed_size else self.size 126 127 def prepare(self, creator, cr_workdir, oe_builddir, rootfs_dir, 128 bootimg_dir, kernel_dir, native_sysroot, updated_fstab_path): 129 """ 130 Prepare content for individual partitions, depending on 131 partition command parameters. 132 """ 133 self.updated_fstab_path = updated_fstab_path 134 if self.updated_fstab_path and not (self.fstype.startswith("ext") or self.fstype == "msdos"): 135 self.update_fstab_in_rootfs = True 136 137 if not self.source: 138 if self.fstype == "none" or self.no_table: 139 return 140 if not self.size and not self.fixed_size: 141 raise WicError("The %s partition has a size of zero. Please " 142 "specify a non-zero --size/--fixed-size for that " 143 "partition." % self.mountpoint) 144 145 if self.fstype == "swap": 146 self.prepare_swap_partition(cr_workdir, oe_builddir, 147 native_sysroot) 148 self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype) 149 else: 150 if self.fstype in ('squashfs', 'erofs'): 151 raise WicError("It's not possible to create empty %s " 152 "partition '%s'" % (self.fstype, self.mountpoint)) 153 154 rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label, 155 self.lineno, self.fstype) 156 if os.path.isfile(rootfs): 157 os.remove(rootfs) 158 159 prefix = "ext" if self.fstype.startswith("ext") else self.fstype 160 method = getattr(self, "prepare_empty_partition_" + prefix) 161 method(rootfs, oe_builddir, native_sysroot) 162 self.source_file = rootfs 163 return 164 165 plugins = PluginMgr.get_plugins('source') 166 167 if self.source not in plugins: 168 raise WicError("The '%s' --source specified for %s doesn't exist.\n\t" 169 "See 'wic list source-plugins' for a list of available" 170 " --sources.\n\tSee 'wic help source-plugins' for " 171 "details on adding a new source plugin." % 172 (self.source, self.mountpoint)) 173 174 srcparams_dict = {} 175 if self.sourceparams: 176 # Split sourceparams string of the form key1=val1[,key2=val2,...] 177 # into a dict. Also accepts valueless keys i.e. without = 178 splitted = self.sourceparams.split(',') 179 srcparams_dict = dict((par.split('=', 1) + [None])[:2] for par in splitted if par) 180 181 plugin = PluginMgr.get_plugins('source')[self.source] 182 plugin.do_configure_partition(self, srcparams_dict, creator, 183 cr_workdir, oe_builddir, bootimg_dir, 184 kernel_dir, native_sysroot) 185 plugin.do_stage_partition(self, srcparams_dict, creator, 186 cr_workdir, oe_builddir, bootimg_dir, 187 kernel_dir, native_sysroot) 188 plugin.do_prepare_partition(self, srcparams_dict, creator, 189 cr_workdir, oe_builddir, bootimg_dir, 190 kernel_dir, rootfs_dir, native_sysroot) 191 plugin.do_post_partition(self, srcparams_dict, creator, 192 cr_workdir, oe_builddir, bootimg_dir, 193 kernel_dir, rootfs_dir, native_sysroot) 194 195 # further processing required Partition.size to be an integer, make 196 # sure that it is one 197 if not isinstance(self.size, int): 198 raise WicError("Partition %s internal size is not an integer. " 199 "This a bug in source plugin %s and needs to be fixed." % 200 (self.mountpoint, self.source)) 201 202 if self.fixed_size and self.size > self.fixed_size: 203 raise WicError("File system image of partition %s is " 204 "larger (%d kB) than its allowed size %d kB" % 205 (self.mountpoint, self.size, self.fixed_size)) 206 207 def prepare_rootfs(self, cr_workdir, oe_builddir, rootfs_dir, 208 native_sysroot, real_rootfs = True, pseudo_dir = None): 209 """ 210 Prepare content for a rootfs partition i.e. create a partition 211 and fill it from a /rootfs dir. 212 213 Currently handles ext2/3/4, btrfs, vfat and squashfs. 214 """ 215 216 rootfs = "%s/rootfs_%s.%s.%s" % (cr_workdir, self.label, 217 self.lineno, self.fstype) 218 if os.path.isfile(rootfs): 219 os.remove(rootfs) 220 221 p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot) 222 if (pseudo_dir): 223 # Canonicalize the ignore paths. This corresponds to 224 # calling oe.path.canonicalize(), which is used in bitbake.conf. 225 ignore_paths = [rootfs] + (get_bitbake_var("PSEUDO_IGNORE_PATHS") or "").split(",") 226 canonical_paths = [] 227 for path in ignore_paths: 228 if "$" not in path: 229 trailing_slash = path.endswith("/") and "/" or "" 230 canonical_paths.append(os.path.realpath(path) + trailing_slash) 231 ignore_paths = ",".join(canonical_paths) 232 233 pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix 234 pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir 235 pseudo += "export PSEUDO_PASSWD=%s;" % rootfs_dir 236 pseudo += "export PSEUDO_NOSYMLINKEXP=1;" 237 pseudo += "export PSEUDO_IGNORE_PATHS=%s;" % ignore_paths 238 pseudo += "%s " % get_bitbake_var("FAKEROOTCMD") 239 else: 240 pseudo = None 241 242 if not self.size and real_rootfs: 243 # The rootfs size is not set in .ks file so try to get it 244 # from bitbake variable 245 rsize_bb = get_bitbake_var('ROOTFS_SIZE') 246 rdir = get_bitbake_var('IMAGE_ROOTFS') 247 if rsize_bb and rdir == rootfs_dir: 248 # Bitbake variable ROOTFS_SIZE is calculated in 249 # Image._get_rootfs_size method from meta/lib/oe/image.py 250 # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, 251 # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE 252 self.size = int(round(float(rsize_bb))) 253 else: 254 # Bitbake variable ROOTFS_SIZE is not defined so compute it 255 # from the rootfs_dir size using the same logic found in 256 # get_rootfs_size() from meta/classes/image.bbclass 257 du_cmd = "du -ks %s" % rootfs_dir 258 out = exec_cmd(du_cmd) 259 self.size = int(out.split()[0]) 260 261 prefix = "ext" if self.fstype.startswith("ext") else self.fstype 262 method = getattr(self, "prepare_rootfs_" + prefix) 263 method(rootfs, cr_workdir, oe_builddir, rootfs_dir, native_sysroot, pseudo) 264 self.source_file = rootfs 265 266 # get the rootfs size in the right units for kickstart (kB) 267 du_cmd = "du -Lbks %s" % rootfs 268 out = exec_cmd(du_cmd) 269 self.size = int(out.split()[0]) 270 271 def prepare_rootfs_ext(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, 272 native_sysroot, pseudo): 273 """ 274 Prepare content for an ext2/3/4 rootfs partition. 275 """ 276 du_cmd = "du -ks %s" % rootfs_dir 277 out = exec_cmd(du_cmd) 278 actual_rootfs_size = int(out.split()[0]) 279 280 rootfs_size = self.get_rootfs_size(actual_rootfs_size) 281 282 with open(rootfs, 'w') as sparse: 283 os.ftruncate(sparse.fileno(), rootfs_size * 1024) 284 285 extraopts = self.mkfs_extraopts or "-F -i 8192" 286 287 # use hash_seed to generate reproducible ext4 images 288 (extraopts, pseudo) = self.get_hash_seed_ext4(extraopts, pseudo) 289 290 label_str = "" 291 if self.label: 292 label_str = "-L %s" % self.label 293 294 mkfs_cmd = "mkfs.%s %s %s %s -U %s -d %s" % \ 295 (self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir) 296 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) 297 298 if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update: 299 debugfs_script_path = os.path.join(cr_workdir, "debugfs_script") 300 with open(debugfs_script_path, "w") as f: 301 f.write("cd etc\n") 302 f.write("rm fstab\n") 303 f.write("write %s fstab\n" % (self.updated_fstab_path)) 304 debugfs_cmd = "debugfs -w -f %s %s" % (debugfs_script_path, rootfs) 305 exec_native_cmd(debugfs_cmd, native_sysroot) 306 307 mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs) 308 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) 309 310 if os.getenv('SOURCE_DATE_EPOCH'): 311 sde_time = hex(int(os.getenv('SOURCE_DATE_EPOCH'))) 312 debugfs_script_path = os.path.join(cr_workdir, "debugfs_script") 313 files = [] 314 for root, dirs, others in os.walk(rootfs_dir): 315 base = root.replace(rootfs_dir, "").rstrip(os.sep) 316 files += [ "/" if base == "" else base ] 317 files += [ base + "/" + n for n in dirs + others ] 318 with open(debugfs_script_path, "w") as f: 319 f.write("set_current_time %s\n" % (sde_time)) 320 if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update: 321 f.write("set_inode_field /etc/fstab mtime %s\n" % (sde_time)) 322 f.write("set_inode_field /etc/fstab mtime_extra 0\n") 323 for file in set(files): 324 for time in ["atime", "ctime", "crtime"]: 325 f.write("set_inode_field \"%s\" %s %s\n" % (file, time, sde_time)) 326 f.write("set_inode_field \"%s\" %s_extra 0\n" % (file, time)) 327 for time in ["wtime", "mkfs_time", "lastcheck"]: 328 f.write("set_super_value %s %s\n" % (time, sde_time)) 329 for time in ["mtime", "first_error_time", "last_error_time"]: 330 f.write("set_super_value %s 0\n" % (time)) 331 debugfs_cmd = "debugfs -w -f %s %s" % (debugfs_script_path, rootfs) 332 exec_native_cmd(debugfs_cmd, native_sysroot) 333 334 self.check_for_Y2038_problem(rootfs, native_sysroot) 335 336 def get_hash_seed_ext4(self, extraopts, pseudo): 337 if os.getenv('SOURCE_DATE_EPOCH'): 338 sde_time = int(os.getenv('SOURCE_DATE_EPOCH')) 339 if pseudo: 340 pseudo = "export E2FSPROGS_FAKE_TIME=%s;%s " % (sde_time, pseudo) 341 else: 342 pseudo = "export E2FSPROGS_FAKE_TIME=%s; " % sde_time 343 344 # Set hash_seed to generate deterministic directory indexes 345 namespace = uuid.UUID("e7429877-e7b3-4a68-a5c9-2f2fdf33d460") 346 if self.fsuuid: 347 namespace = uuid.UUID(self.fsuuid) 348 hash_seed = str(uuid.uuid5(namespace, str(sde_time))) 349 extraopts += " -E hash_seed=%s" % hash_seed 350 351 return (extraopts, pseudo) 352 353 def prepare_rootfs_btrfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, 354 native_sysroot, pseudo): 355 """ 356 Prepare content for a btrfs rootfs partition. 357 """ 358 du_cmd = "du -ks %s" % rootfs_dir 359 out = exec_cmd(du_cmd) 360 actual_rootfs_size = int(out.split()[0]) 361 362 rootfs_size = self.get_rootfs_size(actual_rootfs_size) 363 364 with open(rootfs, 'w') as sparse: 365 os.ftruncate(sparse.fileno(), rootfs_size * 1024) 366 367 label_str = "" 368 if self.label: 369 label_str = "-L %s" % self.label 370 371 mkfs_cmd = "mkfs.%s -b %d -r %s %s %s -U %s %s" % \ 372 (self.fstype, rootfs_size * 1024, rootfs_dir, label_str, 373 self.mkfs_extraopts, self.fsuuid, rootfs) 374 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) 375 376 def prepare_rootfs_msdos(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, 377 native_sysroot, pseudo): 378 """ 379 Prepare content for a msdos/vfat rootfs partition. 380 """ 381 du_cmd = "du -bks %s" % rootfs_dir 382 out = exec_cmd(du_cmd) 383 blocks = int(out.split()[0]) 384 385 rootfs_size = self.get_rootfs_size(blocks) 386 387 label_str = "-n boot" 388 if self.label: 389 label_str = "-n %s" % self.label 390 391 size_str = "" 392 393 extraopts = self.mkfs_extraopts or '-S 512' 394 395 dosfs_cmd = "mkdosfs %s -i %s %s %s -C %s %d" % \ 396 (label_str, self.fsuuid, size_str, extraopts, rootfs, 397 rootfs_size) 398 exec_native_cmd(dosfs_cmd, native_sysroot) 399 400 mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir) 401 exec_native_cmd(mcopy_cmd, native_sysroot) 402 403 if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update: 404 mcopy_cmd = "mcopy -m -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path) 405 exec_native_cmd(mcopy_cmd, native_sysroot) 406 407 chmod_cmd = "chmod 644 %s" % rootfs 408 exec_cmd(chmod_cmd) 409 410 prepare_rootfs_vfat = prepare_rootfs_msdos 411 412 def prepare_rootfs_squashfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, 413 native_sysroot, pseudo): 414 """ 415 Prepare content for a squashfs rootfs partition. 416 """ 417 extraopts = self.mkfs_extraopts or '-noappend' 418 squashfs_cmd = "mksquashfs %s %s %s" % \ 419 (rootfs_dir, rootfs, extraopts) 420 exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo) 421 422 def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, 423 native_sysroot, pseudo): 424 """ 425 Prepare content for a erofs rootfs partition. 426 """ 427 extraopts = self.mkfs_extraopts or '' 428 erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \ 429 (extraopts, self.fsuuid, rootfs, rootfs_dir) 430 exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo) 431 432 def prepare_empty_partition_none(self, rootfs, oe_builddir, native_sysroot): 433 pass 434 435 def prepare_empty_partition_ext(self, rootfs, oe_builddir, 436 native_sysroot): 437 """ 438 Prepare an empty ext2/3/4 partition. 439 """ 440 size = self.disk_size 441 with open(rootfs, 'w') as sparse: 442 os.ftruncate(sparse.fileno(), size * 1024) 443 444 extraopts = self.mkfs_extraopts or "-i 8192" 445 446 # use hash_seed to generate reproducible ext4 images 447 (extraopts, pseudo) = self.get_hash_seed_ext4(extraopts, None) 448 449 label_str = "" 450 if self.label: 451 label_str = "-L %s" % self.label 452 453 mkfs_cmd = "mkfs.%s -F %s %s -U %s %s" % \ 454 (self.fstype, extraopts, label_str, self.fsuuid, rootfs) 455 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) 456 457 self.check_for_Y2038_problem(rootfs, native_sysroot) 458 459 def prepare_empty_partition_btrfs(self, rootfs, oe_builddir, 460 native_sysroot): 461 """ 462 Prepare an empty btrfs partition. 463 """ 464 size = self.disk_size 465 with open(rootfs, 'w') as sparse: 466 os.ftruncate(sparse.fileno(), size * 1024) 467 468 label_str = "" 469 if self.label: 470 label_str = "-L %s" % self.label 471 472 mkfs_cmd = "mkfs.%s -b %d %s -U %s %s %s" % \ 473 (self.fstype, self.size * 1024, label_str, self.fsuuid, 474 self.mkfs_extraopts, rootfs) 475 exec_native_cmd(mkfs_cmd, native_sysroot) 476 477 def prepare_empty_partition_msdos(self, rootfs, oe_builddir, 478 native_sysroot): 479 """ 480 Prepare an empty vfat partition. 481 """ 482 blocks = self.disk_size 483 484 label_str = "-n boot" 485 if self.label: 486 label_str = "-n %s" % self.label 487 488 size_str = "" 489 490 extraopts = self.mkfs_extraopts or '-S 512' 491 492 dosfs_cmd = "mkdosfs %s -i %s %s %s -C %s %d" % \ 493 (label_str, self.fsuuid, extraopts, size_str, rootfs, 494 blocks) 495 496 exec_native_cmd(dosfs_cmd, native_sysroot) 497 498 chmod_cmd = "chmod 644 %s" % rootfs 499 exec_cmd(chmod_cmd) 500 501 prepare_empty_partition_vfat = prepare_empty_partition_msdos 502 503 def prepare_swap_partition(self, cr_workdir, oe_builddir, native_sysroot): 504 """ 505 Prepare a swap partition. 506 """ 507 path = "%s/fs.%s" % (cr_workdir, self.fstype) 508 509 with open(path, 'w') as sparse: 510 os.ftruncate(sparse.fileno(), self.size * 1024) 511 512 label_str = "" 513 if self.label: 514 label_str = "-L %s" % self.label 515 516 mkswap_cmd = "mkswap %s -U %s %s" % (label_str, self.fsuuid, path) 517 exec_native_cmd(mkswap_cmd, native_sysroot) 518 519 def check_for_Y2038_problem(self, rootfs, native_sysroot): 520 """ 521 Check if the filesystem is affected by the Y2038 problem 522 (Y2038 problem = 32 bit time_t overflow in January 2038) 523 """ 524 def get_err_str(part): 525 err = "The {} filesystem {} has no Y2038 support." 526 if part.mountpoint: 527 args = [part.fstype, "mounted at %s" % part.mountpoint] 528 elif part.label: 529 args = [part.fstype, "labeled '%s'" % part.label] 530 elif part.part_name: 531 args = [part.fstype, "in partition '%s'" % part.part_name] 532 else: 533 args = [part.fstype, "in partition %s" % part.num] 534 return err.format(*args) 535 536 # ext2 and ext3 are always affected by the Y2038 problem 537 if self.fstype in ["ext2", "ext3"]: 538 logger.warn(get_err_str(self)) 539 return 540 541 ret, out = exec_native_cmd("dumpe2fs %s" % rootfs, native_sysroot) 542 543 # if ext4 is affected by the Y2038 problem depends on the inode size 544 for line in out.splitlines(): 545 if line.startswith("Inode size:"): 546 size = int(line.split(":")[1].strip()) 547 if size < 256: 548 logger.warn("%s Inodes (of size %d) are too small." % 549 (get_err_str(self), size)) 550 break 551 552