xref: /openbmc/openbmc/poky/scripts/lib/wic/engine.py (revision 595f6308)
1#
2# Copyright (c) 2013, Intel Corporation.
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6# DESCRIPTION
7
8# This module implements the image creation engine used by 'wic' to
9# create images.  The engine parses through the OpenEmbedded kickstart
10# (wks) file specified and generates images that can then be directly
11# written onto media.
12#
13# AUTHORS
14# Tom Zanussi <tom.zanussi (at] linux.intel.com>
15#
16
17import logging
18import os
19import tempfile
20import json
21import subprocess
22import shutil
23import re
24
25from collections import namedtuple, OrderedDict
26
27from wic import WicError
28from wic.filemap import sparse_copy
29from wic.pluginbase import PluginMgr
30from wic.misc import get_bitbake_var, exec_cmd
31
32logger = logging.getLogger('wic')
33
34def verify_build_env():
35    """
36    Verify that the build environment is sane.
37
38    Returns True if it is, false otherwise
39    """
40    if not os.environ.get("BUILDDIR"):
41        raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)")
42
43    return True
44
45
46CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts
47SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR
48WIC_DIR = "wic"
49
50def build_canned_image_list(path):
51    layers_path = get_bitbake_var("BBLAYERS")
52    canned_wks_layer_dirs = []
53
54    if layers_path is not None:
55        for layer_path in layers_path.split():
56            for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR):
57                cpath = os.path.join(layer_path, wks_path)
58                if os.path.isdir(cpath):
59                    canned_wks_layer_dirs.append(cpath)
60
61    cpath = os.path.join(path, CANNED_IMAGE_DIR)
62    canned_wks_layer_dirs.append(cpath)
63
64    return canned_wks_layer_dirs
65
66def find_canned_image(scripts_path, wks_file):
67    """
68    Find a .wks file with the given name in the canned files dir.
69
70    Return False if not found
71    """
72    layers_canned_wks_dir = build_canned_image_list(scripts_path)
73
74    for canned_wks_dir in layers_canned_wks_dir:
75        for root, dirs, files in os.walk(canned_wks_dir):
76            for fname in files:
77                if fname.endswith("~") or fname.endswith("#"):
78                    continue
79                if ((fname.endswith(".wks") and wks_file + ".wks" == fname) or \
80                   (fname.endswith(".wks.in") and wks_file + ".wks.in" == fname)):
81                    fullpath = os.path.join(canned_wks_dir, fname)
82                    return fullpath
83    return None
84
85
86def list_canned_images(scripts_path):
87    """
88    List the .wks files in the canned image dir, minus the extension.
89    """
90    layers_canned_wks_dir = build_canned_image_list(scripts_path)
91
92    for canned_wks_dir in layers_canned_wks_dir:
93        for root, dirs, files in os.walk(canned_wks_dir):
94            for fname in files:
95                if fname.endswith("~") or fname.endswith("#"):
96                    continue
97                if fname.endswith(".wks") or fname.endswith(".wks.in"):
98                    fullpath = os.path.join(canned_wks_dir, fname)
99                    with open(fullpath) as wks:
100                        for line in wks:
101                            desc = ""
102                            idx = line.find("short-description:")
103                            if idx != -1:
104                                desc = line[idx + len("short-description:"):].strip()
105                                break
106                    basename = fname.split('.')[0]
107                    print("  %s\t\t%s" % (basename.ljust(30), desc))
108
109
110def list_canned_image_help(scripts_path, fullpath):
111    """
112    List the help and params in the specified canned image.
113    """
114    found = False
115    with open(fullpath) as wks:
116        for line in wks:
117            if not found:
118                idx = line.find("long-description:")
119                if idx != -1:
120                    print()
121                    print(line[idx + len("long-description:"):].strip())
122                    found = True
123                continue
124            if not line.strip():
125                break
126            idx = line.find("#")
127            if idx != -1:
128                print(line[idx + len("#:"):].rstrip())
129            else:
130                break
131
132
133def list_source_plugins():
134    """
135    List the available source plugins i.e. plugins available for --source.
136    """
137    plugins = PluginMgr.get_plugins('source')
138
139    for plugin in plugins:
140        print("  %s" % plugin)
141
142
143def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
144               native_sysroot, options):
145    """
146    Create image
147
148    wks_file - user-defined OE kickstart file
149    rootfs_dir - absolute path to the build's /rootfs dir
150    bootimg_dir - absolute path to the build's boot artifacts directory
151    kernel_dir - absolute path to the build's kernel directory
152    native_sysroot - absolute path to the build's native sysroots dir
153    image_output_dir - dirname to create for image
154    options - wic command line options (debug, bmap, etc)
155
156    Normally, the values for the build artifacts values are determined
157    by 'wic -e' from the output of the 'bitbake -e' command given an
158    image name e.g. 'core-image-minimal' and a given machine set in
159    local.conf.  If that's the case, the variables get the following
160    values from the output of 'bitbake -e':
161
162    rootfs_dir:        IMAGE_ROOTFS
163    kernel_dir:        DEPLOY_DIR_IMAGE
164    native_sysroot:    STAGING_DIR_NATIVE
165
166    In the above case, bootimg_dir remains unset and the
167    plugin-specific image creation code is responsible for finding the
168    bootimg artifacts.
169
170    In the case where the values are passed in explicitly i.e 'wic -e'
171    is not used but rather the individual 'wic' options are used to
172    explicitly specify these values.
173    """
174    try:
175        oe_builddir = os.environ["BUILDDIR"]
176    except KeyError:
177        raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)")
178
179    if not os.path.exists(options.outdir):
180        os.makedirs(options.outdir)
181
182    pname = options.imager
183    plugin_class = PluginMgr.get_plugins('imager').get(pname)
184    if not plugin_class:
185        raise WicError('Unknown plugin: %s' % pname)
186
187    plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
188                          native_sysroot, oe_builddir, options)
189
190    plugin.do_create()
191
192    logger.info("The image(s) were created using OE kickstart file:\n  %s", wks_file)
193
194
195def wic_list(args, scripts_path):
196    """
197    Print the list of images or source plugins.
198    """
199    if args.list_type is None:
200        return False
201
202    if args.list_type == "images":
203
204        list_canned_images(scripts_path)
205        return True
206    elif args.list_type == "source-plugins":
207        list_source_plugins()
208        return True
209    elif len(args.help_for) == 1 and args.help_for[0] == 'help':
210        wks_file = args.list_type
211        fullpath = find_canned_image(scripts_path, wks_file)
212        if not fullpath:
213            raise WicError("No image named %s found, exiting. "
214                           "(Use 'wic list images' to list available images, "
215                           "or specify a fully-qualified OE kickstart (.wks) "
216                           "filename)" % wks_file)
217
218        list_canned_image_help(scripts_path, fullpath)
219        return True
220
221    return False
222
223
224class Disk:
225    def __init__(self, imagepath, native_sysroot, fstypes=('fat', 'ext')):
226        self.imagepath = imagepath
227        self.native_sysroot = native_sysroot
228        self.fstypes = fstypes
229        self._partitions = None
230        self._partimages = {}
231        self._lsector_size = None
232        self._psector_size = None
233        self._ptable_format = None
234
235        # find parted
236        # read paths from $PATH environment variable
237        # if it fails, use hardcoded paths
238        pathlist = "/bin:/usr/bin:/usr/sbin:/sbin/"
239        try:
240            self.paths = os.environ['PATH'] + ":" + pathlist
241        except KeyError:
242            self.paths = pathlist
243
244        if native_sysroot:
245            for path in pathlist.split(':'):
246                self.paths = "%s%s:%s" % (native_sysroot, path, self.paths)
247
248        self.parted = shutil.which("parted", path=self.paths)
249        if not self.parted:
250            raise WicError("Can't find executable parted")
251
252        self.partitions = self.get_partitions()
253
254    def __del__(self):
255        for path in self._partimages.values():
256            os.unlink(path)
257
258    def get_partitions(self):
259        if self._partitions is None:
260            self._partitions = OrderedDict()
261            out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath))
262            parttype = namedtuple("Part", "pnum start end size fstype")
263            splitted = out.splitlines()
264            # skip over possible errors in exec_cmd output
265            try:
266                idx =splitted.index("BYT;")
267            except ValueError:
268                raise WicError("Error getting partition information from %s" % (self.parted))
269            lsector_size, psector_size, self._ptable_format = splitted[idx + 1].split(":")[3:6]
270            self._lsector_size = int(lsector_size)
271            self._psector_size = int(psector_size)
272            for line in splitted[idx + 2:]:
273                pnum, start, end, size, fstype = line.split(':')[:5]
274                partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]),
275                                     int(size[:-1]), fstype)
276                self._partitions[pnum] = partition
277
278        return self._partitions
279
280    def __getattr__(self, name):
281        """Get path to the executable in a lazy way."""
282        if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck",
283                    "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"):
284            aname = "_%s" % name
285            if aname not in self.__dict__:
286                setattr(self, aname, shutil.which(name, path=self.paths))
287                if aname not in self.__dict__ or self.__dict__[aname] is None:
288                    raise WicError("Can't find executable '{}'".format(name))
289            return self.__dict__[aname]
290        return self.__dict__[name]
291
292    def _get_part_image(self, pnum):
293        if pnum not in self.partitions:
294            raise WicError("Partition %s is not in the image" % pnum)
295        part = self.partitions[pnum]
296        # check if fstype is supported
297        for fstype in self.fstypes:
298            if part.fstype.startswith(fstype):
299                break
300        else:
301            raise WicError("Not supported fstype: {}".format(part.fstype))
302        if pnum not in self._partimages:
303            tmpf = tempfile.NamedTemporaryFile(prefix="wic-part")
304            dst_fname = tmpf.name
305            tmpf.close()
306            sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size)
307            self._partimages[pnum] = dst_fname
308
309        return self._partimages[pnum]
310
311    def _put_part_image(self, pnum):
312        """Put partition image into partitioned image."""
313        sparse_copy(self._partimages[pnum], self.imagepath,
314                    seek=self.partitions[pnum].start)
315
316    def dir(self, pnum, path):
317        if pnum not in self.partitions:
318            raise WicError("Partition %s is not in the image" % pnum)
319
320        if self.partitions[pnum].fstype.startswith('ext'):
321            return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs,
322                                                         self._get_part_image(pnum),
323                                                         path), as_shell=True)
324        else: # fat
325            return exec_cmd("{} -i {} ::{}".format(self.mdir,
326                                                   self._get_part_image(pnum),
327                                                   path))
328
329    def copy(self, src, dest):
330        """Copy partition image into wic image."""
331        pnum =  dest.part if isinstance(src, str) else src.part
332
333        if self.partitions[pnum].fstype.startswith('ext'):
334            if isinstance(src, str):
335                cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
336                      format(os.path.dirname(dest.path), src, os.path.basename(src),
337                             self.debugfs, self._get_part_image(pnum))
338            else: # copy from wic
339                # run both dump and rdump to support both files and directory
340                cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\
341                      format(os.path.dirname(src.path), src.path,
342                             dest, src.path, dest, self.debugfs,
343                             self._get_part_image(pnum))
344        else: # fat
345            if isinstance(src, str):
346                cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
347                                                  self._get_part_image(pnum),
348                                                  src, dest.path)
349            else:
350                cmd = "{} -i {} -snop ::{} {}".format(self.mcopy,
351                                                  self._get_part_image(pnum),
352                                                  src.path, dest)
353
354        exec_cmd(cmd, as_shell=True)
355        self._put_part_image(pnum)
356
357    def remove_ext(self, pnum, path, recursive):
358        """
359        Remove files/dirs and their contents from the partition.
360        This only applies to ext* partition.
361        """
362        abs_path = re.sub('\/\/+', '/', path)
363        cmd = "{} {} -wR 'rm \"{}\"'".format(self.debugfs,
364                                            self._get_part_image(pnum),
365                                            abs_path)
366        out = exec_cmd(cmd , as_shell=True)
367        for line in out.splitlines():
368            if line.startswith("rm:"):
369                if "file is a directory" in line:
370                    if recursive:
371                        # loop through content and delete them one by one if
372                        # flaged with -r
373                        subdirs = iter(self.dir(pnum, abs_path).splitlines())
374                        next(subdirs)
375                        for subdir in subdirs:
376                            dir = subdir.split(':')[1].split(" ", 1)[1]
377                            if not dir == "." and not dir == "..":
378                                self.remove_ext(pnum, "%s/%s" % (abs_path, dir), recursive)
379
380                    rmdir_out = exec_cmd("{} {} -wR 'rmdir \"{}\"'".format(self.debugfs,
381                                                    self._get_part_image(pnum),
382                                                    abs_path.rstrip('/'))
383                                                    , as_shell=True)
384
385                    for rmdir_line in rmdir_out.splitlines():
386                        if "directory not empty" in rmdir_line:
387                            raise WicError("Could not complete operation: \n%s \n"
388                                            "use -r to remove non-empty directory" % rmdir_line)
389                        if rmdir_line.startswith("rmdir:"):
390                            raise WicError("Could not complete operation: \n%s "
391                                            "\n%s" % (str(line), rmdir_line))
392
393                else:
394                    raise WicError("Could not complete operation: \n%s "
395                                    "\nUnable to remove %s" % (str(line), abs_path))
396
397    def remove(self, pnum, path, recursive):
398        """Remove files/dirs from the partition."""
399        partimg = self._get_part_image(pnum)
400        if self.partitions[pnum].fstype.startswith('ext'):
401            self.remove_ext(pnum, path, recursive)
402
403        else: # fat
404            cmd = "{} -i {} ::{}".format(self.mdel, partimg, path)
405            try:
406                exec_cmd(cmd)
407            except WicError as err:
408                if "not found" in str(err) or "non empty" in str(err):
409                    # mdel outputs 'File ... not found' or 'directory .. non empty"
410                    # try to use mdeltree as path could be a directory
411                    cmd = "{} -i {} ::{}".format(self.mdeltree,
412                                                 partimg, path)
413                    exec_cmd(cmd)
414                else:
415                    raise err
416        self._put_part_image(pnum)
417
418    def write(self, target, expand):
419        """Write disk image to the media or file."""
420        def write_sfdisk_script(outf, parts):
421            for key, val in parts['partitiontable'].items():
422                if key in ("partitions", "device", "firstlba", "lastlba"):
423                    continue
424                if key == "id":
425                    key = "label-id"
426                outf.write("{}: {}\n".format(key, val))
427            outf.write("\n")
428            for part in parts['partitiontable']['partitions']:
429                line = ''
430                for name in ('attrs', 'name', 'size', 'type', 'uuid'):
431                    if name == 'size' and part['type'] == 'f':
432                        # don't write size for extended partition
433                        continue
434                    val = part.get(name)
435                    if val:
436                        line += '{}={}, '.format(name, val)
437                if line:
438                    line = line[:-2] # strip ', '
439                if part.get('bootable'):
440                    line += ' ,bootable'
441                outf.write("{}\n".format(line))
442            outf.flush()
443
444        def read_ptable(path):
445            out = exec_cmd("{} -J {}".format(self.sfdisk, path))
446            return json.loads(out)
447
448        def write_ptable(parts, target):
449            with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-", mode='w') as outf:
450                write_sfdisk_script(outf, parts)
451                cmd = "{} --no-reread {} < {} ".format(self.sfdisk, target, outf.name)
452                exec_cmd(cmd, as_shell=True)
453
454        if expand is None:
455            sparse_copy(self.imagepath, target)
456        else:
457            # copy first sectors that may contain bootloader
458            sparse_copy(self.imagepath, target, length=2048 * self._lsector_size)
459
460            # copy source partition table to the target
461            parts = read_ptable(self.imagepath)
462            write_ptable(parts, target)
463
464            # get size of unpartitioned space
465            free = None
466            for line in exec_cmd("{} -F {}".format(self.sfdisk, target)).splitlines():
467                if line.startswith("Unpartitioned space ") and line.endswith("sectors"):
468                    free = int(line.split()[-2])
469                    # Align free space to a 2048 sector boundary. YOCTO #12840.
470                    free = free - (free % 2048)
471            if free is None:
472                raise WicError("Can't get size of unpartitioned space")
473
474            # calculate expanded partitions sizes
475            sizes = {}
476            num_auto_resize = 0
477            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
478                if num in expand:
479                    if expand[num] != 0: # don't resize partition if size is set to 0
480                        sectors = expand[num] // self._lsector_size
481                        free -= sectors - part['size']
482                        part['size'] = sectors
483                        sizes[num] = sectors
484                elif part['type'] != 'f':
485                    sizes[num] = -1
486                    num_auto_resize += 1
487
488            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
489                if sizes.get(num) == -1:
490                    part['size'] += free // num_auto_resize
491
492            # write resized partition table to the target
493            write_ptable(parts, target)
494
495            # read resized partition table
496            parts = read_ptable(target)
497
498            # copy partitions content
499            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
500                pnum = str(num)
501                fstype = self.partitions[pnum].fstype
502
503                # copy unchanged partition
504                if part['size'] == self.partitions[pnum].size // self._lsector_size:
505                    logger.info("copying unchanged partition {}".format(pnum))
506                    sparse_copy(self._get_part_image(pnum), target, seek=part['start'] * self._lsector_size)
507                    continue
508
509                # resize or re-create partitions
510                if fstype.startswith('ext') or fstype.startswith('fat') or \
511                   fstype.startswith('linux-swap'):
512
513                    partfname = None
514                    with tempfile.NamedTemporaryFile(prefix="wic-part{}-".format(pnum)) as partf:
515                        partfname = partf.name
516
517                    if fstype.startswith('ext'):
518                        logger.info("resizing ext partition {}".format(pnum))
519                        partimg = self._get_part_image(pnum)
520                        sparse_copy(partimg, partfname)
521                        exec_cmd("{} -pf {}".format(self.e2fsck, partfname))
522                        exec_cmd("{} {} {}s".format(\
523                                 self.resize2fs, partfname, part['size']))
524                    elif fstype.startswith('fat'):
525                        logger.info("copying content of the fat partition {}".format(pnum))
526                        with tempfile.TemporaryDirectory(prefix='wic-fatdir-') as tmpdir:
527                            # copy content to the temporary directory
528                            cmd = "{} -snompi {} :: {}".format(self.mcopy,
529                                                               self._get_part_image(pnum),
530                                                               tmpdir)
531                            exec_cmd(cmd)
532                            # create new msdos partition
533                            label = part.get("name")
534                            label_str = "-n {}".format(label) if label else ''
535
536                            cmd = "{} {} -C {} {}".format(self.mkdosfs, label_str, partfname,
537                                                          part['size'])
538                            exec_cmd(cmd)
539                            # copy content from the temporary directory to the new partition
540                            cmd = "{} -snompi {} {}/* ::".format(self.mcopy, partfname, tmpdir)
541                            exec_cmd(cmd, as_shell=True)
542                    elif fstype.startswith('linux-swap'):
543                        logger.info("creating swap partition {}".format(pnum))
544                        label = part.get("name")
545                        label_str = "-L {}".format(label) if label else ''
546                        out = exec_cmd("{} --probe {}".format(self.blkid, self._get_part_image(pnum)))
547                        uuid = out[out.index("UUID=\"")+6:out.index("UUID=\"")+42]
548                        uuid_str = "-U {}".format(uuid) if uuid else ''
549                        with open(partfname, 'w') as sparse:
550                            os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size)
551                        exec_cmd("{} {} {} {}".format(self.mkswap, label_str, uuid_str, partfname))
552                    sparse_copy(partfname, target, seek=part['start'] * self._lsector_size)
553                    os.unlink(partfname)
554                elif part['type'] != 'f':
555                    logger.warning("skipping partition {}: unsupported fstype {}".format(pnum, fstype))
556
557def wic_ls(args, native_sysroot):
558    """List contents of partitioned image or vfat partition."""
559    disk = Disk(args.path.image, native_sysroot)
560    if not args.path.part:
561        if disk.partitions:
562            print('Num     Start        End          Size      Fstype')
563            for part in disk.partitions.values():
564                print("{:2d}  {:12d} {:12d} {:12d}  {}".format(\
565                          part.pnum, part.start, part.end,
566                          part.size, part.fstype))
567    else:
568        path = args.path.path or '/'
569        print(disk.dir(args.path.part, path))
570
571def wic_cp(args, native_sysroot):
572    """
573    Copy file or directory to/from the vfat/ext partition of
574    partitioned image.
575    """
576    if isinstance(args.dest, str):
577        disk = Disk(args.src.image, native_sysroot)
578    else:
579        disk = Disk(args.dest.image, native_sysroot)
580    disk.copy(args.src, args.dest)
581
582
583def wic_rm(args, native_sysroot):
584    """
585    Remove files or directories from the vfat partition of
586    partitioned image.
587    """
588    disk = Disk(args.path.image, native_sysroot)
589    disk.remove(args.path.part, args.path.path, args.recursive_delete)
590
591def wic_write(args, native_sysroot):
592    """
593    Write image to a target device.
594    """
595    disk = Disk(args.image, native_sysroot, ('fat', 'ext', 'linux-swap'))
596    disk.write(args.target, args.expand)
597
598def find_canned(scripts_path, file_name):
599    """
600    Find a file either by its path or by name in the canned files dir.
601
602    Return None if not found
603    """
604    if os.path.exists(file_name):
605        return file_name
606
607    layers_canned_wks_dir = build_canned_image_list(scripts_path)
608    for canned_wks_dir in layers_canned_wks_dir:
609        for root, dirs, files in os.walk(canned_wks_dir):
610            for fname in files:
611                if fname == file_name:
612                    fullpath = os.path.join(canned_wks_dir, fname)
613                    return fullpath
614
615def get_custom_config(boot_file):
616    """
617    Get the custom configuration to be used for the bootloader.
618
619    Return None if the file can't be found.
620    """
621    # Get the scripts path of poky
622    scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__))
623
624    cfg_file = find_canned(scripts_path, boot_file)
625    if cfg_file:
626        with open(cfg_file, "r") as f:
627            config = f.read()
628        return config
629