1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6from abc import ABCMeta, abstractmethod 7from oe.utils import execute_pre_post_process 8from oe.package_manager import * 9from oe.manifest import * 10import oe.path 11import shutil 12import os 13import subprocess 14import re 15 16class Rootfs(object, metaclass=ABCMeta): 17 """ 18 This is an abstract class. Do not instantiate this directly. 19 """ 20 21 def __init__(self, d, progress_reporter=None, logcatcher=None): 22 self.d = d 23 self.pm = None 24 self.image_rootfs = self.d.getVar('IMAGE_ROOTFS') 25 self.deploydir = self.d.getVar('IMGDEPLOYDIR') 26 self.progress_reporter = progress_reporter 27 self.logcatcher = logcatcher 28 29 self.install_order = Manifest.INSTALL_ORDER 30 31 @abstractmethod 32 def _create(self): 33 pass 34 35 @abstractmethod 36 def _get_delayed_postinsts(self): 37 pass 38 39 @abstractmethod 40 def _save_postinsts(self): 41 pass 42 43 @abstractmethod 44 def _log_check(self): 45 pass 46 47 def _log_check_common(self, type, match): 48 # Ignore any lines containing log_check to avoid recursion, and ignore 49 # lines beginning with a + since sh -x may emit code which isn't 50 # actually executed, but may contain error messages 51 excludes = [ 'log_check', r'^\+' ] 52 if hasattr(self, 'log_check_expected_regexes'): 53 excludes.extend(self.log_check_expected_regexes) 54 # Insert custom log_check excludes 55 excludes += [x for x in (self.d.getVar("IMAGE_LOG_CHECK_EXCLUDES") or "").split(" ") if x] 56 excludes = [re.compile(x) for x in excludes] 57 r = re.compile(match) 58 log_path = self.d.expand("${T}/log.do_rootfs") 59 messages = [] 60 with open(log_path, 'r') as log: 61 for line in log: 62 if self.logcatcher and self.logcatcher.contains(line.rstrip()): 63 continue 64 for ee in excludes: 65 m = ee.search(line) 66 if m: 67 break 68 if m: 69 continue 70 71 m = r.search(line) 72 if m: 73 messages.append('[log_check] %s' % line) 74 if messages: 75 if len(messages) == 1: 76 msg = '1 %s message' % type 77 else: 78 msg = '%d %s messages' % (len(messages), type) 79 msg = '[log_check] %s: found %s in the logfile:\n%s' % \ 80 (self.d.getVar('PN'), msg, ''.join(messages)) 81 if type == 'error': 82 bb.fatal(msg) 83 else: 84 bb.warn(msg) 85 86 def _log_check_warn(self): 87 self._log_check_common('warning', '^(warn|Warn|WARNING:)') 88 89 def _log_check_error(self): 90 self._log_check_common('error', self.log_check_regex) 91 92 def _insert_feed_uris(self): 93 if bb.utils.contains("IMAGE_FEATURES", "package-management", 94 True, False, self.d): 95 self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "", 96 self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "", 97 self.d.getVar('PACKAGE_FEED_ARCHS')) 98 99 100 """ 101 The _cleanup() method should be used to clean-up stuff that we don't really 102 want to end up on target. For example, in the case of RPM, the DB locks. 103 The method is called, once, at the end of create() method. 104 """ 105 @abstractmethod 106 def _cleanup(self): 107 pass 108 109 def _setup_dbg_rootfs(self, package_paths): 110 gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0' 111 if gen_debugfs != '1': 112 return 113 114 bb.note(" Renaming the original rootfs...") 115 try: 116 shutil.rmtree(self.image_rootfs + '-orig') 117 except: 118 pass 119 bb.utils.rename(self.image_rootfs, self.image_rootfs + '-orig') 120 121 bb.note(" Creating debug rootfs...") 122 bb.utils.mkdirhier(self.image_rootfs) 123 124 bb.note(" Copying back package database...") 125 for path in package_paths: 126 bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path)) 127 if os.path.isdir(self.image_rootfs + '-orig' + path): 128 shutil.copytree(self.image_rootfs + '-orig' + path, self.image_rootfs + path, symlinks=True) 129 elif os.path.isfile(self.image_rootfs + '-orig' + path): 130 shutil.copyfile(self.image_rootfs + '-orig' + path, self.image_rootfs + path) 131 132 # Copy files located in /usr/lib/debug or /usr/src/debug 133 for dir in ["/usr/lib/debug", "/usr/src/debug"]: 134 src = self.image_rootfs + '-orig' + dir 135 if os.path.exists(src): 136 dst = self.image_rootfs + dir 137 bb.utils.mkdirhier(os.path.dirname(dst)) 138 shutil.copytree(src, dst) 139 140 # Copy files with suffix '.debug' or located in '.debug' dir. 141 for root, dirs, files in os.walk(self.image_rootfs + '-orig'): 142 relative_dir = root[len(self.image_rootfs + '-orig'):] 143 for f in files: 144 if f.endswith('.debug') or '/.debug' in relative_dir: 145 bb.utils.mkdirhier(self.image_rootfs + relative_dir) 146 shutil.copy(os.path.join(root, f), 147 self.image_rootfs + relative_dir) 148 149 bb.note(" Install complementary '*-dbg' packages...") 150 self.pm.install_complementary('*-dbg') 151 152 if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg': 153 bb.note(" Install complementary '*-src' packages...") 154 self.pm.install_complementary('*-src') 155 156 """ 157 Install additional debug packages. Possibility to install additional packages, 158 which are not automatically installed as complementary package of 159 standard one, e.g. debug package of static libraries. 160 """ 161 extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS') 162 if extra_debug_pkgs: 163 bb.note(" Install extra debug packages...") 164 self.pm.install(extra_debug_pkgs.split(), True) 165 166 bb.note(" Removing package database...") 167 for path in package_paths: 168 if os.path.isdir(self.image_rootfs + path): 169 shutil.rmtree(self.image_rootfs + path) 170 elif os.path.isfile(self.image_rootfs + path): 171 os.remove(self.image_rootfs + path) 172 173 bb.note(" Rename debug rootfs...") 174 try: 175 shutil.rmtree(self.image_rootfs + '-dbg') 176 except: 177 pass 178 bb.utils.rename(self.image_rootfs, self.image_rootfs + '-dbg') 179 180 bb.note(" Restoring original rootfs...") 181 bb.utils.rename(self.image_rootfs + '-orig', self.image_rootfs) 182 183 def _exec_shell_cmd(self, cmd): 184 try: 185 subprocess.check_output(cmd, stderr=subprocess.STDOUT) 186 except subprocess.CalledProcessError as e: 187 return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output)) 188 189 return None 190 191 def create(self): 192 bb.note("###### Generate rootfs #######") 193 pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND") 194 post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND") 195 rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND') 196 197 def make_last(command, commands): 198 commands = commands.split() 199 if command in commands: 200 commands.remove(command) 201 commands.append(command) 202 return "".join(commands) 203 204 # We want this to run as late as possible, in particular after 205 # systemd_sysusers_create and set_user_group. Using :append is not enough 206 make_last("tidy_shadowutils_files", post_process_cmds) 207 make_last("rootfs_reproducible", post_process_cmds) 208 209 execute_pre_post_process(self.d, pre_process_cmds) 210 211 if self.progress_reporter: 212 self.progress_reporter.next_stage() 213 214 # call the package manager dependent create method 215 self._create() 216 217 sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir') 218 bb.utils.mkdirhier(sysconfdir) 219 with open(sysconfdir + "/version", "w+") as ver: 220 ver.write(self.d.getVar('BUILDNAME') + "\n") 221 222 execute_pre_post_process(self.d, rootfs_post_install_cmds) 223 224 self.pm.run_intercepts() 225 226 execute_pre_post_process(self.d, post_process_cmds) 227 228 if self.progress_reporter: 229 self.progress_reporter.next_stage() 230 231 if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 232 True, False, self.d) and \ 233 not bb.utils.contains("IMAGE_FEATURES", 234 "read-only-rootfs-delayed-postinsts", 235 True, False, self.d): 236 delayed_postinsts = self._get_delayed_postinsts() 237 if delayed_postinsts is not None: 238 bb.fatal("The following packages could not be configured " 239 "offline and rootfs is read-only: %s" % 240 delayed_postinsts) 241 242 if self.d.getVar('USE_DEVFS') != "1": 243 self._create_devfs() 244 245 self._uninstall_unneeded() 246 247 if self.progress_reporter: 248 self.progress_reporter.next_stage() 249 250 self._insert_feed_uris() 251 252 self._run_ldconfig() 253 254 if self.d.getVar('USE_DEPMOD') != "0": 255 self._generate_kernel_module_deps() 256 257 self._cleanup() 258 self._log_check() 259 260 if self.progress_reporter: 261 self.progress_reporter.next_stage() 262 263 264 def _uninstall_unneeded(self): 265 # Remove the run-postinsts package if no delayed postinsts are found 266 delayed_postinsts = self._get_delayed_postinsts() 267 if delayed_postinsts is None: 268 if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")) or os.path.exists(self.d.expand("${IMAGE_ROOTFS}${systemd_system_unitdir}/run-postinsts.service")): 269 self.pm.remove(["run-postinsts"]) 270 271 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 272 True, False, self.d) 273 image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE') 274 275 if image_rorfs or image_rorfs_force == "1": 276 # Remove components that we don't need if it's a read-only rootfs 277 unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split() 278 pkgs_installed = image_list_installed_packages(self.d) 279 # Make sure update-alternatives is removed last. This is 280 # because its database has to available while uninstalling 281 # other packages, allowing alternative symlinks of packages 282 # to be uninstalled or to be managed correctly otherwise. 283 provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives") 284 pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider) 285 286 # update-alternatives provider is removed in its own remove() 287 # call because all package managers do not guarantee the packages 288 # are removed in the order they given in the list (which is 289 # passed to the command line). The sorting done earlier is 290 # utilized to implement the 2-stage removal. 291 if len(pkgs_to_remove) > 1: 292 self.pm.remove(pkgs_to_remove[:-1], False) 293 if len(pkgs_to_remove) > 0: 294 self.pm.remove([pkgs_to_remove[-1]], False) 295 296 if delayed_postinsts: 297 self._save_postinsts() 298 if image_rorfs: 299 bb.warn("There are post install scripts " 300 "in a read-only rootfs") 301 302 post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND") 303 execute_pre_post_process(self.d, post_uninstall_cmds) 304 305 runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", 306 True, False, self.d) 307 if not runtime_pkgmanage: 308 # Remove the package manager data files 309 self.pm.remove_packaging_data() 310 311 def _run_ldconfig(self): 312 if self.d.getVar('LDCONFIGDEPEND'): 313 bb.note("Executing: ldconfig -r " + self.image_rootfs + " -c new -v -X") 314 self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c', 315 'new', '-v', '-X']) 316 317 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 318 True, False, self.d) 319 ldconfig_in_features = bb.utils.contains("DISTRO_FEATURES", "ldconfig", 320 True, False, self.d) 321 if image_rorfs or not ldconfig_in_features: 322 ldconfig_cache_dir = os.path.join(self.image_rootfs, "var/cache/ldconfig") 323 if os.path.exists(ldconfig_cache_dir): 324 bb.note("Removing ldconfig auxiliary cache...") 325 shutil.rmtree(ldconfig_cache_dir) 326 327 def _check_for_kernel_modules(self, modules_dir): 328 for root, dirs, files in os.walk(modules_dir, topdown=True): 329 for name in files: 330 found_ko = name.endswith((".ko", ".ko.gz", ".ko.xz", ".ko.zst")) 331 if found_ko: 332 return found_ko 333 return False 334 335 def _generate_kernel_module_deps(self): 336 modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules') 337 # if we don't have any modules don't bother to do the depmod 338 if not self._check_for_kernel_modules(modules_dir): 339 bb.note("No Kernel Modules found, not running depmod") 340 return 341 342 pkgdatadir = self.d.getVar('PKGDATA_DIR') 343 344 # PKGDATA_DIR can include multiple kernels so we run depmod for each 345 # one of them. 346 for direntry in os.listdir(pkgdatadir): 347 match = re.match('(.*)-depmod', direntry) 348 if not match: 349 continue 350 kernel_package_name = match.group(1) 351 352 kernel_abi_ver_file = oe.path.join(pkgdatadir, direntry, kernel_package_name + '-abiversion') 353 if not os.path.exists(kernel_abi_ver_file): 354 bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file) 355 356 with open(kernel_abi_ver_file) as f: 357 kernel_ver = f.read().strip(' \n') 358 359 versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver) 360 361 bb.utils.mkdirhier(versioned_modules_dir) 362 363 bb.note("Running depmodwrapper for %s ..." % versioned_modules_dir) 364 if self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver, kernel_package_name]): 365 bb.fatal("Kernel modules dependency generation failed") 366 367 """ 368 Create devfs: 369 * IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file 370 * IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached 371 for in the BBPATH 372 If neither are specified then the default name of files/device_table-minimal.txt 373 is searched for in the BBPATH (same as the old version.) 374 """ 375 def _create_devfs(self): 376 devtable_list = [] 377 devtable = self.d.getVar('IMAGE_DEVICE_TABLE') 378 if devtable is not None: 379 devtable_list.append(devtable) 380 else: 381 devtables = self.d.getVar('IMAGE_DEVICE_TABLES') 382 if devtables is None: 383 devtables = 'files/device_table-minimal.txt' 384 for devtable in devtables.split(): 385 devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable)) 386 387 for devtable in devtable_list: 388 self._exec_shell_cmd(["makedevs", "-r", 389 self.image_rootfs, "-D", devtable]) 390 391 392def get_class_for_type(imgtype): 393 import importlib 394 mod = importlib.import_module('oe.package_manager.' + imgtype + '.rootfs') 395 return mod.PkgRootfs 396 397def variable_depends(d, manifest_dir=None): 398 img_type = d.getVar('IMAGE_PKGTYPE') 399 cls = get_class_for_type(img_type) 400 return cls._depends_list() 401 402def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None): 403 env_bkp = os.environ.copy() 404 405 img_type = d.getVar('IMAGE_PKGTYPE') 406 407 cls = get_class_for_type(img_type) 408 cls(d, manifest_dir, progress_reporter, logcatcher).create() 409 os.environ.clear() 410 os.environ.update(env_bkp) 411 412 413def image_list_installed_packages(d, rootfs_dir=None): 414 # Theres no rootfs for baremetal images 415 if bb.data.inherits_class('baremetal-image', d): 416 return "" 417 418 if not rootfs_dir: 419 rootfs_dir = d.getVar('IMAGE_ROOTFS') 420 421 img_type = d.getVar('IMAGE_PKGTYPE') 422 423 import importlib 424 cls = importlib.import_module('oe.package_manager.' + img_type) 425 return cls.PMPkgsList(d, rootfs_dir).list_pkgs() 426 427if __name__ == "__main__": 428 """ 429 We should be able to run this as a standalone script, from outside bitbake 430 environment. 431 """ 432 """ 433 TBD 434 """ 435