1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6from abc import ABCMeta, abstractmethod 7from oe.utils import execute_pre_post_process 8from oe.package_manager import * 9from oe.manifest import * 10import oe.path 11import shutil 12import os 13import subprocess 14import re 15 16class Rootfs(object, metaclass=ABCMeta): 17 """ 18 This is an abstract class. Do not instantiate this directly. 19 """ 20 21 def __init__(self, d, progress_reporter=None, logcatcher=None): 22 self.d = d 23 self.pm = None 24 self.image_rootfs = self.d.getVar('IMAGE_ROOTFS') 25 self.deploydir = self.d.getVar('IMGDEPLOYDIR') 26 self.progress_reporter = progress_reporter 27 self.logcatcher = logcatcher 28 29 self.install_order = Manifest.INSTALL_ORDER 30 31 @abstractmethod 32 def _create(self): 33 pass 34 35 @abstractmethod 36 def _get_delayed_postinsts(self): 37 pass 38 39 @abstractmethod 40 def _save_postinsts(self): 41 pass 42 43 @abstractmethod 44 def _log_check(self): 45 pass 46 47 def _log_check_common(self, type, match): 48 # Ignore any lines containing log_check to avoid recursion, and ignore 49 # lines beginning with a + since sh -x may emit code which isn't 50 # actually executed, but may contain error messages 51 excludes = [ 'log_check', r'^\+' ] 52 if hasattr(self, 'log_check_expected_regexes'): 53 excludes.extend(self.log_check_expected_regexes) 54 # Insert custom log_check excludes 55 excludes += [x for x in (self.d.getVar("IMAGE_LOG_CHECK_EXCLUDES") or "").split(" ") if x] 56 excludes = [re.compile(x) for x in excludes] 57 r = re.compile(match) 58 log_path = self.d.expand("${T}/log.do_rootfs") 59 messages = [] 60 with open(log_path, 'r') as log: 61 for line in log: 62 if self.logcatcher and self.logcatcher.contains(line.rstrip()): 63 continue 64 for ee in excludes: 65 m = ee.search(line) 66 if m: 67 break 68 if m: 69 continue 70 71 m = r.search(line) 72 if m: 73 messages.append('[log_check] %s' % line) 74 if messages: 75 if len(messages) == 1: 76 msg = '1 %s message' % type 77 else: 78 msg = '%d %s messages' % (len(messages), type) 79 msg = '[log_check] %s: found %s in the logfile:\n%s' % \ 80 (self.d.getVar('PN'), msg, ''.join(messages)) 81 if type == 'error': 82 bb.fatal(msg) 83 else: 84 bb.warn(msg) 85 86 def _log_check_warn(self): 87 self._log_check_common('warning', '^(warn|Warn|WARNING:)') 88 89 def _log_check_error(self): 90 self._log_check_common('error', self.log_check_regex) 91 92 def _insert_feed_uris(self): 93 if bb.utils.contains("IMAGE_FEATURES", "package-management", 94 True, False, self.d): 95 self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "", 96 self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "", 97 self.d.getVar('PACKAGE_FEED_ARCHS')) 98 99 100 """ 101 The _cleanup() method should be used to clean-up stuff that we don't really 102 want to end up on target. For example, in the case of RPM, the DB locks. 103 The method is called, once, at the end of create() method. 104 """ 105 @abstractmethod 106 def _cleanup(self): 107 pass 108 109 def _setup_dbg_rootfs(self, package_paths): 110 gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0' 111 if gen_debugfs != '1': 112 return 113 114 bb.note(" Renaming the original rootfs...") 115 try: 116 shutil.rmtree(self.image_rootfs + '-orig') 117 except: 118 pass 119 bb.utils.rename(self.image_rootfs, self.image_rootfs + '-orig') 120 121 bb.note(" Creating debug rootfs...") 122 bb.utils.mkdirhier(self.image_rootfs) 123 124 bb.note(" Copying back package database...") 125 for path in package_paths: 126 bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path)) 127 if os.path.isdir(self.image_rootfs + '-orig' + path): 128 shutil.copytree(self.image_rootfs + '-orig' + path, self.image_rootfs + path, symlinks=True) 129 elif os.path.isfile(self.image_rootfs + '-orig' + path): 130 shutil.copyfile(self.image_rootfs + '-orig' + path, self.image_rootfs + path) 131 132 # Copy files located in /usr/lib/debug or /usr/src/debug 133 for dir in ["/usr/lib/debug", "/usr/src/debug"]: 134 src = self.image_rootfs + '-orig' + dir 135 if os.path.exists(src): 136 dst = self.image_rootfs + dir 137 bb.utils.mkdirhier(os.path.dirname(dst)) 138 shutil.copytree(src, dst) 139 140 # Copy files with suffix '.debug' or located in '.debug' dir. 141 for root, dirs, files in os.walk(self.image_rootfs + '-orig'): 142 relative_dir = root[len(self.image_rootfs + '-orig'):] 143 for f in files: 144 if f.endswith('.debug') or '/.debug' in relative_dir: 145 bb.utils.mkdirhier(self.image_rootfs + relative_dir) 146 shutil.copy(os.path.join(root, f), 147 self.image_rootfs + relative_dir) 148 149 bb.note(" Install complementary '*-dbg' packages...") 150 self.pm.install_complementary('*-dbg') 151 152 if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg': 153 bb.note(" Install complementary '*-src' packages...") 154 self.pm.install_complementary('*-src') 155 156 """ 157 Install additional debug packages. Possibility to install additional packages, 158 which are not automatically installed as complementary package of 159 standard one, e.g. debug package of static libraries. 160 """ 161 extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS') 162 if extra_debug_pkgs: 163 bb.note(" Install extra debug packages...") 164 self.pm.install(extra_debug_pkgs.split(), True) 165 166 bb.note(" Removing package database...") 167 for path in package_paths: 168 if os.path.isdir(self.image_rootfs + path): 169 shutil.rmtree(self.image_rootfs + path) 170 elif os.path.isfile(self.image_rootfs + path): 171 os.remove(self.image_rootfs + path) 172 173 bb.note(" Rename debug rootfs...") 174 try: 175 shutil.rmtree(self.image_rootfs + '-dbg') 176 except: 177 pass 178 bb.utils.rename(self.image_rootfs, self.image_rootfs + '-dbg') 179 180 bb.note(" Restoring original rootfs...") 181 bb.utils.rename(self.image_rootfs + '-orig', self.image_rootfs) 182 183 def _exec_shell_cmd(self, cmd): 184 try: 185 subprocess.check_output(cmd, stderr=subprocess.STDOUT) 186 except subprocess.CalledProcessError as e: 187 return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output)) 188 189 return None 190 191 def create(self): 192 bb.note("###### Generate rootfs #######") 193 pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND") 194 post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND") 195 rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND') 196 197 def make_last(command, commands): 198 commands = commands.split() 199 if command in commands: 200 commands.remove(command) 201 commands.append(command) 202 return " ".join(commands) 203 204 # We want this to run as late as possible, in particular after 205 # systemd_sysusers_create and set_user_group. Using :append is not enough 206 post_process_cmds = make_last("tidy_shadowutils_files", post_process_cmds) 207 post_process_cmds = make_last("rootfs_reproducible", post_process_cmds) 208 209 execute_pre_post_process(self.d, pre_process_cmds) 210 211 if self.progress_reporter: 212 self.progress_reporter.next_stage() 213 214 # call the package manager dependent create method 215 self._create() 216 217 sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir') 218 bb.utils.mkdirhier(sysconfdir) 219 with open(sysconfdir + "/version", "w+") as ver: 220 ver.write(self.d.getVar('BUILDNAME') + "\n") 221 222 execute_pre_post_process(self.d, rootfs_post_install_cmds) 223 224 self.pm.run_intercepts() 225 226 execute_pre_post_process(self.d, post_process_cmds) 227 228 if self.progress_reporter: 229 self.progress_reporter.next_stage() 230 231 if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 232 True, False, self.d) and \ 233 not bb.utils.contains("IMAGE_FEATURES", 234 "read-only-rootfs-delayed-postinsts", 235 True, False, self.d): 236 delayed_postinsts = self._get_delayed_postinsts() 237 if delayed_postinsts is not None: 238 bb.fatal("The following packages could not be configured " 239 "offline and rootfs is read-only: %s" % 240 delayed_postinsts) 241 242 if self.d.getVar('USE_DEVFS') != "1": 243 self._create_devfs() 244 245 self._uninstall_unneeded() 246 247 if self.progress_reporter: 248 self.progress_reporter.next_stage() 249 250 self._insert_feed_uris() 251 252 self._run_ldconfig() 253 254 if self.d.getVar('USE_DEPMOD') != "0": 255 self._generate_kernel_module_deps() 256 257 self._cleanup() 258 self._log_check() 259 260 if self.progress_reporter: 261 self.progress_reporter.next_stage() 262 263 264 def _uninstall_unneeded(self): 265 # Remove the run-postinsts package if no delayed postinsts are found 266 delayed_postinsts = self._get_delayed_postinsts() 267 if delayed_postinsts is None: 268 if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")) or os.path.exists(self.d.expand("${IMAGE_ROOTFS}${systemd_system_unitdir}/run-postinsts.service")): 269 self.pm.remove(["run-postinsts"]) 270 271 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 272 True, False, self.d) and \ 273 not bb.utils.contains("IMAGE_FEATURES", 274 "read-only-rootfs-delayed-postinsts", 275 True, False, self.d) 276 277 image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE') 278 279 if image_rorfs or image_rorfs_force == "1": 280 # Remove components that we don't need if it's a read-only rootfs 281 unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split() 282 pkgs_installed = image_list_installed_packages(self.d) 283 # Make sure update-alternatives is removed last. This is 284 # because its database has to available while uninstalling 285 # other packages, allowing alternative symlinks of packages 286 # to be uninstalled or to be managed correctly otherwise. 287 provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives") 288 pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider) 289 290 # update-alternatives provider is removed in its own remove() 291 # call because all package managers do not guarantee the packages 292 # are removed in the order they given in the list (which is 293 # passed to the command line). The sorting done earlier is 294 # utilized to implement the 2-stage removal. 295 if len(pkgs_to_remove) > 1: 296 self.pm.remove(pkgs_to_remove[:-1], False) 297 if len(pkgs_to_remove) > 0: 298 self.pm.remove([pkgs_to_remove[-1]], False) 299 300 if delayed_postinsts: 301 self._save_postinsts() 302 if image_rorfs: 303 bb.warn("There are post install scripts " 304 "in a read-only rootfs") 305 306 post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND") 307 execute_pre_post_process(self.d, post_uninstall_cmds) 308 309 runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", 310 True, False, self.d) 311 if not runtime_pkgmanage: 312 # Remove the package manager data files 313 self.pm.remove_packaging_data() 314 315 def _run_ldconfig(self): 316 if self.d.getVar('LDCONFIGDEPEND'): 317 bb.note("Executing: ldconfig -r " + self.image_rootfs + " -c new -v -X") 318 self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c', 319 'new', '-v', '-X']) 320 321 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 322 True, False, self.d) 323 ldconfig_in_features = bb.utils.contains("DISTRO_FEATURES", "ldconfig", 324 True, False, self.d) 325 if image_rorfs or not ldconfig_in_features: 326 ldconfig_cache_dir = os.path.join(self.image_rootfs, "var/cache/ldconfig") 327 if os.path.exists(ldconfig_cache_dir): 328 bb.note("Removing ldconfig auxiliary cache...") 329 shutil.rmtree(ldconfig_cache_dir) 330 331 def _check_for_kernel_modules(self, modules_dir): 332 for root, dirs, files in os.walk(modules_dir, topdown=True): 333 for name in files: 334 found_ko = name.endswith((".ko", ".ko.gz", ".ko.xz", ".ko.zst")) 335 if found_ko: 336 return found_ko 337 return False 338 339 def _generate_kernel_module_deps(self): 340 modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules') 341 # if we don't have any modules don't bother to do the depmod 342 if not self._check_for_kernel_modules(modules_dir): 343 bb.note("No Kernel Modules found, not running depmod") 344 return 345 346 pkgdatadir = self.d.getVar('PKGDATA_DIR') 347 348 # PKGDATA_DIR can include multiple kernels so we run depmod for each 349 # one of them. 350 for direntry in os.listdir(pkgdatadir): 351 match = re.match('(.*)-depmod', direntry) 352 if not match: 353 continue 354 kernel_package_name = match.group(1) 355 356 kernel_abi_ver_file = oe.path.join(pkgdatadir, direntry, kernel_package_name + '-abiversion') 357 if not os.path.exists(kernel_abi_ver_file): 358 bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file) 359 360 with open(kernel_abi_ver_file) as f: 361 kernel_ver = f.read().strip(' \n') 362 363 versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver) 364 365 bb.utils.mkdirhier(versioned_modules_dir) 366 367 bb.note("Running depmodwrapper for %s ..." % versioned_modules_dir) 368 if self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver, kernel_package_name]): 369 bb.fatal("Kernel modules dependency generation failed") 370 371 """ 372 Create devfs: 373 * IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file 374 * IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached 375 for in the BBPATH 376 If neither are specified then the default name of files/device_table-minimal.txt 377 is searched for in the BBPATH (same as the old version.) 378 """ 379 def _create_devfs(self): 380 devtable_list = [] 381 devtable = self.d.getVar('IMAGE_DEVICE_TABLE') 382 if devtable is not None: 383 devtable_list.append(devtable) 384 else: 385 devtables = self.d.getVar('IMAGE_DEVICE_TABLES') 386 if devtables is None: 387 devtables = 'files/device_table-minimal.txt' 388 for devtable in devtables.split(): 389 devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable)) 390 391 for devtable in devtable_list: 392 self._exec_shell_cmd(["makedevs", "-r", 393 self.image_rootfs, "-D", devtable]) 394 395 396def get_class_for_type(imgtype): 397 import importlib 398 mod = importlib.import_module('oe.package_manager.' + imgtype + '.rootfs') 399 return mod.PkgRootfs 400 401def variable_depends(d, manifest_dir=None): 402 img_type = d.getVar('IMAGE_PKGTYPE') 403 cls = get_class_for_type(img_type) 404 return cls._depends_list() 405 406def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None): 407 env_bkp = os.environ.copy() 408 409 img_type = d.getVar('IMAGE_PKGTYPE') 410 411 cls = get_class_for_type(img_type) 412 cls(d, manifest_dir, progress_reporter, logcatcher).create() 413 os.environ.clear() 414 os.environ.update(env_bkp) 415 416 417def image_list_installed_packages(d, rootfs_dir=None): 418 # Theres no rootfs for baremetal images 419 if bb.data.inherits_class('baremetal-image', d): 420 return "" 421 422 if not rootfs_dir: 423 rootfs_dir = d.getVar('IMAGE_ROOTFS') 424 425 img_type = d.getVar('IMAGE_PKGTYPE') 426 427 import importlib 428 cls = importlib.import_module('oe.package_manager.' + img_type) 429 return cls.PMPkgsList(d, rootfs_dir).list_pkgs() 430 431if __name__ == "__main__": 432 """ 433 We should be able to run this as a standalone script, from outside bitbake 434 environment. 435 """ 436 """ 437 TBD 438 """ 439