1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6from abc import ABCMeta, abstractmethod 7from oe.utils import execute_pre_post_process 8from oe.package_manager import * 9from oe.manifest import * 10import oe.path 11import shutil 12import os 13import subprocess 14import re 15 16class Rootfs(object, metaclass=ABCMeta): 17 """ 18 This is an abstract class. Do not instantiate this directly. 19 """ 20 21 def __init__(self, d, progress_reporter=None, logcatcher=None): 22 self.d = d 23 self.pm = None 24 self.image_rootfs = self.d.getVar('IMAGE_ROOTFS') 25 self.deploydir = self.d.getVar('IMGDEPLOYDIR') 26 self.progress_reporter = progress_reporter 27 self.logcatcher = logcatcher 28 29 self.install_order = Manifest.INSTALL_ORDER 30 31 @abstractmethod 32 def _create(self): 33 pass 34 35 @abstractmethod 36 def _get_delayed_postinsts(self): 37 pass 38 39 @abstractmethod 40 def _save_postinsts(self): 41 pass 42 43 @abstractmethod 44 def _log_check(self): 45 pass 46 47 def _log_check_common(self, type, match): 48 # Ignore any lines containing log_check to avoid recursion, and ignore 49 # lines beginning with a + since sh -x may emit code which isn't 50 # actually executed, but may contain error messages 51 excludes = [ 'log_check', r'^\+' ] 52 if hasattr(self, 'log_check_expected_regexes'): 53 excludes.extend(self.log_check_expected_regexes) 54 # Insert custom log_check excludes 55 excludes += [x for x in (self.d.getVar("IMAGE_LOG_CHECK_EXCLUDES") or "").split(" ") if x] 56 excludes = [re.compile(x) for x in excludes] 57 r = re.compile(match) 58 log_path = self.d.expand("${T}/log.do_rootfs") 59 messages = [] 60 with open(log_path, 'r') as log: 61 for line in log: 62 if self.logcatcher and self.logcatcher.contains(line.rstrip()): 63 continue 64 for ee in excludes: 65 m = ee.search(line) 66 if m: 67 break 68 if m: 69 continue 70 71 m = r.search(line) 72 if m: 73 messages.append('[log_check] %s' % line) 74 if messages: 75 if len(messages) == 1: 76 msg = '1 %s message' % type 77 else: 78 msg = '%d %s messages' % (len(messages), type) 79 msg = '[log_check] %s: found %s in the logfile:\n%s' % \ 80 (self.d.getVar('PN'), msg, ''.join(messages)) 81 if type == 'error': 82 bb.fatal(msg) 83 else: 84 bb.warn(msg) 85 86 def _log_check_warn(self): 87 self._log_check_common('warning', '^(warn|Warn|WARNING:)') 88 89 def _log_check_error(self): 90 self._log_check_common('error', self.log_check_regex) 91 92 def _insert_feed_uris(self): 93 if bb.utils.contains("IMAGE_FEATURES", "package-management", 94 True, False, self.d): 95 self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "", 96 self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "", 97 self.d.getVar('PACKAGE_FEED_ARCHS')) 98 99 100 """ 101 The _cleanup() method should be used to clean-up stuff that we don't really 102 want to end up on target. For example, in the case of RPM, the DB locks. 103 The method is called, once, at the end of create() method. 104 """ 105 @abstractmethod 106 def _cleanup(self): 107 pass 108 109 def _setup_dbg_rootfs(self, dirs): 110 gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0' 111 if gen_debugfs != '1': 112 return 113 114 bb.note(" Renaming the original rootfs...") 115 try: 116 shutil.rmtree(self.image_rootfs + '-orig') 117 except: 118 pass 119 bb.utils.rename(self.image_rootfs, self.image_rootfs + '-orig') 120 121 bb.note(" Creating debug rootfs...") 122 bb.utils.mkdirhier(self.image_rootfs) 123 124 bb.note(" Copying back package database...") 125 for dir in dirs: 126 if not os.path.isdir(self.image_rootfs + '-orig' + dir): 127 continue 128 bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir)) 129 shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir, symlinks=True) 130 131 # Copy files located in /usr/lib/debug or /usr/src/debug 132 for dir in ["/usr/lib/debug", "/usr/src/debug"]: 133 src = self.image_rootfs + '-orig' + dir 134 if os.path.exists(src): 135 dst = self.image_rootfs + dir 136 bb.utils.mkdirhier(os.path.dirname(dst)) 137 shutil.copytree(src, dst) 138 139 # Copy files with suffix '.debug' or located in '.debug' dir. 140 for root, dirs, files in os.walk(self.image_rootfs + '-orig'): 141 relative_dir = root[len(self.image_rootfs + '-orig'):] 142 for f in files: 143 if f.endswith('.debug') or '/.debug' in relative_dir: 144 bb.utils.mkdirhier(self.image_rootfs + relative_dir) 145 shutil.copy(os.path.join(root, f), 146 self.image_rootfs + relative_dir) 147 148 bb.note(" Install complementary '*-dbg' packages...") 149 self.pm.install_complementary('*-dbg') 150 151 if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg': 152 bb.note(" Install complementary '*-src' packages...") 153 self.pm.install_complementary('*-src') 154 155 """ 156 Install additional debug packages. Possibility to install additional packages, 157 which are not automatically installed as complementary package of 158 standard one, e.g. debug package of static libraries. 159 """ 160 extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS') 161 if extra_debug_pkgs: 162 bb.note(" Install extra debug packages...") 163 self.pm.install(extra_debug_pkgs.split(), True) 164 165 bb.note(" Rename debug rootfs...") 166 try: 167 shutil.rmtree(self.image_rootfs + '-dbg') 168 except: 169 pass 170 bb.utils.rename(self.image_rootfs, self.image_rootfs + '-dbg') 171 172 bb.note(" Restoring original rootfs...") 173 bb.utils.rename(self.image_rootfs + '-orig', self.image_rootfs) 174 175 def _exec_shell_cmd(self, cmd): 176 try: 177 subprocess.check_output(cmd, stderr=subprocess.STDOUT) 178 except subprocess.CalledProcessError as e: 179 return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output)) 180 181 return None 182 183 def create(self): 184 bb.note("###### Generate rootfs #######") 185 pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND") 186 post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND") 187 rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND') 188 189 execute_pre_post_process(self.d, pre_process_cmds) 190 191 if self.progress_reporter: 192 self.progress_reporter.next_stage() 193 194 # call the package manager dependent create method 195 self._create() 196 197 sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir') 198 bb.utils.mkdirhier(sysconfdir) 199 with open(sysconfdir + "/version", "w+") as ver: 200 ver.write(self.d.getVar('BUILDNAME') + "\n") 201 202 execute_pre_post_process(self.d, rootfs_post_install_cmds) 203 204 self.pm.run_intercepts() 205 206 execute_pre_post_process(self.d, post_process_cmds) 207 208 if self.progress_reporter: 209 self.progress_reporter.next_stage() 210 211 if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 212 True, False, self.d) and \ 213 not bb.utils.contains("IMAGE_FEATURES", 214 "read-only-rootfs-delayed-postinsts", 215 True, False, self.d): 216 delayed_postinsts = self._get_delayed_postinsts() 217 if delayed_postinsts is not None: 218 bb.fatal("The following packages could not be configured " 219 "offline and rootfs is read-only: %s" % 220 delayed_postinsts) 221 222 if self.d.getVar('USE_DEVFS') != "1": 223 self._create_devfs() 224 225 self._uninstall_unneeded() 226 227 if self.progress_reporter: 228 self.progress_reporter.next_stage() 229 230 self._insert_feed_uris() 231 232 self._run_ldconfig() 233 234 if self.d.getVar('USE_DEPMOD') != "0": 235 self._generate_kernel_module_deps() 236 237 self._cleanup() 238 self._log_check() 239 240 if self.progress_reporter: 241 self.progress_reporter.next_stage() 242 243 244 def _uninstall_unneeded(self): 245 # Remove the run-postinsts package if no delayed postinsts are found 246 delayed_postinsts = self._get_delayed_postinsts() 247 if delayed_postinsts is None: 248 if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")) or os.path.exists(self.d.expand("${IMAGE_ROOTFS}${systemd_system_unitdir}/run-postinsts.service")): 249 self.pm.remove(["run-postinsts"]) 250 251 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 252 True, False, self.d) 253 image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE') 254 255 if image_rorfs or image_rorfs_force == "1": 256 # Remove components that we don't need if it's a read-only rootfs 257 unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split() 258 pkgs_installed = image_list_installed_packages(self.d) 259 # Make sure update-alternatives is removed last. This is 260 # because its database has to available while uninstalling 261 # other packages, allowing alternative symlinks of packages 262 # to be uninstalled or to be managed correctly otherwise. 263 provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives") 264 pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider) 265 266 # update-alternatives provider is removed in its own remove() 267 # call because all package managers do not guarantee the packages 268 # are removed in the order they given in the list (which is 269 # passed to the command line). The sorting done earlier is 270 # utilized to implement the 2-stage removal. 271 if len(pkgs_to_remove) > 1: 272 self.pm.remove(pkgs_to_remove[:-1], False) 273 if len(pkgs_to_remove) > 0: 274 self.pm.remove([pkgs_to_remove[-1]], False) 275 276 if delayed_postinsts: 277 self._save_postinsts() 278 if image_rorfs: 279 bb.warn("There are post install scripts " 280 "in a read-only rootfs") 281 282 post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND") 283 execute_pre_post_process(self.d, post_uninstall_cmds) 284 285 runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", 286 True, False, self.d) 287 if not runtime_pkgmanage: 288 # Remove the package manager data files 289 self.pm.remove_packaging_data() 290 291 def _run_ldconfig(self): 292 if self.d.getVar('LDCONFIGDEPEND'): 293 bb.note("Executing: ldconfig -r " + self.image_rootfs + " -c new -v -X") 294 self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c', 295 'new', '-v', '-X']) 296 297 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 298 True, False, self.d) 299 ldconfig_in_features = bb.utils.contains("DISTRO_FEATURES", "ldconfig", 300 True, False, self.d) 301 if image_rorfs or not ldconfig_in_features: 302 ldconfig_cache_dir = os.path.join(self.image_rootfs, "var/cache/ldconfig") 303 if os.path.exists(ldconfig_cache_dir): 304 bb.note("Removing ldconfig auxiliary cache...") 305 shutil.rmtree(ldconfig_cache_dir) 306 307 def _check_for_kernel_modules(self, modules_dir): 308 for root, dirs, files in os.walk(modules_dir, topdown=True): 309 for name in files: 310 found_ko = name.endswith((".ko", ".ko.gz", ".ko.xz", ".ko.zst")) 311 if found_ko: 312 return found_ko 313 return False 314 315 def _generate_kernel_module_deps(self): 316 modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules') 317 # if we don't have any modules don't bother to do the depmod 318 if not self._check_for_kernel_modules(modules_dir): 319 bb.note("No Kernel Modules found, not running depmod") 320 return 321 322 pkgdatadir = self.d.getVar('PKGDATA_DIR') 323 324 # PKGDATA_DIR can include multiple kernels so we run depmod for each 325 # one of them. 326 for direntry in os.listdir(pkgdatadir): 327 match = re.match('(.*)-depmod', direntry) 328 if not match: 329 continue 330 kernel_package_name = match.group(1) 331 332 kernel_abi_ver_file = oe.path.join(pkgdatadir, direntry, kernel_package_name + '-abiversion') 333 if not os.path.exists(kernel_abi_ver_file): 334 bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file) 335 336 with open(kernel_abi_ver_file) as f: 337 kernel_ver = f.read().strip(' \n') 338 339 versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver) 340 341 bb.utils.mkdirhier(versioned_modules_dir) 342 343 bb.note("Running depmodwrapper for %s ..." % versioned_modules_dir) 344 self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver, kernel_package_name]) 345 346 """ 347 Create devfs: 348 * IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file 349 * IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached 350 for in the BBPATH 351 If neither are specified then the default name of files/device_table-minimal.txt 352 is searched for in the BBPATH (same as the old version.) 353 """ 354 def _create_devfs(self): 355 devtable_list = [] 356 devtable = self.d.getVar('IMAGE_DEVICE_TABLE') 357 if devtable is not None: 358 devtable_list.append(devtable) 359 else: 360 devtables = self.d.getVar('IMAGE_DEVICE_TABLES') 361 if devtables is None: 362 devtables = 'files/device_table-minimal.txt' 363 for devtable in devtables.split(): 364 devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable)) 365 366 for devtable in devtable_list: 367 self._exec_shell_cmd(["makedevs", "-r", 368 self.image_rootfs, "-D", devtable]) 369 370 371def get_class_for_type(imgtype): 372 import importlib 373 mod = importlib.import_module('oe.package_manager.' + imgtype + '.rootfs') 374 return mod.PkgRootfs 375 376def variable_depends(d, manifest_dir=None): 377 img_type = d.getVar('IMAGE_PKGTYPE') 378 cls = get_class_for_type(img_type) 379 return cls._depends_list() 380 381def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None): 382 env_bkp = os.environ.copy() 383 384 img_type = d.getVar('IMAGE_PKGTYPE') 385 386 cls = get_class_for_type(img_type) 387 cls(d, manifest_dir, progress_reporter, logcatcher).create() 388 os.environ.clear() 389 os.environ.update(env_bkp) 390 391 392def image_list_installed_packages(d, rootfs_dir=None): 393 # Theres no rootfs for baremetal images 394 if bb.data.inherits_class('baremetal-image', d): 395 return "" 396 397 if not rootfs_dir: 398 rootfs_dir = d.getVar('IMAGE_ROOTFS') 399 400 img_type = d.getVar('IMAGE_PKGTYPE') 401 402 import importlib 403 cls = importlib.import_module('oe.package_manager.' + img_type) 404 return cls.PMPkgsList(d, rootfs_dir).list_pkgs() 405 406if __name__ == "__main__": 407 """ 408 We should be able to run this as a standalone script, from outside bitbake 409 environment. 410 """ 411 """ 412 TBD 413 """ 414