1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4 5from abc import ABCMeta, abstractmethod 6import os 7import glob 8import subprocess 9import shutil 10import re 11import collections 12import bb 13import tempfile 14import oe.utils 15import oe.path 16import string 17from oe.gpg_sign import get_signer 18import hashlib 19import fnmatch 20 21# this can be used by all PM backends to create the index files in parallel 22def create_index(arg): 23 index_cmd = arg 24 25 bb.note("Executing '%s' ..." % index_cmd) 26 result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") 27 if result: 28 bb.note(result) 29 30def opkg_query(cmd_output): 31 """ 32 This method parse the output from the package managerand return 33 a dictionary with the information of the packages. This is used 34 when the packages are in deb or ipk format. 35 """ 36 verregex = re.compile(r' \([=<>]* [^ )]*\)') 37 output = dict() 38 pkg = "" 39 arch = "" 40 ver = "" 41 filename = "" 42 dep = [] 43 prov = [] 44 pkgarch = "" 45 for line in cmd_output.splitlines()+['']: 46 line = line.rstrip() 47 if ':' in line: 48 if line.startswith("Package: "): 49 pkg = line.split(": ")[1] 50 elif line.startswith("Architecture: "): 51 arch = line.split(": ")[1] 52 elif line.startswith("Version: "): 53 ver = line.split(": ")[1] 54 elif line.startswith("File: ") or line.startswith("Filename:"): 55 filename = line.split(": ")[1] 56 if "/" in filename: 57 filename = os.path.basename(filename) 58 elif line.startswith("Depends: "): 59 depends = verregex.sub('', line.split(": ")[1]) 60 for depend in depends.split(", "): 61 dep.append(depend) 62 elif line.startswith("Recommends: "): 63 recommends = verregex.sub('', line.split(": ")[1]) 64 for recommend in recommends.split(", "): 65 dep.append("%s [REC]" % recommend) 66 elif line.startswith("PackageArch: "): 67 pkgarch = line.split(": ")[1] 68 elif line.startswith("Provides: "): 69 provides = verregex.sub('', line.split(": ")[1]) 70 for provide in provides.split(", "): 71 prov.append(provide) 72 73 # When there is a blank line save the package information 74 elif not line: 75 # IPK doesn't include the filename 76 if not filename: 77 filename = "%s_%s_%s.ipk" % (pkg, ver, arch) 78 if pkg: 79 output[pkg] = {"arch":arch, "ver":ver, 80 "filename":filename, "deps": dep, "pkgarch":pkgarch, "provs": prov} 81 pkg = "" 82 arch = "" 83 ver = "" 84 filename = "" 85 dep = [] 86 prov = [] 87 pkgarch = "" 88 89 return output 90 91def failed_postinsts_abort(pkgs, log_path): 92 bb.fatal("""Postinstall scriptlets of %s have failed. If the intention is to defer them to first boot, 93then please place them into pkg_postinst_ontarget:${PN} (). 94Deferring to first boot via 'exit 1' is no longer supported. 95Details of the failure are in %s.""" %(pkgs, log_path)) 96 97def generate_locale_archive(d, rootfs, target_arch, localedir): 98 # Pretty sure we don't need this for locale archive generation but 99 # keeping it to be safe... 100 locale_arch_options = { \ 101 "arc": ["--uint32-align=4", "--little-endian"], 102 "arceb": ["--uint32-align=4", "--big-endian"], 103 "arm": ["--uint32-align=4", "--little-endian"], 104 "armeb": ["--uint32-align=4", "--big-endian"], 105 "aarch64": ["--uint32-align=4", "--little-endian"], 106 "aarch64_be": ["--uint32-align=4", "--big-endian"], 107 "sh4": ["--uint32-align=4", "--big-endian"], 108 "powerpc": ["--uint32-align=4", "--big-endian"], 109 "powerpc64": ["--uint32-align=4", "--big-endian"], 110 "powerpc64le": ["--uint32-align=4", "--little-endian"], 111 "mips": ["--uint32-align=4", "--big-endian"], 112 "mipsisa32r6": ["--uint32-align=4", "--big-endian"], 113 "mips64": ["--uint32-align=4", "--big-endian"], 114 "mipsisa64r6": ["--uint32-align=4", "--big-endian"], 115 "mipsel": ["--uint32-align=4", "--little-endian"], 116 "mipsisa32r6el": ["--uint32-align=4", "--little-endian"], 117 "mips64el": ["--uint32-align=4", "--little-endian"], 118 "mipsisa64r6el": ["--uint32-align=4", "--little-endian"], 119 "riscv64": ["--uint32-align=4", "--little-endian"], 120 "riscv32": ["--uint32-align=4", "--little-endian"], 121 "i586": ["--uint32-align=4", "--little-endian"], 122 "i686": ["--uint32-align=4", "--little-endian"], 123 "x86_64": ["--uint32-align=4", "--little-endian"] 124 } 125 if target_arch in locale_arch_options: 126 arch_options = locale_arch_options[target_arch] 127 else: 128 bb.error("locale_arch_options not found for target_arch=" + target_arch) 129 bb.fatal("unknown arch:" + target_arch + " for locale_arch_options") 130 131 # Need to set this so cross-localedef knows where the archive is 132 env = dict(os.environ) 133 env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive") 134 135 for name in sorted(os.listdir(localedir)): 136 path = os.path.join(localedir, name) 137 if os.path.isdir(path): 138 cmd = ["cross-localedef", "--verbose"] 139 cmd += arch_options 140 cmd += ["--add-to-archive", path] 141 subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT) 142 143class Indexer(object, metaclass=ABCMeta): 144 def __init__(self, d, deploy_dir): 145 self.d = d 146 self.deploy_dir = deploy_dir 147 148 @abstractmethod 149 def write_index(self): 150 pass 151 152class PkgsList(object, metaclass=ABCMeta): 153 def __init__(self, d, rootfs_dir): 154 self.d = d 155 self.rootfs_dir = rootfs_dir 156 157 @abstractmethod 158 def list_pkgs(self): 159 pass 160 161class PackageManager(object, metaclass=ABCMeta): 162 """ 163 This is an abstract class. Do not instantiate this directly. 164 """ 165 166 def __init__(self, d, target_rootfs): 167 self.d = d 168 self.target_rootfs = target_rootfs 169 self.deploy_dir = None 170 self.deploy_lock = None 171 self._initialize_intercepts() 172 173 def _initialize_intercepts(self): 174 bb.note("Initializing intercept dir for %s" % self.target_rootfs) 175 # As there might be more than one instance of PackageManager operating at the same time 176 # we need to isolate the intercept_scripts directories from each other, 177 # hence the ugly hash digest in dir name. 178 self.intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts-%s" % 179 (hashlib.sha256(self.target_rootfs.encode()).hexdigest())) 180 181 postinst_intercepts = (self.d.getVar("POSTINST_INTERCEPTS") or "").split() 182 if not postinst_intercepts: 183 postinst_intercepts_path = self.d.getVar("POSTINST_INTERCEPTS_PATH") 184 if not postinst_intercepts_path: 185 postinst_intercepts_path = self.d.getVar("POSTINST_INTERCEPTS_DIR") or self.d.expand("${COREBASE}/scripts/postinst-intercepts") 186 postinst_intercepts = oe.path.which_wild('*', postinst_intercepts_path) 187 188 bb.debug(1, 'Collected intercepts:\n%s' % ''.join(' %s\n' % i for i in postinst_intercepts)) 189 bb.utils.remove(self.intercepts_dir, True) 190 bb.utils.mkdirhier(self.intercepts_dir) 191 for intercept in postinst_intercepts: 192 shutil.copy(intercept, os.path.join(self.intercepts_dir, os.path.basename(intercept))) 193 194 @abstractmethod 195 def _handle_intercept_failure(self, failed_script): 196 pass 197 198 def _postpone_to_first_boot(self, postinst_intercept_hook): 199 with open(postinst_intercept_hook) as intercept: 200 registered_pkgs = None 201 for line in intercept.read().split("\n"): 202 m = re.match(r"^##PKGS:(.*)", line) 203 if m is not None: 204 registered_pkgs = m.group(1).strip() 205 break 206 207 if registered_pkgs is not None: 208 bb.note("If an image is being built, the postinstalls for the following packages " 209 "will be postponed for first boot: %s" % 210 registered_pkgs) 211 212 # call the backend dependent handler 213 self._handle_intercept_failure(registered_pkgs) 214 215 216 def run_intercepts(self, populate_sdk=None): 217 intercepts_dir = self.intercepts_dir 218 219 bb.note("Running intercept scripts:") 220 os.environ['D'] = self.target_rootfs 221 os.environ['STAGING_DIR_NATIVE'] = self.d.getVar('STAGING_DIR_NATIVE') 222 for script in os.listdir(intercepts_dir): 223 script_full = os.path.join(intercepts_dir, script) 224 225 if script == "postinst_intercept" or not os.access(script_full, os.X_OK): 226 continue 227 228 # we do not want to run any multilib variant of this 229 if script.startswith("delay_to_first_boot"): 230 self._postpone_to_first_boot(script_full) 231 continue 232 233 if populate_sdk == 'host' and self.d.getVar('SDK_OS') == 'mingw32': 234 bb.note("The postinstall intercept hook '%s' could not be executed due to missing wine support, details in %s/log.do_%s" 235 % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) 236 continue 237 238 bb.note("> Executing %s intercept ..." % script) 239 240 try: 241 output = subprocess.check_output(script_full, stderr=subprocess.STDOUT) 242 if output: bb.note(output.decode("utf-8")) 243 except subprocess.CalledProcessError as e: 244 bb.note("Exit code %d. Output:\n%s" % (e.returncode, e.output.decode("utf-8"))) 245 if populate_sdk == 'host': 246 bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) 247 elif populate_sdk == 'target': 248 if "qemuwrapper: qemu usermode is not supported" in e.output.decode("utf-8"): 249 bb.note("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s" 250 % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) 251 else: 252 bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) 253 else: 254 if "qemuwrapper: qemu usermode is not supported" in e.output.decode("utf-8"): 255 bb.note("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s" 256 % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) 257 self._postpone_to_first_boot(script_full) 258 else: 259 bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) 260 261 @abstractmethod 262 def update(self): 263 """ 264 Update the package manager package database. 265 """ 266 pass 267 268 @abstractmethod 269 def install(self, pkgs, attempt_only=False): 270 """ 271 Install a list of packages. 'pkgs' is a list object. If 'attempt_only' is 272 True, installation failures are ignored. 273 """ 274 pass 275 276 @abstractmethod 277 def remove(self, pkgs, with_dependencies=True): 278 """ 279 Remove a list of packages. 'pkgs' is a list object. If 'with_dependencies' 280 is False, then any dependencies are left in place. 281 """ 282 pass 283 284 @abstractmethod 285 def write_index(self): 286 """ 287 This function creates the index files 288 """ 289 pass 290 291 @abstractmethod 292 def remove_packaging_data(self): 293 pass 294 295 @abstractmethod 296 def list_installed(self): 297 pass 298 299 @abstractmethod 300 def extract(self, pkg): 301 """ 302 Returns the path to a tmpdir where resides the contents of a package. 303 Deleting the tmpdir is responsability of the caller. 304 """ 305 pass 306 307 @abstractmethod 308 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): 309 """ 310 Add remote package feeds into repository manager configuration. The parameters 311 for the feeds are set by feed_uris, feed_base_paths and feed_archs. 312 See http://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-PACKAGE_FEED_URIS 313 for their description. 314 """ 315 pass 316 317 def install_glob(self, globs, sdk=False): 318 """ 319 Install all packages that match a glob. 320 """ 321 # TODO don't have sdk here but have a property on the superclass 322 # (and respect in install_complementary) 323 if sdk: 324 pkgdatadir = self.d.expand("${TMPDIR}/pkgdata/${SDK_SYS}") 325 else: 326 pkgdatadir = self.d.getVar("PKGDATA_DIR") 327 328 try: 329 bb.note("Installing globbed packages...") 330 cmd = ["oe-pkgdata-util", "-p", pkgdatadir, "list-pkgs", globs] 331 bb.note('Running %s' % cmd) 332 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 333 stdout, stderr = proc.communicate() 334 if stderr: bb.note(stderr.decode("utf-8")) 335 pkgs = stdout.decode("utf-8") 336 self.install(pkgs.split(), attempt_only=True) 337 except subprocess.CalledProcessError as e: 338 # Return code 1 means no packages matched 339 if e.returncode != 1: 340 bb.fatal("Could not compute globbed packages list. Command " 341 "'%s' returned %d:\n%s" % 342 (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) 343 344 def install_complementary(self, globs=None): 345 """ 346 Install complementary packages based upon the list of currently installed 347 packages e.g. locales, *-dev, *-dbg, etc. Note: every backend needs to 348 call this function explicitly after the normal package installation. 349 """ 350 if globs is None: 351 globs = self.d.getVar('IMAGE_INSTALL_COMPLEMENTARY') 352 split_linguas = set() 353 354 for translation in self.d.getVar('IMAGE_LINGUAS').split(): 355 split_linguas.add(translation) 356 split_linguas.add(translation.split('-')[0]) 357 358 split_linguas = sorted(split_linguas) 359 360 for lang in split_linguas: 361 globs += " *-locale-%s" % lang 362 for complementary_linguas in (self.d.getVar('IMAGE_LINGUAS_COMPLEMENTARY') or "").split(): 363 globs += (" " + complementary_linguas) % lang 364 365 if globs is None: 366 return 367 368 # we need to write the list of installed packages to a file because the 369 # oe-pkgdata-util reads it from a file 370 with tempfile.NamedTemporaryFile(mode="w+", prefix="installed-pkgs") as installed_pkgs: 371 pkgs = self.list_installed() 372 373 provided_pkgs = set() 374 for pkg in pkgs.values(): 375 provided_pkgs |= set(pkg.get('provs', [])) 376 377 output = oe.utils.format_pkg_list(pkgs, "arch") 378 installed_pkgs.write(output) 379 installed_pkgs.flush() 380 381 cmd = ["oe-pkgdata-util", 382 "-p", self.d.getVar('PKGDATA_DIR'), "glob", installed_pkgs.name, 383 globs] 384 exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY') 385 if exclude: 386 cmd.extend(['--exclude=' + '|'.join(exclude.split())]) 387 try: 388 bb.note('Running %s' % cmd) 389 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 390 stdout, stderr = proc.communicate() 391 if stderr: bb.note(stderr.decode("utf-8")) 392 complementary_pkgs = stdout.decode("utf-8") 393 complementary_pkgs = set(complementary_pkgs.split()) 394 skip_pkgs = sorted(complementary_pkgs & provided_pkgs) 395 install_pkgs = sorted(complementary_pkgs - provided_pkgs) 396 bb.note("Installing complementary packages ... %s (skipped already provided packages %s)" % ( 397 ' '.join(install_pkgs), 398 ' '.join(skip_pkgs))) 399 self.install(install_pkgs) 400 except subprocess.CalledProcessError as e: 401 bb.fatal("Could not compute complementary packages list. Command " 402 "'%s' returned %d:\n%s" % 403 (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) 404 405 if self.d.getVar('IMAGE_LOCALES_ARCHIVE') == '1': 406 target_arch = self.d.getVar('TARGET_ARCH') 407 localedir = oe.path.join(self.target_rootfs, self.d.getVar("libdir"), "locale") 408 if os.path.exists(localedir) and os.listdir(localedir): 409 generate_locale_archive(self.d, self.target_rootfs, target_arch, localedir) 410 # And now delete the binary locales 411 self.remove(fnmatch.filter(self.list_installed(), "glibc-binary-localedata-*"), False) 412 413 def deploy_dir_lock(self): 414 if self.deploy_dir is None: 415 raise RuntimeError("deploy_dir is not set!") 416 417 lock_file_name = os.path.join(self.deploy_dir, "deploy.lock") 418 419 self.deploy_lock = bb.utils.lockfile(lock_file_name) 420 421 def deploy_dir_unlock(self): 422 if self.deploy_lock is None: 423 return 424 425 bb.utils.unlockfile(self.deploy_lock) 426 427 self.deploy_lock = None 428 429 def construct_uris(self, uris, base_paths): 430 """ 431 Construct URIs based on the following pattern: uri/base_path where 'uri' 432 and 'base_path' correspond to each element of the corresponding array 433 argument leading to len(uris) x len(base_paths) elements on the returned 434 array 435 """ 436 def _append(arr1, arr2, sep='/'): 437 res = [] 438 narr1 = [a.rstrip(sep) for a in arr1] 439 narr2 = [a.rstrip(sep).lstrip(sep) for a in arr2] 440 for a1 in narr1: 441 if arr2: 442 for a2 in narr2: 443 res.append("%s%s%s" % (a1, sep, a2)) 444 else: 445 res.append(a1) 446 return res 447 return _append(uris, base_paths) 448 449def create_packages_dir(d, subrepo_dir, deploydir, taskname, filterbydependencies): 450 """ 451 Go through our do_package_write_X dependencies and hardlink the packages we depend 452 upon into the repo directory. This prevents us seeing other packages that may 453 have been built that we don't depend upon and also packages for architectures we don't 454 support. 455 """ 456 import errno 457 458 taskdepdata = d.getVar("BB_TASKDEPDATA", False) 459 mytaskname = d.getVar("BB_RUNTASK") 460 pn = d.getVar("PN") 461 seendirs = set() 462 multilibs = {} 463 464 bb.utils.remove(subrepo_dir, recurse=True) 465 bb.utils.mkdirhier(subrepo_dir) 466 467 # Detect bitbake -b usage 468 nodeps = d.getVar("BB_LIMITEDDEPS") or False 469 if nodeps or not filterbydependencies: 470 oe.path.symlink(deploydir, subrepo_dir, True) 471 return 472 473 start = None 474 for dep in taskdepdata: 475 data = taskdepdata[dep] 476 if data[1] == mytaskname and data[0] == pn: 477 start = dep 478 break 479 if start is None: 480 bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?") 481 pkgdeps = set() 482 start = [start] 483 seen = set(start) 484 # Support direct dependencies (do_rootfs -> do_package_write_X) 485 # or indirect dependencies within PN (do_populate_sdk_ext -> do_rootfs -> do_package_write_X) 486 while start: 487 next = [] 488 for dep2 in start: 489 for dep in taskdepdata[dep2][3]: 490 if taskdepdata[dep][0] != pn: 491 if "do_" + taskname in dep: 492 pkgdeps.add(dep) 493 elif dep not in seen: 494 next.append(dep) 495 seen.add(dep) 496 start = next 497 498 for dep in pkgdeps: 499 c = taskdepdata[dep][0] 500 manifest, d2 = oe.sstatesig.find_sstate_manifest(c, taskdepdata[dep][2], taskname, d, multilibs) 501 if not manifest: 502 bb.fatal("No manifest generated from: %s in %s" % (c, taskdepdata[dep][2])) 503 if not os.path.exists(manifest): 504 continue 505 with open(manifest, "r") as f: 506 for l in f: 507 l = l.strip() 508 deploydir = os.path.normpath(deploydir) 509 if bb.data.inherits_class('packagefeed-stability', d): 510 dest = l.replace(deploydir + "-prediff", "") 511 else: 512 dest = l.replace(deploydir, "") 513 dest = subrepo_dir + dest 514 if l.endswith("/"): 515 if dest not in seendirs: 516 bb.utils.mkdirhier(dest) 517 seendirs.add(dest) 518 continue 519 # Try to hardlink the file, copy if that fails 520 destdir = os.path.dirname(dest) 521 if destdir not in seendirs: 522 bb.utils.mkdirhier(destdir) 523 seendirs.add(destdir) 524 try: 525 os.link(l, dest) 526 except OSError as err: 527 if err.errno == errno.EXDEV: 528 bb.utils.copyfile(l, dest) 529 else: 530 raise 531 532 533def generate_index_files(d): 534 from oe.package_manager.rpm import RpmSubdirIndexer 535 from oe.package_manager.ipk import OpkgIndexer 536 from oe.package_manager.deb import DpkgIndexer 537 538 classes = d.getVar('PACKAGE_CLASSES').replace("package_", "").split() 539 540 indexer_map = { 541 "rpm": (RpmSubdirIndexer, d.getVar('DEPLOY_DIR_RPM')), 542 "ipk": (OpkgIndexer, d.getVar('DEPLOY_DIR_IPK')), 543 "deb": (DpkgIndexer, d.getVar('DEPLOY_DIR_DEB')) 544 } 545 546 result = None 547 548 for pkg_class in classes: 549 if not pkg_class in indexer_map: 550 continue 551 552 if os.path.exists(indexer_map[pkg_class][1]): 553 result = indexer_map[pkg_class][0](d, indexer_map[pkg_class][1]).write_index() 554 555 if result is not None: 556 bb.fatal(result) 557