1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import re 8import shutil 9import subprocess 10from oe.package_manager import * 11 12class OpkgIndexer(Indexer): 13 def write_index(self): 14 arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS", 15 "SDK_PACKAGE_ARCHS", 16 ] 17 18 opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") 19 opkg_index_cmd_extra_params = self.d.getVar('OPKG_MAKE_INDEX_EXTRA_PARAMS') or "" 20 if self.d.getVar('PACKAGE_FEED_SIGN') == '1': 21 signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) 22 else: 23 signer = None 24 25 if not os.path.exists(os.path.join(self.deploy_dir, "Packages")): 26 open(os.path.join(self.deploy_dir, "Packages"), "w").close() 27 28 index_cmds = set() 29 index_sign_files = set() 30 for arch_var in arch_vars: 31 archs = self.d.getVar(arch_var) 32 if archs is None: 33 continue 34 35 for arch in archs.split(): 36 pkgs_dir = os.path.join(self.deploy_dir, arch) 37 pkgs_file = os.path.join(pkgs_dir, "Packages") 38 39 if not os.path.isdir(pkgs_dir): 40 continue 41 42 if not os.path.exists(pkgs_file): 43 open(pkgs_file, "w").close() 44 45 index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s %s' % 46 (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params)) 47 48 index_sign_files.add(pkgs_file) 49 50 if len(index_cmds) == 0: 51 bb.note("There are no packages in %s!" % self.deploy_dir) 52 return 53 54 oe.utils.multiprocess_launch(create_index, index_cmds, self.d) 55 56 if signer: 57 feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE') 58 is_ascii_sig = (feed_sig_type.upper() != "BIN") 59 for f in index_sign_files: 60 signer.detach_sign(f, 61 self.d.getVar('PACKAGE_FEED_GPG_NAME'), 62 self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'), 63 armor=is_ascii_sig) 64 65class PMPkgsList(PkgsList): 66 def __init__(self, d, rootfs_dir): 67 super(PMPkgsList, self).__init__(d, rootfs_dir) 68 config_file = d.getVar("IPKGCONF_TARGET") 69 70 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") 71 self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir) 72 self.opkg_args += self.d.getVar("OPKG_ARGS") 73 74 def list_pkgs(self, format=None): 75 cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args) 76 77 # opkg returns success even when it printed some 78 # "Collected errors:" report to stderr. Mixing stderr into 79 # stdout then leads to random failures later on when 80 # parsing the output. To avoid this we need to collect both 81 # output streams separately and check for empty stderr. 82 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 83 cmd_output, cmd_stderr = p.communicate() 84 cmd_output = cmd_output.decode("utf-8") 85 cmd_stderr = cmd_stderr.decode("utf-8") 86 if p.returncode or cmd_stderr: 87 bb.fatal("Cannot get the installed packages list. Command '%s' " 88 "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr)) 89 90 return opkg_query(cmd_output) 91 92 93 94class OpkgDpkgPM(PackageManager): 95 def __init__(self, d, target_rootfs): 96 """ 97 This is an abstract class. Do not instantiate this directly. 98 """ 99 super(OpkgDpkgPM, self).__init__(d, target_rootfs) 100 101 def package_info(self, pkg, cmd): 102 """ 103 Returns a dictionary with the package info. 104 105 This method extracts the common parts for Opkg and Dpkg 106 """ 107 108 proc = subprocess.run(cmd, capture_output=True, encoding="utf-8", shell=True) 109 if proc.returncode: 110 bb.fatal("Unable to list available packages. Command '%s' " 111 "returned %d:\n%s" % (cmd, proc.returncode, proc.stderr)) 112 elif proc.stderr: 113 bb.note("Command '%s' returned stderr: %s" % (cmd, proc.stderr)) 114 115 return opkg_query(proc.stdout) 116 117 def extract(self, pkg, pkg_info): 118 """ 119 Returns the path to a tmpdir where resides the contents of a package. 120 121 Deleting the tmpdir is responsability of the caller. 122 123 This method extracts the common parts for Opkg and Dpkg 124 """ 125 126 ar_cmd = bb.utils.which(os.getenv("PATH"), "ar") 127 tar_cmd = bb.utils.which(os.getenv("PATH"), "tar") 128 pkg_path = pkg_info[pkg]["filepath"] 129 130 if not os.path.isfile(pkg_path): 131 bb.fatal("Unable to extract package for '%s'." 132 "File %s doesn't exists" % (pkg, pkg_path)) 133 134 tmp_dir = tempfile.mkdtemp() 135 current_dir = os.getcwd() 136 os.chdir(tmp_dir) 137 data_tar = 'data.tar.zst' 138 139 try: 140 cmd = [ar_cmd, 'x', pkg_path] 141 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 142 cmd = [tar_cmd, 'xf', data_tar] 143 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 144 except subprocess.CalledProcessError as e: 145 bb.utils.remove(tmp_dir, recurse=True) 146 bb.fatal("Unable to extract %s package. Command '%s' " 147 "returned %d:\n%s" % (pkg_path, ' '.join(cmd), e.returncode, e.output.decode("utf-8"))) 148 except OSError as e: 149 bb.utils.remove(tmp_dir, recurse=True) 150 bb.fatal("Unable to extract %s package. Command '%s' " 151 "returned %d:\n%s at %s" % (pkg_path, ' '.join(cmd), e.errno, e.strerror, e.filename)) 152 153 bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) 154 bb.utils.remove(os.path.join(tmp_dir, "debian-binary")) 155 bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz")) 156 os.chdir(current_dir) 157 158 return tmp_dir 159 160 def _handle_intercept_failure(self, registered_pkgs): 161 self.mark_packages("unpacked", registered_pkgs.split()) 162 163class OpkgPM(OpkgDpkgPM): 164 def __init__(self, d, target_rootfs, config_file, archs, task_name='target', ipk_repo_workdir="oe-rootfs-repo", filterbydependencies=True, prepare_index=True): 165 super(OpkgPM, self).__init__(d, target_rootfs) 166 167 self.config_file = config_file 168 self.pkg_archs = archs 169 self.task_name = task_name 170 171 self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir) 172 self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") 173 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") 174 self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs) 175 self.opkg_args += self.d.getVar("OPKG_ARGS") 176 177 if prepare_index: 178 create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies) 179 180 self.opkg_dir = oe.path.join(target_rootfs, self.d.getVar('OPKGLIBDIR'), "opkg") 181 bb.utils.mkdirhier(self.opkg_dir) 182 183 self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name) 184 if not os.path.exists(self.d.expand('${T}/saved')): 185 bb.utils.mkdirhier(self.d.expand('${T}/saved')) 186 187 self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1" 188 if self.from_feeds: 189 self._create_custom_config() 190 else: 191 self._create_config() 192 193 self.indexer = OpkgIndexer(self.d, self.deploy_dir) 194 195 def mark_packages(self, status_tag, packages=None): 196 """ 197 This function will change a package's status in /var/lib/opkg/status file. 198 If 'packages' is None then the new_status will be applied to all 199 packages 200 """ 201 status_file = os.path.join(self.opkg_dir, "status") 202 203 with open(status_file, "r") as sf: 204 with open(status_file + ".tmp", "w+") as tmp_sf: 205 if packages is None: 206 tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)", 207 r"Package: \1\n\2Status: \3%s" % status_tag, 208 sf.read())) 209 else: 210 if type(packages).__name__ != "list": 211 raise TypeError("'packages' should be a list object") 212 213 status = sf.read() 214 for pkg in packages: 215 status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg, 216 r"Package: %s\n\1Status: \2%s" % (pkg, status_tag), 217 status) 218 219 tmp_sf.write(status) 220 221 bb.utils.rename(status_file + ".tmp", status_file) 222 223 def _create_custom_config(self): 224 bb.note("Building from feeds activated!") 225 226 with open(self.config_file, "w+") as config_file: 227 priority = 1 228 for arch in self.pkg_archs.split(): 229 config_file.write("arch %s %d\n" % (arch, priority)) 230 priority += 5 231 232 for line in (self.d.getVar('IPK_FEED_URIS') or "").split(): 233 feed_match = re.match(r"^[ \t]*(.*)##([^ \t]*)[ \t]*$", line) 234 235 if feed_match is not None: 236 feed_name = feed_match.group(1) 237 feed_uri = feed_match.group(2) 238 239 bb.note("Add %s feed with URL %s" % (feed_name, feed_uri)) 240 241 config_file.write("src/gz %s %s\n" % (feed_name, feed_uri)) 242 243 """ 244 Allow to use package deploy directory contents as quick devel-testing 245 feed. This creates individual feed configs for each arch subdir of those 246 specified as compatible for the current machine. 247 NOTE: Development-helper feature, NOT a full-fledged feed. 248 """ 249 if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI') or "") != "": 250 for arch in self.pkg_archs.split(): 251 cfg_file_name = oe.path.join(self.target_rootfs, 252 self.d.getVar("sysconfdir"), 253 "opkg", 254 "local-%s-feed.conf" % arch) 255 256 with open(cfg_file_name, "w+") as cfg_file: 257 cfg_file.write("src/gz local-%s %s/%s" % 258 (arch, 259 self.d.getVar('FEED_DEPLOYDIR_BASE_URI'), 260 arch)) 261 262 if self.d.getVar('OPKGLIBDIR') != '/var/lib': 263 # There is no command line option for this anymore, we need to add 264 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have 265 # the default value of "/var/lib" as defined in opkg: 266 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" 267 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" 268 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" 269 cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) 270 cfg_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) 271 cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) 272 273 274 def _create_config(self): 275 with open(self.config_file, "w+") as config_file: 276 priority = 1 277 for arch in self.pkg_archs.split(): 278 config_file.write("arch %s %d\n" % (arch, priority)) 279 priority += 5 280 281 config_file.write("src oe file:%s\n" % self.deploy_dir) 282 283 for arch in self.pkg_archs.split(): 284 pkgs_dir = os.path.join(self.deploy_dir, arch) 285 if os.path.isdir(pkgs_dir): 286 config_file.write("src oe-%s file:%s\n" % 287 (arch, pkgs_dir)) 288 289 if self.d.getVar('OPKGLIBDIR') != '/var/lib': 290 # There is no command line option for this anymore, we need to add 291 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have 292 # the default value of "/var/lib" as defined in opkg: 293 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" 294 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" 295 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" 296 config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) 297 config_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) 298 config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) 299 300 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): 301 if feed_uris == "": 302 return 303 304 rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf' 305 % self.target_rootfs) 306 307 os.makedirs('%s/etc/opkg' % self.target_rootfs, exist_ok=True) 308 309 feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) 310 archs = self.pkg_archs.split() if feed_archs is None else feed_archs.split() 311 312 with open(rootfs_config, "w+") as config_file: 313 uri_iterator = 0 314 for uri in feed_uris: 315 if archs: 316 for arch in archs: 317 if (feed_archs is None) and (not os.path.exists(oe.path.join(self.deploy_dir, arch))): 318 continue 319 bb.note('Adding opkg feed url-%s-%d (%s)' % 320 (arch, uri_iterator, uri)) 321 config_file.write("src/gz uri-%s-%d %s/%s\n" % 322 (arch, uri_iterator, uri, arch)) 323 else: 324 bb.note('Adding opkg feed url-%d (%s)' % 325 (uri_iterator, uri)) 326 config_file.write("src/gz uri-%d %s\n" % 327 (uri_iterator, uri)) 328 329 uri_iterator += 1 330 331 def update(self): 332 self.deploy_dir_lock() 333 334 cmd = "%s %s update" % (self.opkg_cmd, self.opkg_args) 335 336 try: 337 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) 338 except subprocess.CalledProcessError as e: 339 self.deploy_dir_unlock() 340 bb.fatal("Unable to update the package index files. Command '%s' " 341 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) 342 343 self.deploy_dir_unlock() 344 345 def install(self, pkgs, attempt_only=False, hard_depends_only=False): 346 if not pkgs: 347 return 348 349 cmd = "%s %s" % (self.opkg_cmd, self.opkg_args) 350 for exclude in (self.d.getVar("PACKAGE_EXCLUDE") or "").split(): 351 cmd += " --add-exclude %s" % exclude 352 for bad_recommendation in (self.d.getVar("BAD_RECOMMENDATIONS") or "").split(): 353 cmd += " --add-ignore-recommends %s" % bad_recommendation 354 if hard_depends_only: 355 cmd += " --no-install-recommends" 356 cmd += " install " 357 cmd += " ".join(pkgs) 358 359 os.environ['D'] = self.target_rootfs 360 os.environ['OFFLINE_ROOT'] = self.target_rootfs 361 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs 362 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs 363 os.environ['INTERCEPT_DIR'] = self.intercepts_dir 364 os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') 365 366 try: 367 bb.note("Installing the following packages: %s" % ' '.join(pkgs)) 368 bb.note(cmd) 369 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") 370 bb.note(output) 371 failed_pkgs = [] 372 for line in output.split('\n'): 373 if line.endswith("configuration required on target."): 374 bb.warn(line) 375 failed_pkgs.append(line.split(".")[0]) 376 if failed_pkgs: 377 failed_postinsts_abort(failed_pkgs, self.d.expand("${T}/log.do_${BB_CURRENTTASK}")) 378 except subprocess.CalledProcessError as e: 379 (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. " 380 "Command '%s' returned %d:\n%s" % 381 (cmd, e.returncode, e.output.decode("utf-8"))) 382 383 def remove(self, pkgs, with_dependencies=True): 384 if not pkgs: 385 return 386 387 if with_dependencies: 388 cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \ 389 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) 390 else: 391 cmd = "%s %s --force-depends remove %s" % \ 392 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) 393 394 try: 395 bb.note(cmd) 396 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") 397 bb.note(output) 398 except subprocess.CalledProcessError as e: 399 bb.fatal("Unable to remove packages. Command '%s' " 400 "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) 401 402 def write_index(self): 403 self.deploy_dir_lock() 404 405 result = self.indexer.write_index() 406 407 self.deploy_dir_unlock() 408 409 if result is not None: 410 bb.fatal(result) 411 412 def remove_packaging_data(self): 413 cachedir = oe.path.join(self.target_rootfs, self.d.getVar("localstatedir"), "cache", "opkg") 414 bb.utils.remove(self.opkg_dir, True) 415 bb.utils.remove(cachedir, True) 416 417 def remove_lists(self): 418 if not self.from_feeds: 419 bb.utils.remove(os.path.join(self.opkg_dir, "lists"), True) 420 421 def list_installed(self): 422 return PMPkgsList(self.d, self.target_rootfs).list_pkgs() 423 424 def dummy_install(self, pkgs): 425 """ 426 The following function dummy installs pkgs and returns the log of output. 427 """ 428 if len(pkgs) == 0: 429 return 430 431 # Create an temp dir as opkg root for dummy installation 432 temp_rootfs = self.d.expand('${T}/opkg') 433 opkg_lib_dir = self.d.getVar('OPKGLIBDIR') 434 if opkg_lib_dir[0] == "/": 435 opkg_lib_dir = opkg_lib_dir[1:] 436 temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg') 437 bb.utils.mkdirhier(temp_opkg_dir) 438 439 opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs) 440 opkg_args += self.d.getVar("OPKG_ARGS") 441 442 cmd = "%s %s update" % (self.opkg_cmd, opkg_args) 443 try: 444 subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) 445 except subprocess.CalledProcessError as e: 446 bb.fatal("Unable to update. Command '%s' " 447 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) 448 449 # Dummy installation 450 cmd = "%s %s --noaction install %s " % (self.opkg_cmd, 451 opkg_args, 452 ' '.join(pkgs)) 453 proc = subprocess.run(cmd, capture_output=True, encoding="utf-8", shell=True) 454 if proc.returncode: 455 bb.fatal("Unable to dummy install packages. Command '%s' " 456 "returned %d:\n%s" % (cmd, proc.returncode, proc.stderr)) 457 elif proc.stderr: 458 bb.note("Command '%s' returned stderr: %s" % (cmd, proc.stderr)) 459 460 bb.utils.remove(temp_rootfs, True) 461 462 return proc.stdout 463 464 def backup_packaging_data(self): 465 # Save the opkglib for increment ipk image generation 466 if os.path.exists(self.saved_opkg_dir): 467 bb.utils.remove(self.saved_opkg_dir, True) 468 shutil.copytree(self.opkg_dir, 469 self.saved_opkg_dir, 470 symlinks=True) 471 472 def recover_packaging_data(self): 473 # Move the opkglib back 474 if os.path.exists(self.saved_opkg_dir): 475 if os.path.exists(self.opkg_dir): 476 bb.utils.remove(self.opkg_dir, True) 477 478 bb.note('Recover packaging data') 479 shutil.copytree(self.saved_opkg_dir, 480 self.opkg_dir, 481 symlinks=True) 482 483 def package_info(self, pkg): 484 """ 485 Returns a dictionary with the package info. 486 """ 487 cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg) 488 pkg_info = super(OpkgPM, self).package_info(pkg, cmd) 489 490 pkg_arch = pkg_info[pkg]["arch"] 491 pkg_filename = pkg_info[pkg]["filename"] 492 pkg_info[pkg]["filepath"] = \ 493 os.path.join(self.deploy_dir, pkg_arch, pkg_filename) 494 495 return pkg_info 496 497 def extract(self, pkg): 498 """ 499 Returns the path to a tmpdir where resides the contents of a package. 500 501 Deleting the tmpdir is responsability of the caller. 502 """ 503 pkg_info = self.package_info(pkg) 504 if not pkg_info: 505 bb.fatal("Unable to get information for package '%s' while " 506 "trying to extract the package." % pkg) 507 508 tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) 509 bb.utils.remove(os.path.join(tmp_dir, "data.tar.zst")) 510 511 return tmp_dir 512