1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import re 8import shutil 9import subprocess 10from oe.package_manager import * 11from oe.package_manager.common_deb_ipk import OpkgDpkgPM 12 13class OpkgIndexer(Indexer): 14 def write_index(self): 15 arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS", 16 "SDK_PACKAGE_ARCHS", 17 ] 18 19 opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") 20 opkg_index_cmd_extra_params = self.d.getVar('OPKG_MAKE_INDEX_EXTRA_PARAMS') or "" 21 if self.d.getVar('PACKAGE_FEED_SIGN') == '1': 22 signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) 23 else: 24 signer = None 25 26 if not os.path.exists(os.path.join(self.deploy_dir, "Packages")): 27 open(os.path.join(self.deploy_dir, "Packages"), "w").close() 28 29 index_cmds = set() 30 index_sign_files = set() 31 for arch_var in arch_vars: 32 archs = self.d.getVar(arch_var) 33 if archs is None: 34 continue 35 36 for arch in archs.split(): 37 pkgs_dir = os.path.join(self.deploy_dir, arch) 38 pkgs_file = os.path.join(pkgs_dir, "Packages") 39 40 if not os.path.isdir(pkgs_dir): 41 continue 42 43 if not os.path.exists(pkgs_file): 44 open(pkgs_file, "w").close() 45 46 index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s %s' % 47 (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params)) 48 49 index_sign_files.add(pkgs_file) 50 51 if len(index_cmds) == 0: 52 bb.note("There are no packages in %s!" % self.deploy_dir) 53 return 54 55 oe.utils.multiprocess_launch(create_index, index_cmds, self.d) 56 57 if signer: 58 feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE') 59 is_ascii_sig = (feed_sig_type.upper() != "BIN") 60 for f in index_sign_files: 61 signer.detach_sign(f, 62 self.d.getVar('PACKAGE_FEED_GPG_NAME'), 63 self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'), 64 armor=is_ascii_sig) 65 66class PMPkgsList(PkgsList): 67 def __init__(self, d, rootfs_dir): 68 super(PMPkgsList, self).__init__(d, rootfs_dir) 69 config_file = d.getVar("IPKGCONF_TARGET") 70 71 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") 72 self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir) 73 self.opkg_args += self.d.getVar("OPKG_ARGS") 74 75 def list_pkgs(self, format=None): 76 cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args) 77 78 # opkg returns success even when it printed some 79 # "Collected errors:" report to stderr. Mixing stderr into 80 # stdout then leads to random failures later on when 81 # parsing the output. To avoid this we need to collect both 82 # output streams separately and check for empty stderr. 83 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 84 cmd_output, cmd_stderr = p.communicate() 85 cmd_output = cmd_output.decode("utf-8") 86 cmd_stderr = cmd_stderr.decode("utf-8") 87 if p.returncode or cmd_stderr: 88 bb.fatal("Cannot get the installed packages list. Command '%s' " 89 "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr)) 90 91 return opkg_query(cmd_output) 92 93 94class OpkgPM(OpkgDpkgPM): 95 def __init__(self, d, target_rootfs, config_file, archs, task_name='target', ipk_repo_workdir="oe-rootfs-repo", filterbydependencies=True, prepare_index=True): 96 super(OpkgPM, self).__init__(d, target_rootfs) 97 98 self.config_file = config_file 99 self.pkg_archs = archs 100 self.task_name = task_name 101 102 self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir) 103 self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") 104 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") 105 self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs) 106 self.opkg_args += self.d.getVar("OPKG_ARGS") 107 108 if prepare_index: 109 create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies) 110 111 self.opkg_dir = oe.path.join(target_rootfs, self.d.getVar('OPKGLIBDIR'), "opkg") 112 bb.utils.mkdirhier(self.opkg_dir) 113 114 self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name) 115 if not os.path.exists(self.d.expand('${T}/saved')): 116 bb.utils.mkdirhier(self.d.expand('${T}/saved')) 117 118 self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1" 119 if self.from_feeds: 120 self._create_custom_config() 121 else: 122 self._create_config() 123 124 self.indexer = OpkgIndexer(self.d, self.deploy_dir) 125 126 def mark_packages(self, status_tag, packages=None): 127 """ 128 This function will change a package's status in /var/lib/opkg/status file. 129 If 'packages' is None then the new_status will be applied to all 130 packages 131 """ 132 status_file = os.path.join(self.opkg_dir, "status") 133 134 with open(status_file, "r") as sf: 135 with open(status_file + ".tmp", "w+") as tmp_sf: 136 if packages is None: 137 tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)", 138 r"Package: \1\n\2Status: \3%s" % status_tag, 139 sf.read())) 140 else: 141 if type(packages).__name__ != "list": 142 raise TypeError("'packages' should be a list object") 143 144 status = sf.read() 145 for pkg in packages: 146 status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg, 147 r"Package: %s\n\1Status: \2%s" % (pkg, status_tag), 148 status) 149 150 tmp_sf.write(status) 151 152 bb.utils.rename(status_file + ".tmp", status_file) 153 154 def _create_custom_config(self): 155 bb.note("Building from feeds activated!") 156 157 with open(self.config_file, "w+") as config_file: 158 priority = 1 159 for arch in self.pkg_archs.split(): 160 config_file.write("arch %s %d\n" % (arch, priority)) 161 priority += 5 162 163 for line in (self.d.getVar('IPK_FEED_URIS') or "").split(): 164 feed_match = re.match(r"^[ \t]*(.*)##([^ \t]*)[ \t]*$", line) 165 166 if feed_match is not None: 167 feed_name = feed_match.group(1) 168 feed_uri = feed_match.group(2) 169 170 bb.note("Add %s feed with URL %s" % (feed_name, feed_uri)) 171 172 config_file.write("src/gz %s %s\n" % (feed_name, feed_uri)) 173 174 """ 175 Allow to use package deploy directory contents as quick devel-testing 176 feed. This creates individual feed configs for each arch subdir of those 177 specified as compatible for the current machine. 178 NOTE: Development-helper feature, NOT a full-fledged feed. 179 """ 180 if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI') or "") != "": 181 for arch in self.pkg_archs.split(): 182 cfg_file_name = oe.path.join(self.target_rootfs, 183 self.d.getVar("sysconfdir"), 184 "opkg", 185 "local-%s-feed.conf" % arch) 186 187 with open(cfg_file_name, "w+") as cfg_file: 188 cfg_file.write("src/gz local-%s %s/%s" % 189 (arch, 190 self.d.getVar('FEED_DEPLOYDIR_BASE_URI'), 191 arch)) 192 193 if self.d.getVar('OPKGLIBDIR') != '/var/lib': 194 # There is no command line option for this anymore, we need to add 195 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have 196 # the default value of "/var/lib" as defined in opkg: 197 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" 198 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" 199 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" 200 cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) 201 cfg_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) 202 cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) 203 204 205 def _create_config(self): 206 with open(self.config_file, "w+") as config_file: 207 priority = 1 208 for arch in self.pkg_archs.split(): 209 config_file.write("arch %s %d\n" % (arch, priority)) 210 priority += 5 211 212 config_file.write("src oe file:%s\n" % self.deploy_dir) 213 214 for arch in self.pkg_archs.split(): 215 pkgs_dir = os.path.join(self.deploy_dir, arch) 216 if os.path.isdir(pkgs_dir): 217 config_file.write("src oe-%s file:%s\n" % 218 (arch, pkgs_dir)) 219 220 if self.d.getVar('OPKGLIBDIR') != '/var/lib': 221 # There is no command line option for this anymore, we need to add 222 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have 223 # the default value of "/var/lib" as defined in opkg: 224 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" 225 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" 226 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" 227 config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) 228 config_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) 229 config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) 230 231 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): 232 if feed_uris == "": 233 return 234 235 rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf' 236 % self.target_rootfs) 237 238 os.makedirs('%s/etc/opkg' % self.target_rootfs, exist_ok=True) 239 240 feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) 241 archs = self.pkg_archs.split() if feed_archs is None else feed_archs.split() 242 243 with open(rootfs_config, "w+") as config_file: 244 uri_iterator = 0 245 for uri in feed_uris: 246 if archs: 247 for arch in archs: 248 if (feed_archs is None) and (not os.path.exists(oe.path.join(self.deploy_dir, arch))): 249 continue 250 bb.note('Adding opkg feed url-%s-%d (%s)' % 251 (arch, uri_iterator, uri)) 252 config_file.write("src/gz uri-%s-%d %s/%s\n" % 253 (arch, uri_iterator, uri, arch)) 254 else: 255 bb.note('Adding opkg feed url-%d (%s)' % 256 (uri_iterator, uri)) 257 config_file.write("src/gz uri-%d %s\n" % 258 (uri_iterator, uri)) 259 260 uri_iterator += 1 261 262 def update(self): 263 self.deploy_dir_lock() 264 265 cmd = "%s %s update" % (self.opkg_cmd, self.opkg_args) 266 267 try: 268 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) 269 except subprocess.CalledProcessError as e: 270 self.deploy_dir_unlock() 271 bb.fatal("Unable to update the package index files. Command '%s' " 272 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) 273 274 self.deploy_dir_unlock() 275 276 def install(self, pkgs, attempt_only=False, hard_depends_only=False): 277 if not pkgs: 278 return 279 280 cmd = "%s %s" % (self.opkg_cmd, self.opkg_args) 281 for exclude in (self.d.getVar("PACKAGE_EXCLUDE") or "").split(): 282 cmd += " --add-exclude %s" % exclude 283 for bad_recommendation in (self.d.getVar("BAD_RECOMMENDATIONS") or "").split(): 284 cmd += " --add-ignore-recommends %s" % bad_recommendation 285 if hard_depends_only: 286 cmd += " --no-install-recommends" 287 cmd += " install " 288 cmd += " ".join(pkgs) 289 290 os.environ['D'] = self.target_rootfs 291 os.environ['OFFLINE_ROOT'] = self.target_rootfs 292 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs 293 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs 294 os.environ['INTERCEPT_DIR'] = self.intercepts_dir 295 os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') 296 297 try: 298 bb.note("Installing the following packages: %s" % ' '.join(pkgs)) 299 bb.note(cmd) 300 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") 301 bb.note(output) 302 failed_pkgs = [] 303 for line in output.split('\n'): 304 if line.endswith("configuration required on target."): 305 bb.warn(line) 306 failed_pkgs.append(line.split(".")[0]) 307 if failed_pkgs: 308 failed_postinsts_abort(failed_pkgs, self.d.expand("${T}/log.do_${BB_CURRENTTASK}")) 309 except subprocess.CalledProcessError as e: 310 (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. " 311 "Command '%s' returned %d:\n%s" % 312 (cmd, e.returncode, e.output.decode("utf-8"))) 313 314 def remove(self, pkgs, with_dependencies=True): 315 if not pkgs: 316 return 317 318 if with_dependencies: 319 cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \ 320 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) 321 else: 322 cmd = "%s %s --force-depends remove %s" % \ 323 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) 324 325 try: 326 bb.note(cmd) 327 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") 328 bb.note(output) 329 except subprocess.CalledProcessError as e: 330 bb.fatal("Unable to remove packages. Command '%s' " 331 "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) 332 333 def write_index(self): 334 self.deploy_dir_lock() 335 336 result = self.indexer.write_index() 337 338 self.deploy_dir_unlock() 339 340 if result is not None: 341 bb.fatal(result) 342 343 def remove_packaging_data(self): 344 cachedir = oe.path.join(self.target_rootfs, self.d.getVar("localstatedir"), "cache", "opkg") 345 bb.utils.remove(self.opkg_dir, True) 346 bb.utils.remove(cachedir, True) 347 348 def remove_lists(self): 349 if not self.from_feeds: 350 bb.utils.remove(os.path.join(self.opkg_dir, "lists"), True) 351 352 def list_installed(self): 353 return PMPkgsList(self.d, self.target_rootfs).list_pkgs() 354 355 def dummy_install(self, pkgs): 356 """ 357 The following function dummy installs pkgs and returns the log of output. 358 """ 359 if len(pkgs) == 0: 360 return 361 362 # Create an temp dir as opkg root for dummy installation 363 temp_rootfs = self.d.expand('${T}/opkg') 364 opkg_lib_dir = self.d.getVar('OPKGLIBDIR') 365 if opkg_lib_dir[0] == "/": 366 opkg_lib_dir = opkg_lib_dir[1:] 367 temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg') 368 bb.utils.mkdirhier(temp_opkg_dir) 369 370 opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs) 371 opkg_args += self.d.getVar("OPKG_ARGS") 372 373 cmd = "%s %s update" % (self.opkg_cmd, opkg_args) 374 try: 375 subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) 376 except subprocess.CalledProcessError as e: 377 bb.fatal("Unable to update. Command '%s' " 378 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) 379 380 # Dummy installation 381 cmd = "%s %s --noaction install %s " % (self.opkg_cmd, 382 opkg_args, 383 ' '.join(pkgs)) 384 proc = subprocess.run(cmd, capture_output=True, encoding="utf-8", shell=True) 385 if proc.returncode: 386 bb.fatal("Unable to dummy install packages. Command '%s' " 387 "returned %d:\n%s" % (cmd, proc.returncode, proc.stderr)) 388 elif proc.stderr: 389 bb.note("Command '%s' returned stderr: %s" % (cmd, proc.stderr)) 390 391 bb.utils.remove(temp_rootfs, True) 392 393 return proc.stdout 394 395 def backup_packaging_data(self): 396 # Save the opkglib for increment ipk image generation 397 if os.path.exists(self.saved_opkg_dir): 398 bb.utils.remove(self.saved_opkg_dir, True) 399 shutil.copytree(self.opkg_dir, 400 self.saved_opkg_dir, 401 symlinks=True) 402 403 def recover_packaging_data(self): 404 # Move the opkglib back 405 if os.path.exists(self.saved_opkg_dir): 406 if os.path.exists(self.opkg_dir): 407 bb.utils.remove(self.opkg_dir, True) 408 409 bb.note('Recover packaging data') 410 shutil.copytree(self.saved_opkg_dir, 411 self.opkg_dir, 412 symlinks=True) 413 414 def package_info(self, pkg): 415 """ 416 Returns a dictionary with the package info. 417 """ 418 cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg) 419 pkg_info = self._common_package_info(cmd) 420 421 pkg_arch = pkg_info[pkg]["arch"] 422 pkg_filename = pkg_info[pkg]["filename"] 423 pkg_info[pkg]["filepath"] = \ 424 os.path.join(self.deploy_dir, pkg_arch, pkg_filename) 425 426 return pkg_info 427