1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import re
8import subprocess
9from oe.package_manager import *
10
11class DpkgIndexer(Indexer):
12    def _create_configs(self):
13        bb.utils.mkdirhier(self.apt_conf_dir)
14        bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "lists", "partial"))
15        bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "apt.conf.d"))
16        bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "preferences.d"))
17
18        with open(os.path.join(self.apt_conf_dir, "preferences"),
19                "w") as prefs_file:
20            pass
21        with open(os.path.join(self.apt_conf_dir, "sources.list"),
22                "w+") as sources_file:
23            pass
24
25        with open(self.apt_conf_file, "w") as apt_conf:
26            with open(os.path.join(self.d.expand("${STAGING_ETCDIR_NATIVE}"),
27                "apt", "apt.conf.sample")) as apt_conf_sample:
28                for line in apt_conf_sample.read().split("\n"):
29                    line = re.sub(r"#ROOTFS#", "/dev/null", line)
30                    line = re.sub(r"#APTCONF#", self.apt_conf_dir, line)
31                    apt_conf.write(line + "\n")
32
33    def write_index(self):
34        self.apt_conf_dir = os.path.join(self.d.expand("${APTCONF_TARGET}"),
35                "apt-ftparchive")
36        self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf")
37        self._create_configs()
38
39        os.environ['APT_CONFIG'] = self.apt_conf_file
40
41        pkg_archs = self.d.getVar('PACKAGE_ARCHS')
42        if pkg_archs is not None:
43            arch_list = pkg_archs.split()
44        sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS')
45        if sdk_pkg_archs is not None:
46            for a in sdk_pkg_archs.split():
47                if a not in pkg_archs:
48                    arch_list.append(a)
49
50        all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split()
51        arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in arch_list)
52
53        apt_ftparchive = bb.utils.which(os.getenv('PATH'), "apt-ftparchive")
54        gzip = bb.utils.which(os.getenv('PATH'), "gzip")
55
56        index_cmds = []
57        deb_dirs_found = False
58        index_sign_files = set()
59        for arch in arch_list:
60            arch_dir = os.path.join(self.deploy_dir, arch)
61            if not os.path.isdir(arch_dir):
62                continue
63
64            cmd = "cd %s; PSEUDO_UNLOAD=1 %s packages . > Packages;" % (arch_dir, apt_ftparchive)
65
66            cmd += "%s -fcn Packages > Packages.gz;" % gzip
67
68            release_file = os.path.join(arch_dir, "Release")
69            index_sign_files.add(release_file)
70
71            with open(release_file, "w+") as release:
72                release.write("Label: %s\n" % arch)
73
74            cmd += "PSEUDO_UNLOAD=1 %s release . >> Release" % apt_ftparchive
75
76            index_cmds.append(cmd)
77
78            deb_dirs_found = True
79
80        if not deb_dirs_found:
81            bb.note("There are no packages in %s" % self.deploy_dir)
82            return
83
84        oe.utils.multiprocess_launch(create_index, index_cmds, self.d)
85        if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
86            signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND'))
87        else:
88            signer = None
89        if signer:
90            for f in index_sign_files:
91                signer.detach_sign(f,
92                                   self.d.getVar('PACKAGE_FEED_GPG_NAME'),
93                                   self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'),
94                                   output_suffix="gpg",
95                                   use_sha256=True)
96
97class PMPkgsList(PkgsList):
98
99    def list_pkgs(self):
100        cmd = [bb.utils.which(os.getenv('PATH'), "dpkg-query"),
101               "--admindir=%s/var/lib/dpkg" % self.rootfs_dir,
102               "-W"]
103
104        cmd.append("-f=Package: ${Package}\nArchitecture: ${PackageArch}\nVersion: ${Version}\nFile: ${Package}_${Version}_${Architecture}.deb\nDepends: ${Depends}\nRecommends: ${Recommends}\nProvides: ${Provides}\n\n")
105
106        try:
107            cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8")
108        except subprocess.CalledProcessError as e:
109            bb.fatal("Cannot get the installed packages list. Command '%s' "
110                     "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
111
112        return opkg_query(cmd_output)
113
114class OpkgDpkgPM(PackageManager):
115    def __init__(self, d, target_rootfs):
116        """
117        This is an abstract class. Do not instantiate this directly.
118        """
119        super(OpkgDpkgPM, self).__init__(d, target_rootfs)
120
121    def package_info(self, pkg, cmd):
122        """
123        Returns a dictionary with the package info.
124
125        This method extracts the common parts for Opkg and Dpkg
126        """
127
128        try:
129            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
130        except subprocess.CalledProcessError as e:
131            bb.fatal("Unable to list available packages. Command '%s' "
132                     "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
133        return opkg_query(output)
134
135    def extract(self, pkg, pkg_info):
136        """
137        Returns the path to a tmpdir where resides the contents of a package.
138
139        Deleting the tmpdir is responsability of the caller.
140
141        This method extracts the common parts for Opkg and Dpkg
142        """
143
144        ar_cmd = bb.utils.which(os.getenv("PATH"), "ar")
145        tar_cmd = bb.utils.which(os.getenv("PATH"), "tar")
146        pkg_path = pkg_info[pkg]["filepath"]
147
148        if not os.path.isfile(pkg_path):
149            bb.fatal("Unable to extract package for '%s'."
150                     "File %s doesn't exists" % (pkg, pkg_path))
151
152        tmp_dir = tempfile.mkdtemp()
153        current_dir = os.getcwd()
154        os.chdir(tmp_dir)
155        data_tar = 'data.tar.xz'
156
157        try:
158            cmd = [ar_cmd, 'x', pkg_path]
159            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
160            cmd = [tar_cmd, 'xf', data_tar]
161            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
162        except subprocess.CalledProcessError as e:
163            bb.utils.remove(tmp_dir, recurse=True)
164            bb.fatal("Unable to extract %s package. Command '%s' "
165                     "returned %d:\n%s" % (pkg_path, ' '.join(cmd), e.returncode, e.output.decode("utf-8")))
166        except OSError as e:
167            bb.utils.remove(tmp_dir, recurse=True)
168            bb.fatal("Unable to extract %s package. Command '%s' "
169                     "returned %d:\n%s at %s" % (pkg_path, ' '.join(cmd), e.errno, e.strerror, e.filename))
170
171        bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
172        bb.utils.remove(os.path.join(tmp_dir, "debian-binary"))
173        bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz"))
174        os.chdir(current_dir)
175
176        return tmp_dir
177
178    def _handle_intercept_failure(self, registered_pkgs):
179        self.mark_packages("unpacked", registered_pkgs.split())
180
181class DpkgPM(OpkgDpkgPM):
182    def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None, deb_repo_workdir="oe-rootfs-repo", filterbydependencies=True):
183        super(DpkgPM, self).__init__(d, target_rootfs)
184        self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), deb_repo_workdir)
185
186        create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_DEB"), "package_write_deb", filterbydependencies)
187
188        if apt_conf_dir is None:
189            self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt")
190        else:
191            self.apt_conf_dir = apt_conf_dir
192        self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf")
193        self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get")
194        self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache")
195
196        self.apt_args = d.getVar("APT_ARGS")
197
198        self.all_arch_list = archs.split()
199        all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split()
200        self.all_arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in self.all_arch_list)
201
202        self._create_configs(archs, base_archs)
203
204        self.indexer = DpkgIndexer(self.d, self.deploy_dir)
205
206    def mark_packages(self, status_tag, packages=None):
207        """
208        This function will change a package's status in /var/lib/dpkg/status file.
209        If 'packages' is None then the new_status will be applied to all
210        packages
211        """
212        status_file = self.target_rootfs + "/var/lib/dpkg/status"
213
214        with open(status_file, "r") as sf:
215            with open(status_file + ".tmp", "w+") as tmp_sf:
216                if packages is None:
217                    tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)",
218                                        r"Package: \1\n\2Status: \3%s" % status_tag,
219                                        sf.read()))
220                else:
221                    if type(packages).__name__ != "list":
222                        raise TypeError("'packages' should be a list object")
223
224                    status = sf.read()
225                    for pkg in packages:
226                        status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg,
227                                        r"Package: %s\n\1Status: \2%s" % (pkg, status_tag),
228                                        status)
229
230                    tmp_sf.write(status)
231
232        bb.utils.rename(status_file + ".tmp", status_file)
233
234    def run_pre_post_installs(self, package_name=None):
235        """
236        Run the pre/post installs for package "package_name". If package_name is
237        None, then run all pre/post install scriptlets.
238        """
239        info_dir = self.target_rootfs + "/var/lib/dpkg/info"
240        ControlScript = collections.namedtuple("ControlScript", ["suffix", "name", "argument"])
241        control_scripts = [
242                ControlScript(".preinst", "Preinstall", "install"),
243                ControlScript(".postinst", "Postinstall", "configure")]
244        status_file = self.target_rootfs + "/var/lib/dpkg/status"
245        installed_pkgs = []
246
247        with open(status_file, "r") as status:
248            for line in status.read().split('\n'):
249                m = re.match(r"^Package: (.*)", line)
250                if m is not None:
251                    installed_pkgs.append(m.group(1))
252
253        if package_name is not None and not package_name in installed_pkgs:
254            return
255
256        os.environ['D'] = self.target_rootfs
257        os.environ['OFFLINE_ROOT'] = self.target_rootfs
258        os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
259        os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
260        os.environ['INTERCEPT_DIR'] = self.intercepts_dir
261        os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE')
262
263        for pkg_name in installed_pkgs:
264            for control_script in control_scripts:
265                p_full = os.path.join(info_dir, pkg_name + control_script.suffix)
266                if os.path.exists(p_full):
267                    try:
268                        bb.note("Executing %s for package: %s ..." %
269                                 (control_script.name.lower(), pkg_name))
270                        output = subprocess.check_output([p_full, control_script.argument],
271                                stderr=subprocess.STDOUT).decode("utf-8")
272                        bb.note(output)
273                    except subprocess.CalledProcessError as e:
274                        bb.warn("%s for package %s failed with %d:\n%s" %
275                                (control_script.name, pkg_name, e.returncode,
276                                    e.output.decode("utf-8")))
277                        failed_postinsts_abort([pkg_name], self.d.expand("${T}/log.do_${BB_CURRENTTASK}"))
278
279    def update(self):
280        os.environ['APT_CONFIG'] = self.apt_conf_file
281
282        self.deploy_dir_lock()
283
284        cmd = "%s update" % self.apt_get_cmd
285
286        try:
287            subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
288        except subprocess.CalledProcessError as e:
289            bb.fatal("Unable to update the package index files. Command '%s' "
290                     "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8")))
291
292        self.deploy_dir_unlock()
293
294    def install(self, pkgs, attempt_only=False, hard_depends_only=False):
295        if attempt_only and len(pkgs) == 0:
296            return
297
298        os.environ['APT_CONFIG'] = self.apt_conf_file
299
300        extra_args = ""
301        if hard_depends_only:
302            extra_args = "--no-install-recommends"
303
304        cmd = "%s %s install --allow-downgrades --allow-remove-essential --allow-change-held-packages --allow-unauthenticated --no-remove %s %s" % \
305              (self.apt_get_cmd, self.apt_args, extra_args, ' '.join(pkgs))
306
307        try:
308            bb.note("Installing the following packages: %s" % ' '.join(pkgs))
309            output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
310            bb.note(output.decode("utf-8"))
311        except subprocess.CalledProcessError as e:
312            (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. "
313                                              "Command '%s' returned %d:\n%s" %
314                                              (cmd, e.returncode, e.output.decode("utf-8")))
315
316        # rename *.dpkg-new files/dirs
317        for root, dirs, files in os.walk(self.target_rootfs):
318            for dir in dirs:
319                new_dir = re.sub(r"\.dpkg-new", "", dir)
320                if dir != new_dir:
321                    bb.utils.rename(os.path.join(root, dir),
322                              os.path.join(root, new_dir))
323
324            for file in files:
325                new_file = re.sub(r"\.dpkg-new", "", file)
326                if file != new_file:
327                    bb.utils.rename(os.path.join(root, file),
328                              os.path.join(root, new_file))
329
330
331    def remove(self, pkgs, with_dependencies=True):
332        if not pkgs:
333            return
334
335        os.environ['D'] = self.target_rootfs
336        os.environ['OFFLINE_ROOT'] = self.target_rootfs
337        os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
338        os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
339        os.environ['INTERCEPT_DIR'] = self.intercepts_dir
340
341        if with_dependencies:
342            os.environ['APT_CONFIG'] = self.apt_conf_file
343            cmd = "%s purge %s" % (self.apt_get_cmd, ' '.join(pkgs))
344        else:
345            cmd = "%s --admindir=%s/var/lib/dpkg --instdir=%s" \
346                  " -P --force-depends %s" % \
347                  (bb.utils.which(os.getenv('PATH'), "dpkg"),
348                   self.target_rootfs, self.target_rootfs, ' '.join(pkgs))
349
350        try:
351            subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
352        except subprocess.CalledProcessError as e:
353            bb.fatal("Unable to remove packages. Command '%s' "
354                     "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8")))
355
356    def write_index(self):
357        self.deploy_dir_lock()
358
359        result = self.indexer.write_index()
360
361        self.deploy_dir_unlock()
362
363        if result is not None:
364            bb.fatal(result)
365
366    def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs):
367        if feed_uris == "":
368            return
369
370
371        sources_conf = os.path.join("%s/etc/apt/sources.list"
372                                    % self.target_rootfs)
373        if not os.path.exists(os.path.dirname(sources_conf)):
374            return
375
376        arch_list = []
377
378        if feed_archs is None:
379            for arch in self.all_arch_list:
380                if not os.path.exists(os.path.join(self.deploy_dir, arch)):
381                    continue
382                arch_list.append(arch)
383        else:
384            arch_list = feed_archs.split()
385
386        feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split())
387
388        with open(sources_conf, "w+") as sources_file:
389            for uri in feed_uris:
390                if arch_list:
391                    for arch in arch_list:
392                        bb.note('Adding dpkg channel at (%s)' % uri)
393                        sources_file.write("deb [trusted=yes] %s/%s ./\n" %
394                                           (uri, arch))
395                else:
396                    bb.note('Adding dpkg channel at (%s)' % uri)
397                    sources_file.write("deb [trusted=yes] %s ./\n" % uri)
398
399    def _create_configs(self, archs, base_archs):
400        base_archs = re.sub(r"_", r"-", base_archs)
401
402        if os.path.exists(self.apt_conf_dir):
403            bb.utils.remove(self.apt_conf_dir, True)
404
405        bb.utils.mkdirhier(self.apt_conf_dir)
406        bb.utils.mkdirhier(self.apt_conf_dir + "/lists/partial/")
407        bb.utils.mkdirhier(self.apt_conf_dir + "/apt.conf.d/")
408        bb.utils.mkdirhier(self.apt_conf_dir + "/preferences.d/")
409
410        arch_list = []
411        for arch in self.all_arch_list:
412            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
413                continue
414            arch_list.append(arch)
415
416        with open(os.path.join(self.apt_conf_dir, "preferences"), "w+") as prefs_file:
417            priority = 801
418            for arch in arch_list:
419                prefs_file.write(
420                    "Package: *\n"
421                    "Pin: release l=%s\n"
422                    "Pin-Priority: %d\n\n" % (arch, priority))
423
424                priority += 5
425
426            pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE') or ""
427            for pkg in pkg_exclude.split():
428                prefs_file.write(
429                    "Package: %s\n"
430                    "Pin: release *\n"
431                    "Pin-Priority: -1\n\n" % pkg)
432
433        arch_list.reverse()
434
435        with open(os.path.join(self.apt_conf_dir, "sources.list"), "w+") as sources_file:
436            for arch in arch_list:
437                sources_file.write("deb [trusted=yes] file:%s/ ./\n" %
438                                   os.path.join(self.deploy_dir, arch))
439
440        base_arch_list = base_archs.split()
441        multilib_variants = self.d.getVar("MULTILIB_VARIANTS");
442        for variant in multilib_variants.split():
443            localdata = bb.data.createCopy(self.d)
444            variant_tune = localdata.getVar("DEFAULTTUNE:virtclass-multilib-" + variant, False)
445            orig_arch = localdata.getVar("DPKG_ARCH")
446            localdata.setVar("DEFAULTTUNE", variant_tune)
447            variant_arch = localdata.getVar("DPKG_ARCH")
448            if variant_arch not in base_arch_list:
449                base_arch_list.append(variant_arch)
450
451        with open(self.apt_conf_file, "w+") as apt_conf:
452            with open(self.d.expand("${STAGING_ETCDIR_NATIVE}/apt/apt.conf.sample")) as apt_conf_sample:
453                for line in apt_conf_sample.read().split("\n"):
454                    match_arch = re.match(r"  Architecture \".*\";$", line)
455                    architectures = ""
456                    if match_arch:
457                        for base_arch in base_arch_list:
458                            architectures += "\"%s\";" % base_arch
459                        apt_conf.write("  Architectures {%s};\n" % architectures);
460                        apt_conf.write("  Architecture \"%s\";\n" % base_archs)
461                    else:
462                        line = re.sub(r"#ROOTFS#", self.target_rootfs, line)
463                        line = re.sub(r"#APTCONF#", self.apt_conf_dir, line)
464                        apt_conf.write(line + "\n")
465
466        target_dpkg_dir = "%s/var/lib/dpkg" % self.target_rootfs
467        bb.utils.mkdirhier(os.path.join(target_dpkg_dir, "info"))
468
469        bb.utils.mkdirhier(os.path.join(target_dpkg_dir, "updates"))
470
471        if not os.path.exists(os.path.join(target_dpkg_dir, "status")):
472            open(os.path.join(target_dpkg_dir, "status"), "w+").close()
473        if not os.path.exists(os.path.join(target_dpkg_dir, "available")):
474            open(os.path.join(target_dpkg_dir, "available"), "w+").close()
475
476    def remove_packaging_data(self):
477        bb.utils.remove(self.target_rootfs + self.d.getVar('opkglibdir'), True)
478        bb.utils.remove(self.target_rootfs + "/var/lib/dpkg/", True)
479
480    def fix_broken_dependencies(self):
481        os.environ['APT_CONFIG'] = self.apt_conf_file
482
483        cmd = "%s %s --allow-unauthenticated -f install" % (self.apt_get_cmd, self.apt_args)
484
485        try:
486            subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
487        except subprocess.CalledProcessError as e:
488            bb.fatal("Cannot fix broken dependencies. Command '%s' "
489                     "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
490
491    def list_installed(self):
492        return PMPkgsList(self.d, self.target_rootfs).list_pkgs()
493
494    def package_info(self, pkg):
495        """
496        Returns a dictionary with the package info.
497        """
498        cmd = "%s show %s" % (self.apt_cache_cmd, pkg)
499        pkg_info = super(DpkgPM, self).package_info(pkg, cmd)
500
501        pkg_arch = pkg_info[pkg]["pkgarch"]
502        pkg_filename = pkg_info[pkg]["filename"]
503        pkg_info[pkg]["filepath"] = \
504                os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
505
506        return pkg_info
507
508    def extract(self, pkg):
509        """
510        Returns the path to a tmpdir where resides the contents of a package.
511
512        Deleting the tmpdir is responsability of the caller.
513        """
514        pkg_info = self.package_info(pkg)
515        if not pkg_info:
516            bb.fatal("Unable to get information for package '%s' while "
517                     "trying to extract the package."  % pkg)
518
519        tmp_dir = super(DpkgPM, self).extract(pkg, pkg_info)
520        bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
521
522        return tmp_dir
523