1#!/usr/bin/env python3
2#
3# Build the required docker image to run package unit tests
4#
5# Script Variables:
6#   DOCKER_IMG_NAME:  <optional, the name of the docker image to generate>
7#                     default is openbmc/ubuntu-unit-test
8#   DISTRO:           <optional, the distro to build a docker image against>
9#   FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker
10#                     images to be rebuilt rather than reusing caches.>
11#   BUILD_URL:        <optional, used to detect running under CI context
12#                     (ex. Jenkins)>
13#   BRANCH:           <optional, branch to build from each of the openbmc/
14#                     repositories>
15#                     default is master, which will be used if input branch not
16#                     provided or not found
17#   UBUNTU_MIRROR:    <optional, the URL of a mirror of Ubuntu to override the
18#                     default ones in /etc/apt/sources.list>
19#                     default is empty, and no mirror is used.
20#   http_proxy        The HTTP address of the proxy server to connect to.
21#                     Default: "", proxy is not setup if this is not set
22
23import os
24import re
25import sys
26import threading
27from datetime import date
28from hashlib import sha256
29
30# typing.Dict is used for type-hints.
31from typing import Any, Callable, Dict, Iterable, Optional  # noqa: F401
32
33from sh import docker, git, nproc, uname  # type: ignore
34
35try:
36    # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'.
37    from typing import TypedDict
38except Exception:
39
40    class TypedDict(dict):  # type: ignore
41        # We need to do this to eat the 'total' argument.
42        def __init_subclass__(cls, **kwargs: Any) -> None:
43            super().__init_subclass__()
44
45
46# Declare some variables used in package definitions.
47prefix = "/usr/local"
48proc_count = nproc().strip()
49
50
51class PackageDef(TypedDict, total=False):
52    """Package Definition for packages dictionary."""
53
54    # rev [optional]: Revision of package to use.
55    rev: str
56    # url [optional]: lambda function to create URL: (package, rev) -> url.
57    url: Callable[[str, str], str]
58    # depends [optional]: List of package dependencies.
59    depends: Iterable[str]
60    # build_type [required]: Build type used for package.
61    #   Currently supported: autoconf, cmake, custom, make, meson
62    build_type: str
63    # build_steps [optional]: Steps to run for 'custom' build_type.
64    build_steps: Iterable[str]
65    # config_flags [optional]: List of options to pass configuration tool.
66    config_flags: Iterable[str]
67    # config_env [optional]: List of environment variables to set for config.
68    config_env: Iterable[str]
69    # custom_post_dl [optional]: List of steps to run after download, but
70    #   before config / build / install.
71    custom_post_dl: Iterable[str]
72    # custom_post_install [optional]: List of steps to run after install.
73    custom_post_install: Iterable[str]
74
75    # __tag [private]: Generated Docker tag name for package stage.
76    __tag: str
77    # __package [private]: Package object associated with this package.
78    __package: Any  # Type is Package, but not defined yet.
79
80
81# Packages to include in image.
82packages = {
83    "boost": PackageDef(
84        rev="1.84.0",
85        url=(
86            lambda pkg, rev: f"https://github.com/boostorg/{pkg}/releases/download/{pkg}-{rev}/{pkg}-{rev}.tar.gz"
87        ),
88        build_type="custom",
89        build_steps=[
90            (
91                "./bootstrap.sh"
92                f" --prefix={prefix} --with-libraries=context,coroutine,url"
93            ),
94            "./b2",
95            f"./b2 install --prefix={prefix} valgrind=on",
96        ],
97    ),
98    "USCiLab/cereal": PackageDef(
99        rev="v1.3.2",
100        build_type="custom",
101        build_steps=[f"cp -a include/cereal/ {prefix}/include/"],
102    ),
103    "danmar/cppcheck": PackageDef(
104        rev="2.12.1",
105        build_type="cmake",
106    ),
107    "CLIUtils/CLI11": PackageDef(
108        rev="v2.3.2",
109        build_type="cmake",
110        config_flags=[
111            "-DBUILD_TESTING=OFF",
112            "-DCLI11_BUILD_DOCS=OFF",
113            "-DCLI11_BUILD_EXAMPLES=OFF",
114        ],
115    ),
116    "fmtlib/fmt": PackageDef(
117        rev="10.1.1",
118        build_type="cmake",
119        config_flags=[
120            "-DFMT_DOC=OFF",
121            "-DFMT_TEST=OFF",
122        ],
123    ),
124    "Naios/function2": PackageDef(
125        rev="4.2.4",
126        build_type="custom",
127        build_steps=[
128            f"mkdir {prefix}/include/function2",
129            f"cp include/function2/function2.hpp {prefix}/include/function2/",
130        ],
131    ),
132    "google/googletest": PackageDef(
133        rev="v1.14.0",
134        build_type="cmake",
135        config_env=["CXXFLAGS=-std=c++20"],
136        config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"],
137    ),
138    "nghttp2/nghttp2": PackageDef(
139        rev="v1.61.0",
140        build_type="cmake",
141        config_env=["CXXFLAGS=-std=c++20"],
142        config_flags=[
143            "-DENABLE_LIB_ONLY=ON",
144            "-DENABLE_STATIC_LIB=ON",
145        ],
146    ),
147    "nlohmann/json": PackageDef(
148        rev="v3.11.2",
149        build_type="cmake",
150        config_flags=["-DJSON_BuildTests=OFF"],
151        custom_post_install=[
152            (
153                f"ln -s {prefix}/include/nlohmann/json.hpp"
154                f" {prefix}/include/json.hpp"
155            ),
156        ],
157    ),
158    "json-c/json-c": PackageDef(
159        rev="json-c-0.17-20230812",
160        build_type="cmake",
161    ),
162    "LibVNC/libvncserver": PackageDef(
163        rev="LibVNCServer-0.9.14",
164        build_type="cmake",
165    ),
166    "leethomason/tinyxml2": PackageDef(
167        rev="9.0.0",
168        build_type="cmake",
169    ),
170    "tristanpenman/valijson": PackageDef(
171        rev="v1.0.1",
172        build_type="cmake",
173        config_flags=[
174            "-Dvalijson_BUILD_TESTS=0",
175            "-Dvalijson_INSTALL_HEADERS=1",
176        ],
177    ),
178    "open-power/pdbg": PackageDef(build_type="autoconf"),
179    "openbmc/gpioplus": PackageDef(
180        depends=["openbmc/stdplus"],
181        build_type="meson",
182        config_flags=[
183            "-Dexamples=false",
184            "-Dtests=disabled",
185        ],
186    ),
187    "openbmc/phosphor-dbus-interfaces": PackageDef(
188        depends=["openbmc/sdbusplus"],
189        build_type="meson",
190        config_flags=["-Dgenerate_md=false"],
191    ),
192    "openbmc/phosphor-logging": PackageDef(
193        depends=[
194            "USCiLab/cereal",
195            "openbmc/phosphor-dbus-interfaces",
196            "openbmc/sdbusplus",
197            "openbmc/sdeventplus",
198        ],
199        build_type="meson",
200        config_flags=[
201            "-Dlibonly=true",
202            "-Dtests=disabled",
203            f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml",
204        ],
205    ),
206    "openbmc/phosphor-objmgr": PackageDef(
207        depends=[
208            "CLIUtils/CLI11",
209            "boost",
210            "leethomason/tinyxml2",
211            "openbmc/phosphor-dbus-interfaces",
212            "openbmc/phosphor-logging",
213            "openbmc/sdbusplus",
214        ],
215        build_type="meson",
216        config_flags=[
217            "-Dtests=disabled",
218        ],
219    ),
220    "openbmc/libpeci": PackageDef(
221        build_type="meson",
222        config_flags=[
223            "-Draw-peci=disabled",
224        ],
225    ),
226    "openbmc/libpldm": PackageDef(
227        build_type="meson",
228        config_flags=[
229            "-Dabi=deprecated,stable",
230            "-Doem-ibm=enabled",
231            "-Dtests=disabled",
232        ],
233    ),
234    "openbmc/sdbusplus": PackageDef(
235        build_type="meson",
236        custom_post_dl=[
237            "cd tools",
238            f"./setup.py install --root=/ --prefix={prefix}",
239            "cd ..",
240        ],
241        config_flags=[
242            "-Dexamples=disabled",
243            "-Dtests=disabled",
244        ],
245    ),
246    "openbmc/sdeventplus": PackageDef(
247        depends=[
248            "Naios/function2",
249            "openbmc/stdplus",
250        ],
251        build_type="meson",
252        config_flags=[
253            "-Dexamples=false",
254            "-Dtests=disabled",
255        ],
256    ),
257    "openbmc/stdplus": PackageDef(
258        depends=[
259            "fmtlib/fmt",
260            "google/googletest",
261            "Naios/function2",
262        ],
263        build_type="meson",
264        config_flags=[
265            "-Dexamples=false",
266            "-Dtests=disabled",
267            "-Dgtest=enabled",
268        ],
269    ),
270}  # type: Dict[str, PackageDef]
271
272# Define common flags used for builds
273configure_flags = " ".join(
274    [
275        f"--prefix={prefix}",
276    ]
277)
278cmake_flags = " ".join(
279    [
280        "-DBUILD_SHARED_LIBS=ON",
281        "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
282        f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}",
283        "-GNinja",
284        "-DCMAKE_MAKE_PROGRAM=ninja",
285    ]
286)
287meson_flags = " ".join(
288    [
289        "--wrap-mode=nodownload",
290        f"-Dprefix={prefix}",
291    ]
292)
293
294
295class Package(threading.Thread):
296    """Class used to build the Docker stages for each package.
297
298    Generally, this class should not be instantiated directly but through
299    Package.generate_all().
300    """
301
302    # Copy the packages dictionary.
303    packages = packages.copy()
304
305    # Lock used for thread-safety.
306    lock = threading.Lock()
307
308    def __init__(self, pkg: str):
309        """pkg - The name of this package (ex. foo/bar )"""
310        super(Package, self).__init__()
311
312        self.package = pkg
313        self.exception = None  # type: Optional[Exception]
314
315        # Reference to this package's
316        self.pkg_def = Package.packages[pkg]
317        self.pkg_def["__package"] = self
318
319    def run(self) -> None:
320        """Thread 'run' function.  Builds the Docker stage."""
321
322        # In case this package has no rev, fetch it from Github.
323        self._update_rev()
324
325        # Find all the Package objects that this package depends on.
326        #   This section is locked because we are looking into another
327        #   package's PackageDef dict, which could be being modified.
328        Package.lock.acquire()
329        deps: Iterable[Package] = [
330            Package.packages[deppkg]["__package"]
331            for deppkg in self.pkg_def.get("depends", [])
332        ]
333        Package.lock.release()
334
335        # Wait until all the depends finish building.  We need them complete
336        # for the "COPY" commands.
337        for deppkg in deps:
338            deppkg.join()
339
340        # Generate this package's Dockerfile.
341        dockerfile = f"""
342FROM {docker_base_img_name}
343{self._df_copycmds()}
344{self._df_build()}
345"""
346
347        # Generate the resulting tag name and save it to the PackageDef.
348        #   This section is locked because we are modifying the PackageDef,
349        #   which can be accessed by other threads.
350        Package.lock.acquire()
351        tag = Docker.tagname(self._stagename(), dockerfile)
352        self.pkg_def["__tag"] = tag
353        Package.lock.release()
354
355        # Do the build / save any exceptions.
356        try:
357            Docker.build(self.package, tag, dockerfile)
358        except Exception as e:
359            self.exception = e
360
361    @classmethod
362    def generate_all(cls) -> None:
363        """Ensure a Docker stage is created for all defined packages.
364
365        These are done in parallel but with appropriate blocking per
366        package 'depends' specifications.
367        """
368
369        # Create a Package for each defined package.
370        pkg_threads = [Package(p) for p in cls.packages.keys()]
371
372        # Start building them all.
373        #   This section is locked because threads depend on each other,
374        #   based on the packages, and they cannot 'join' on a thread
375        #   which is not yet started.  Adding a lock here allows all the
376        #   threads to start before they 'join' their dependencies.
377        Package.lock.acquire()
378        for t in pkg_threads:
379            t.start()
380        Package.lock.release()
381
382        # Wait for completion.
383        for t in pkg_threads:
384            t.join()
385            # Check if the thread saved off its own exception.
386            if t.exception:
387                print(f"Package {t.package} failed!", file=sys.stderr)
388                raise t.exception
389
390    @staticmethod
391    def df_all_copycmds() -> str:
392        """Formulate the Dockerfile snippet necessary to copy all packages
393        into the final image.
394        """
395        return Package.df_copycmds_set(Package.packages.keys())
396
397    @classmethod
398    def depcache(cls) -> str:
399        """Create the contents of the '/tmp/depcache'.
400        This file is a comma-separated list of "<pkg>:<rev>".
401        """
402
403        # This needs to be sorted for consistency.
404        depcache = ""
405        for pkg in sorted(cls.packages.keys()):
406            depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"])
407        return depcache
408
409    def _update_rev(self) -> None:
410        """Look up the HEAD for missing a static rev."""
411
412        if "rev" in self.pkg_def:
413            return
414
415        # Check if Jenkins/Gerrit gave us a revision and use it.
416        if gerrit_project == self.package and gerrit_rev:
417            print(
418                f"Found Gerrit revision for {self.package}: {gerrit_rev}",
419                file=sys.stderr,
420            )
421            self.pkg_def["rev"] = gerrit_rev
422            return
423
424        # Ask Github for all the branches.
425        lookup = git(
426            "ls-remote", "--heads", f"https://github.com/{self.package}"
427        )
428
429        # Find the branch matching {branch} (or fallback to master).
430        #   This section is locked because we are modifying the PackageDef.
431        Package.lock.acquire()
432        for line in lookup.split("\n"):
433            if re.fullmatch(f".*{branch}$", line.strip()):
434                self.pkg_def["rev"] = line.split()[0]
435                break
436            elif (
437                "refs/heads/master" in line or "refs/heads/main" in line
438            ) and "rev" not in self.pkg_def:
439                self.pkg_def["rev"] = line.split()[0]
440        Package.lock.release()
441
442    def _stagename(self) -> str:
443        """Create a name for the Docker stage associated with this pkg."""
444        return self.package.replace("/", "-").lower()
445
446    def _url(self) -> str:
447        """Get the URL for this package."""
448        rev = self.pkg_def["rev"]
449
450        # If the lambda exists, call it.
451        if "url" in self.pkg_def:
452            return self.pkg_def["url"](self.package, rev)
453
454        # Default to the github archive URL.
455        return f"https://github.com/{self.package}/archive/{rev}.tar.gz"
456
457    def _cmd_download(self) -> str:
458        """Formulate the command necessary to download and unpack to source."""
459
460        url = self._url()
461        if ".tar." not in url:
462            raise NotImplementedError(
463                f"Unhandled download type for {self.package}: {url}"
464            )
465
466        cmd = f"curl -L {url} | tar -x"
467
468        if url.endswith(".bz2"):
469            cmd += "j"
470        elif url.endswith(".gz"):
471            cmd += "z"
472        else:
473            raise NotImplementedError(
474                f"Unknown tar flags needed for {self.package}: {url}"
475            )
476
477        return cmd
478
479    def _cmd_cd_srcdir(self) -> str:
480        """Formulate the command necessary to 'cd' into the source dir."""
481        return f"cd {self.package.split('/')[-1]}*"
482
483    def _df_copycmds(self) -> str:
484        """Formulate the dockerfile snippet necessary to COPY all depends."""
485
486        if "depends" not in self.pkg_def:
487            return ""
488        return Package.df_copycmds_set(self.pkg_def["depends"])
489
490    @staticmethod
491    def df_copycmds_set(pkgs: Iterable[str]) -> str:
492        """Formulate the Dockerfile snippet necessary to COPY a set of
493        packages into a Docker stage.
494        """
495
496        copy_cmds = ""
497
498        # Sort the packages for consistency.
499        for p in sorted(pkgs):
500            tag = Package.packages[p]["__tag"]
501            copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n"
502            # Workaround for upstream docker bug and multiple COPY cmds
503            # https://github.com/moby/moby/issues/37965
504            copy_cmds += "RUN true\n"
505
506        return copy_cmds
507
508    def _df_build(self) -> str:
509        """Formulate the Dockerfile snippet necessary to download, build, and
510        install a package into a Docker stage.
511        """
512
513        # Download and extract source.
514        result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && "
515
516        # Handle 'custom_post_dl' commands.
517        custom_post_dl = self.pkg_def.get("custom_post_dl")
518        if custom_post_dl:
519            result += " && ".join(custom_post_dl) + " && "
520
521        # Build and install package based on 'build_type'.
522        build_type = self.pkg_def["build_type"]
523        if build_type == "autoconf":
524            result += self._cmd_build_autoconf()
525        elif build_type == "cmake":
526            result += self._cmd_build_cmake()
527        elif build_type == "custom":
528            result += self._cmd_build_custom()
529        elif build_type == "make":
530            result += self._cmd_build_make()
531        elif build_type == "meson":
532            result += self._cmd_build_meson()
533        else:
534            raise NotImplementedError(
535                f"Unhandled build type for {self.package}: {build_type}"
536            )
537
538        # Handle 'custom_post_install' commands.
539        custom_post_install = self.pkg_def.get("custom_post_install")
540        if custom_post_install:
541            result += " && " + " && ".join(custom_post_install)
542
543        return result
544
545    def _cmd_build_autoconf(self) -> str:
546        options = " ".join(self.pkg_def.get("config_flags", []))
547        env = " ".join(self.pkg_def.get("config_env", []))
548        result = "./bootstrap.sh && "
549        result += f"{env} ./configure {configure_flags} {options} && "
550        result += f"make -j{proc_count} && make install"
551        return result
552
553    def _cmd_build_cmake(self) -> str:
554        options = " ".join(self.pkg_def.get("config_flags", []))
555        env = " ".join(self.pkg_def.get("config_env", []))
556        result = "mkdir builddir && cd builddir && "
557        result += f"{env} cmake {cmake_flags} {options} .. && "
558        result += "cmake --build . --target all && "
559        result += "cmake --build . --target install && "
560        result += "cd .."
561        return result
562
563    def _cmd_build_custom(self) -> str:
564        return " && ".join(self.pkg_def.get("build_steps", []))
565
566    def _cmd_build_make(self) -> str:
567        return f"make -j{proc_count} && make install"
568
569    def _cmd_build_meson(self) -> str:
570        options = " ".join(self.pkg_def.get("config_flags", []))
571        env = " ".join(self.pkg_def.get("config_env", []))
572        result = f"{env} meson setup builddir {meson_flags} {options} && "
573        result += "ninja -C builddir && ninja -C builddir install"
574        return result
575
576
577class Docker:
578    """Class to assist with Docker interactions.  All methods are static."""
579
580    @staticmethod
581    def timestamp() -> str:
582        """Generate a timestamp for today using the ISO week."""
583        today = date.today().isocalendar()
584        return f"{today[0]}-W{today[1]:02}"
585
586    @staticmethod
587    def tagname(pkgname: Optional[str], dockerfile: str) -> str:
588        """Generate a tag name for a package using a hash of the Dockerfile."""
589        result = docker_image_name
590        if pkgname:
591            result += "-" + pkgname
592
593        result += ":" + Docker.timestamp()
594        result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16]
595
596        return result
597
598    @staticmethod
599    def build(pkg: str, tag: str, dockerfile: str) -> None:
600        """Build a docker image using the Dockerfile and tagging it with 'tag'."""
601
602        # If we're not forcing builds, check if it already exists and skip.
603        if not force_build:
604            if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'):
605                print(
606                    f"Image {tag} already exists.  Skipping.", file=sys.stderr
607                )
608                return
609
610        # Build it.
611        #   Capture the output of the 'docker build' command and send it to
612        #   stderr (prefixed with the package name).  This allows us to see
613        #   progress but not pollute stdout.  Later on we output the final
614        #   docker tag to stdout and we want to keep that pristine.
615        #
616        #   Other unusual flags:
617        #       --no-cache: Bypass the Docker cache if 'force_build'.
618        #       --force-rm: Clean up Docker processes if they fail.
619        docker.build(
620            proxy_args,
621            "--network=host",
622            "--force-rm",
623            "--no-cache=true" if force_build else "--no-cache=false",
624            "-t",
625            tag,
626            "-",
627            _in=dockerfile,
628            _out=(
629                lambda line: print(
630                    pkg + ":", line, end="", file=sys.stderr, flush=True
631                )
632            ),
633            _err_to_out=True,
634        )
635
636
637# Read a bunch of environment variables.
638docker_image_name = os.environ.get(
639    "DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test"
640)
641force_build = os.environ.get("FORCE_DOCKER_BUILD")
642is_automated_ci_build = os.environ.get("BUILD_URL", False)
643distro = os.environ.get("DISTRO", "ubuntu:noble")
644branch = os.environ.get("BRANCH", "master")
645ubuntu_mirror = os.environ.get("UBUNTU_MIRROR")
646http_proxy = os.environ.get("http_proxy")
647
648gerrit_project = os.environ.get("GERRIT_PROJECT")
649gerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION")
650
651# Ensure appropriate docker build output to see progress and identify
652# any issues
653os.environ["BUILDKIT_PROGRESS"] = "plain"
654
655# Set up some common variables.
656username = os.environ.get("USER", "root")
657homedir = os.environ.get("HOME", "/root")
658gid = os.getgid()
659uid = os.getuid()
660
661# Use well-known constants if user is root
662if username == "root":
663    homedir = "/root"
664    gid = 0
665    uid = 0
666
667# Determine the architecture for Docker.
668arch = uname("-m").strip()
669if arch == "ppc64le":
670    docker_base = "ppc64le/"
671elif arch == "x86_64":
672    docker_base = ""
673elif arch == "aarch64":
674    docker_base = "arm64v8/"
675else:
676    print(
677        f"Unsupported system architecture({arch}) found for docker image",
678        file=sys.stderr,
679    )
680    sys.exit(1)
681
682# Special flags if setting up a deb mirror.
683mirror = ""
684if "ubuntu" in distro and ubuntu_mirror:
685    mirror = f"""
686RUN echo "deb {ubuntu_mirror} \
687        $(. /etc/os-release && echo $VERSION_CODENAME) \
688        main restricted universe multiverse" > /etc/apt/sources.list && \\
689    echo "deb {ubuntu_mirror} \
690        $(. /etc/os-release && echo $VERSION_CODENAME)-updates \
691            main restricted universe multiverse" >> /etc/apt/sources.list && \\
692    echo "deb {ubuntu_mirror} \
693        $(. /etc/os-release && echo $VERSION_CODENAME)-security \
694            main restricted universe multiverse" >> /etc/apt/sources.list && \\
695    echo "deb {ubuntu_mirror} \
696        $(. /etc/os-release && echo $VERSION_CODENAME)-proposed \
697            main restricted universe multiverse" >> /etc/apt/sources.list && \\
698    echo "deb {ubuntu_mirror} \
699        $(. /etc/os-release && echo $VERSION_CODENAME)-backports \
700            main restricted universe multiverse" >> /etc/apt/sources.list
701"""
702
703# Special flags for proxying.
704proxy_cmd = ""
705proxy_keyserver = ""
706proxy_args = []
707if http_proxy:
708    proxy_cmd = f"""
709RUN echo "[http]" >> {homedir}/.gitconfig && \
710    echo "proxy = {http_proxy}" >> {homedir}/.gitconfig
711"""
712    proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}"
713
714    proxy_args.extend(
715        [
716            "--build-arg",
717            f"http_proxy={http_proxy}",
718            "--build-arg",
719            f"https_proxy={http_proxy}",
720        ]
721    )
722
723# Create base Dockerfile.
724dockerfile_base = f"""
725FROM {docker_base}{distro}
726
727{mirror}
728
729ENV DEBIAN_FRONTEND noninteractive
730
731ENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/"
732
733# Sometimes the ubuntu key expires and we need a way to force an execution
734# of the apt-get commands for the dbgsym-keyring.  When this happens we see
735# an error like: "Release: The following signatures were invalid:"
736# Insert a bogus echo that we can change here when we get this error to force
737# the update.
738RUN echo "ubuntu keyserver rev as of 2021-04-21"
739
740# We need the keys to be imported for dbgsym repos
741# New releases have a package, older ones fall back to manual fetching
742# https://wiki.ubuntu.com/Debug%20Symbol%20Packages
743# Known issue with gpg to get keys via proxy -
744# https://bugs.launchpad.net/ubuntu/+source/gnupg2/+bug/1788190, hence using
745# curl to get keys.
746RUN apt-get update && apt-get dist-upgrade -yy && \
747    ( apt-get install -yy gpgv ubuntu-dbgsym-keyring || \
748        ( apt-get install -yy dirmngr curl && \
749          curl -sSL \
750          'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622' \
751          | apt-key add - ))
752
753# Parse the current repo list into a debug repo list
754RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' \
755        /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list
756
757# Remove non-existent debug repos
758RUN sed -i '/-\\(backports\\|security\\) /d' /etc/apt/sources.list.d/debug.list
759
760RUN cat /etc/apt/sources.list.d/debug.list
761
762RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \
763    abi-compliance-checker \
764    abi-dumper \
765    autoconf \
766    autoconf-archive \
767    bison \
768    clang-17 \
769    clang-format-17 \
770    clang-tidy-17 \
771    clang-tools-17 \
772    cmake \
773    curl \
774    dbus \
775    device-tree-compiler \
776    flex \
777    g++-13 \
778    gcc-13 \
779    git \
780    iproute2 \
781    iputils-ping \
782    libaudit-dev \
783    libc6-dbg \
784    libc6-dev \
785    libconfig++-dev \
786    libcryptsetup-dev \
787    libdbus-1-dev \
788    libevdev-dev \
789    libgpiod-dev \
790    libi2c-dev \
791    libjpeg-dev \
792    libjson-perl \
793    libldap2-dev \
794    libmimetic-dev \
795    libnl-3-dev \
796    libnl-genl-3-dev \
797    libpam0g-dev \
798    libpciaccess-dev \
799    libperlio-gzip-perl \
800    libpng-dev \
801    libprotobuf-dev \
802    libsnmp-dev \
803    libssl-dev \
804    libsystemd-dev \
805    libtool \
806    liburing-dev \
807    libxml2-utils \
808    libxml-simple-perl \
809    ninja-build \
810    npm \
811    pkg-config \
812    protobuf-compiler \
813    python3 \
814    python3-dev\
815    python3-git \
816    python3-mako \
817    python3-pip \
818    python3-protobuf \
819    python3-setuptools \
820    python3-socks \
821    python3-yaml \
822    rsync \
823    shellcheck \
824    socat \
825    sudo \
826    systemd \
827    valgrind \
828    vim \
829    wget \
830    xxd
831
832RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 13 \
833  --slave /usr/bin/g++ g++ /usr/bin/g++-13 \
834  --slave /usr/bin/gcov gcov /usr/bin/gcov-13 \
835  --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-13 \
836  --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-13
837RUN update-alternatives --remove cpp /usr/bin/cpp && \
838    update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-13 13
839
840RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 1000 \
841  --slave /usr/bin/clang++ clang++ /usr/bin/clang++-17 \
842  --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-17 \
843  --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-17 \
844  --slave /usr/bin/run-clang-tidy run-clang-tidy.py \
845        /usr/bin/run-clang-tidy-17 \
846  --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-17
847
848"""
849
850if is_automated_ci_build:
851    dockerfile_base += f"""
852# Run an arbitrary command to pollute the docker cache regularly force us
853# to re-run `apt-get update` daily.
854RUN echo {Docker.timestamp()}
855RUN apt-get update && apt-get dist-upgrade -yy
856
857"""
858
859dockerfile_base += """
860RUN pip3 install --break-system-packages \
861        beautysh \
862        black \
863        codespell \
864        flake8 \
865        gcovr \
866        gitlint \
867        inflection \
868        isort \
869        jsonschema \
870        meson==1.3.0 \
871        requests
872
873RUN npm install -g \
874        eslint@v8.56.0 eslint-plugin-json@v3.1.0 \
875        markdownlint-cli@latest \
876        prettier@latest
877"""
878
879# Build the base and stage docker images.
880docker_base_img_name = Docker.tagname("base", dockerfile_base)
881Docker.build("base", docker_base_img_name, dockerfile_base)
882Package.generate_all()
883
884# Create the final Dockerfile.
885dockerfile = f"""
886# Build the final output image
887FROM {docker_base_img_name}
888{Package.df_all_copycmds()}
889
890# Some of our infrastructure still relies on the presence of this file
891# even though it is no longer needed to rebuild the docker environment
892# NOTE: The file is sorted to ensure the ordering is stable.
893RUN echo '{Package.depcache()}' > /tmp/depcache
894
895# Ensure the group, user, and home directory are created (or rename them if
896# they already exist).
897RUN if grep -q ":{gid}:" /etc/group ; then \
898        groupmod -n {username} $(awk -F : '{{ if ($3 == {gid}) {{ print $1 }} }}' /etc/group) ; \
899    else \
900        groupadd -f -g {gid} {username} ; \
901    fi
902RUN mkdir -p "{os.path.dirname(homedir)}"
903RUN if grep -q ":{uid}:" /etc/passwd ; then \
904        usermod -l {username} -d {homedir} -m $(awk -F : '{{ if ($3 == {uid}) {{ print $1 }} }}' /etc/passwd) ; \
905    else \
906        useradd -d {homedir} -m -u {uid} -g {gid} {username} ; \
907    fi
908RUN sed -i '1iDefaults umask=000' /etc/sudoers
909RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
910
911# Ensure user has ability to write to /usr/local for different tool
912# and data installs
913RUN chown -R {username}:{username} /usr/local/share
914
915# Update library cache
916RUN ldconfig
917
918{proxy_cmd}
919
920RUN /bin/bash
921"""
922
923# Do the final docker build
924docker_final_img_name = Docker.tagname(None, dockerfile)
925Docker.build("final", docker_final_img_name, dockerfile)
926
927# Print the tag of the final image.
928print(docker_final_img_name)
929