1#!/usr/bin/env python3 2# 3# Build the required docker image to run package unit tests 4# 5# Script Variables: 6# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 7# default is openbmc/ubuntu-unit-test 8# DISTRO: <optional, the distro to build a docker image against> 9# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 10# images to be rebuilt rather than reusing caches.> 11# BUILD_URL: <optional, used to detect running under CI context 12# (ex. Jenkins)> 13# BRANCH: <optional, branch to build from each of the openbmc/ 14# repositories> 15# default is master, which will be used if input branch not 16# provided or not found 17# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 18# default ones in /etc/apt/sources.list> 19# default is empty, and no mirror is used. 20# http_proxy The HTTP address of the proxy server to connect to. 21# Default: "", proxy is not setup if this is not set 22 23import os 24import re 25import sys 26import threading 27from datetime import date 28from hashlib import sha256 29 30# typing.Dict is used for type-hints. 31from typing import Any, Callable, Dict, Iterable, Optional # noqa: F401 32 33from sh import docker, git, nproc, uname # type: ignore 34 35try: 36 # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 37 from typing import TypedDict 38except Exception: 39 40 class TypedDict(dict): # type: ignore 41 # We need to do this to eat the 'total' argument. 42 def __init_subclass__(cls, **kwargs: Any) -> None: 43 super().__init_subclass__() 44 45 46# Declare some variables used in package definitions. 47prefix = "/usr/local" 48proc_count = nproc().strip() 49 50 51class PackageDef(TypedDict, total=False): 52 """Package Definition for packages dictionary.""" 53 54 # rev [optional]: Revision of package to use. 55 rev: str 56 # url [optional]: lambda function to create URL: (package, rev) -> url. 57 url: Callable[[str, str], str] 58 # depends [optional]: List of package dependencies. 59 depends: Iterable[str] 60 # build_type [required]: Build type used for package. 61 # Currently supported: autoconf, cmake, custom, make, meson 62 build_type: str 63 # build_steps [optional]: Steps to run for 'custom' build_type. 64 build_steps: Iterable[str] 65 # config_flags [optional]: List of options to pass configuration tool. 66 config_flags: Iterable[str] 67 # config_env [optional]: List of environment variables to set for config. 68 config_env: Iterable[str] 69 # custom_post_dl [optional]: List of steps to run after download, but 70 # before config / build / install. 71 custom_post_dl: Iterable[str] 72 # custom_post_install [optional]: List of steps to run after install. 73 custom_post_install: Iterable[str] 74 75 # __tag [private]: Generated Docker tag name for package stage. 76 __tag: str 77 # __package [private]: Package object associated with this package. 78 __package: Any # Type is Package, but not defined yet. 79 80 81# Packages to include in image. 82packages = { 83 "boost": PackageDef( 84 rev="1.84.0", 85 url=( 86 lambda pkg, rev: f"https://github.com/boostorg/{pkg}/releases/download/{pkg}-{rev}/{pkg}-{rev}.tar.gz" 87 ), 88 build_type="custom", 89 build_steps=[ 90 ( 91 "./bootstrap.sh" 92 f" --prefix={prefix} --with-libraries=context,coroutine,url" 93 ), 94 "./b2", 95 f"./b2 install --prefix={prefix} valgrind=on", 96 ], 97 ), 98 "USCiLab/cereal": PackageDef( 99 rev="v1.3.2", 100 build_type="custom", 101 build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 102 ), 103 "danmar/cppcheck": PackageDef( 104 rev="2.12.1", 105 build_type="cmake", 106 ), 107 "CLIUtils/CLI11": PackageDef( 108 rev="v2.3.2", 109 build_type="cmake", 110 config_flags=[ 111 "-DBUILD_TESTING=OFF", 112 "-DCLI11_BUILD_DOCS=OFF", 113 "-DCLI11_BUILD_EXAMPLES=OFF", 114 ], 115 ), 116 "fmtlib/fmt": PackageDef( 117 rev="10.1.1", 118 build_type="cmake", 119 config_flags=[ 120 "-DFMT_DOC=OFF", 121 "-DFMT_TEST=OFF", 122 ], 123 ), 124 "Naios/function2": PackageDef( 125 rev="4.2.4", 126 build_type="custom", 127 build_steps=[ 128 f"mkdir {prefix}/include/function2", 129 f"cp include/function2/function2.hpp {prefix}/include/function2/", 130 ], 131 ), 132 "google/googletest": PackageDef( 133 rev="v1.14.0", 134 build_type="cmake", 135 config_env=["CXXFLAGS=-std=c++20"], 136 config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 137 ), 138 "nghttp2/nghttp2": PackageDef( 139 rev="v1.61.0", 140 build_type="cmake", 141 config_env=["CXXFLAGS=-std=c++20"], 142 config_flags=[ 143 "-DENABLE_LIB_ONLY=ON", 144 "-DENABLE_STATIC_LIB=ON", 145 ], 146 ), 147 "nlohmann/json": PackageDef( 148 rev="v3.11.2", 149 build_type="cmake", 150 config_flags=["-DJSON_BuildTests=OFF"], 151 custom_post_install=[ 152 ( 153 f"ln -s {prefix}/include/nlohmann/json.hpp" 154 f" {prefix}/include/json.hpp" 155 ), 156 ], 157 ), 158 "json-c/json-c": PackageDef( 159 rev="json-c-0.17-20230812", 160 build_type="cmake", 161 ), 162 "LibVNC/libvncserver": PackageDef( 163 rev="LibVNCServer-0.9.14", 164 build_type="cmake", 165 ), 166 "leethomason/tinyxml2": PackageDef( 167 rev="9.0.0", 168 build_type="cmake", 169 ), 170 "tristanpenman/valijson": PackageDef( 171 rev="v1.0.1", 172 build_type="cmake", 173 config_flags=[ 174 "-Dvalijson_BUILD_TESTS=0", 175 "-Dvalijson_INSTALL_HEADERS=1", 176 ], 177 ), 178 "open-power/pdbg": PackageDef(build_type="autoconf"), 179 "openbmc/gpioplus": PackageDef( 180 depends=["openbmc/stdplus"], 181 build_type="meson", 182 config_flags=[ 183 "-Dexamples=false", 184 "-Dtests=disabled", 185 ], 186 ), 187 "openbmc/phosphor-dbus-interfaces": PackageDef( 188 depends=["openbmc/sdbusplus"], 189 build_type="meson", 190 config_flags=["-Dgenerate_md=false"], 191 ), 192 "openbmc/phosphor-logging": PackageDef( 193 depends=[ 194 "USCiLab/cereal", 195 "openbmc/phosphor-dbus-interfaces", 196 "openbmc/sdbusplus", 197 "openbmc/sdeventplus", 198 ], 199 build_type="meson", 200 config_flags=[ 201 "-Dlibonly=true", 202 "-Dtests=disabled", 203 f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml", 204 ], 205 ), 206 "openbmc/phosphor-objmgr": PackageDef( 207 depends=[ 208 "CLIUtils/CLI11", 209 "boost", 210 "leethomason/tinyxml2", 211 "openbmc/phosphor-dbus-interfaces", 212 "openbmc/phosphor-logging", 213 "openbmc/sdbusplus", 214 ], 215 build_type="meson", 216 config_flags=[ 217 "-Dtests=disabled", 218 ], 219 ), 220 "openbmc/libpeci": PackageDef( 221 build_type="meson", 222 config_flags=[ 223 "-Draw-peci=disabled", 224 ], 225 ), 226 "openbmc/libpldm": PackageDef( 227 build_type="meson", 228 config_flags=[ 229 "-Dabi=deprecated,stable", 230 "-Doem-ibm=enabled", 231 "-Dtests=disabled", 232 ], 233 ), 234 "openbmc/sdbusplus": PackageDef( 235 build_type="meson", 236 custom_post_dl=[ 237 "cd tools", 238 f"./setup.py install --root=/ --prefix={prefix}", 239 "cd ..", 240 ], 241 config_flags=[ 242 "-Dexamples=disabled", 243 "-Dtests=disabled", 244 ], 245 ), 246 "openbmc/sdeventplus": PackageDef( 247 depends=[ 248 "Naios/function2", 249 "openbmc/stdplus", 250 ], 251 build_type="meson", 252 config_flags=[ 253 "-Dexamples=false", 254 "-Dtests=disabled", 255 ], 256 ), 257 "openbmc/stdplus": PackageDef( 258 depends=[ 259 "fmtlib/fmt", 260 "google/googletest", 261 "Naios/function2", 262 ], 263 build_type="meson", 264 config_flags=[ 265 "-Dexamples=false", 266 "-Dtests=disabled", 267 "-Dgtest=enabled", 268 ], 269 ), 270} # type: Dict[str, PackageDef] 271 272# Define common flags used for builds 273configure_flags = " ".join( 274 [ 275 f"--prefix={prefix}", 276 ] 277) 278cmake_flags = " ".join( 279 [ 280 "-DBUILD_SHARED_LIBS=ON", 281 "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 282 f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 283 "-GNinja", 284 "-DCMAKE_MAKE_PROGRAM=ninja", 285 ] 286) 287meson_flags = " ".join( 288 [ 289 "--wrap-mode=nodownload", 290 f"-Dprefix={prefix}", 291 ] 292) 293 294 295class Package(threading.Thread): 296 """Class used to build the Docker stages for each package. 297 298 Generally, this class should not be instantiated directly but through 299 Package.generate_all(). 300 """ 301 302 # Copy the packages dictionary. 303 packages = packages.copy() 304 305 # Lock used for thread-safety. 306 lock = threading.Lock() 307 308 def __init__(self, pkg: str): 309 """pkg - The name of this package (ex. foo/bar )""" 310 super(Package, self).__init__() 311 312 self.package = pkg 313 self.exception = None # type: Optional[Exception] 314 315 # Reference to this package's 316 self.pkg_def = Package.packages[pkg] 317 self.pkg_def["__package"] = self 318 319 def run(self) -> None: 320 """Thread 'run' function. Builds the Docker stage.""" 321 322 # In case this package has no rev, fetch it from Github. 323 self._update_rev() 324 325 # Find all the Package objects that this package depends on. 326 # This section is locked because we are looking into another 327 # package's PackageDef dict, which could be being modified. 328 Package.lock.acquire() 329 deps: Iterable[Package] = [ 330 Package.packages[deppkg]["__package"] 331 for deppkg in self.pkg_def.get("depends", []) 332 ] 333 Package.lock.release() 334 335 # Wait until all the depends finish building. We need them complete 336 # for the "COPY" commands. 337 for deppkg in deps: 338 deppkg.join() 339 340 # Generate this package's Dockerfile. 341 dockerfile = f""" 342FROM {docker_base_img_name} 343{self._df_copycmds()} 344{self._df_build()} 345""" 346 347 # Generate the resulting tag name and save it to the PackageDef. 348 # This section is locked because we are modifying the PackageDef, 349 # which can be accessed by other threads. 350 Package.lock.acquire() 351 tag = Docker.tagname(self._stagename(), dockerfile) 352 self.pkg_def["__tag"] = tag 353 Package.lock.release() 354 355 # Do the build / save any exceptions. 356 try: 357 Docker.build(self.package, tag, dockerfile) 358 except Exception as e: 359 self.exception = e 360 361 @classmethod 362 def generate_all(cls) -> None: 363 """Ensure a Docker stage is created for all defined packages. 364 365 These are done in parallel but with appropriate blocking per 366 package 'depends' specifications. 367 """ 368 369 # Create a Package for each defined package. 370 pkg_threads = [Package(p) for p in cls.packages.keys()] 371 372 # Start building them all. 373 # This section is locked because threads depend on each other, 374 # based on the packages, and they cannot 'join' on a thread 375 # which is not yet started. Adding a lock here allows all the 376 # threads to start before they 'join' their dependencies. 377 Package.lock.acquire() 378 for t in pkg_threads: 379 t.start() 380 Package.lock.release() 381 382 # Wait for completion. 383 for t in pkg_threads: 384 t.join() 385 # Check if the thread saved off its own exception. 386 if t.exception: 387 print(f"Package {t.package} failed!", file=sys.stderr) 388 raise t.exception 389 390 @staticmethod 391 def df_all_copycmds() -> str: 392 """Formulate the Dockerfile snippet necessary to copy all packages 393 into the final image. 394 """ 395 return Package.df_copycmds_set(Package.packages.keys()) 396 397 @classmethod 398 def depcache(cls) -> str: 399 """Create the contents of the '/tmp/depcache'. 400 This file is a comma-separated list of "<pkg>:<rev>". 401 """ 402 403 # This needs to be sorted for consistency. 404 depcache = "" 405 for pkg in sorted(cls.packages.keys()): 406 depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 407 return depcache 408 409 def _update_rev(self) -> None: 410 """Look up the HEAD for missing a static rev.""" 411 412 if "rev" in self.pkg_def: 413 return 414 415 # Check if Jenkins/Gerrit gave us a revision and use it. 416 if gerrit_project == self.package and gerrit_rev: 417 print( 418 f"Found Gerrit revision for {self.package}: {gerrit_rev}", 419 file=sys.stderr, 420 ) 421 self.pkg_def["rev"] = gerrit_rev 422 return 423 424 # Ask Github for all the branches. 425 lookup = git( 426 "ls-remote", "--heads", f"https://github.com/{self.package}" 427 ) 428 429 # Find the branch matching {branch} (or fallback to master). 430 # This section is locked because we are modifying the PackageDef. 431 Package.lock.acquire() 432 for line in lookup.split("\n"): 433 if re.fullmatch(f".*{branch}$", line.strip()): 434 self.pkg_def["rev"] = line.split()[0] 435 break 436 elif ( 437 "refs/heads/master" in line or "refs/heads/main" in line 438 ) and "rev" not in self.pkg_def: 439 self.pkg_def["rev"] = line.split()[0] 440 Package.lock.release() 441 442 def _stagename(self) -> str: 443 """Create a name for the Docker stage associated with this pkg.""" 444 return self.package.replace("/", "-").lower() 445 446 def _url(self) -> str: 447 """Get the URL for this package.""" 448 rev = self.pkg_def["rev"] 449 450 # If the lambda exists, call it. 451 if "url" in self.pkg_def: 452 return self.pkg_def["url"](self.package, rev) 453 454 # Default to the github archive URL. 455 return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 456 457 def _cmd_download(self) -> str: 458 """Formulate the command necessary to download and unpack to source.""" 459 460 url = self._url() 461 if ".tar." not in url: 462 raise NotImplementedError( 463 f"Unhandled download type for {self.package}: {url}" 464 ) 465 466 cmd = f"curl -L {url} | tar -x" 467 468 if url.endswith(".bz2"): 469 cmd += "j" 470 elif url.endswith(".gz"): 471 cmd += "z" 472 else: 473 raise NotImplementedError( 474 f"Unknown tar flags needed for {self.package}: {url}" 475 ) 476 477 return cmd 478 479 def _cmd_cd_srcdir(self) -> str: 480 """Formulate the command necessary to 'cd' into the source dir.""" 481 return f"cd {self.package.split('/')[-1]}*" 482 483 def _df_copycmds(self) -> str: 484 """Formulate the dockerfile snippet necessary to COPY all depends.""" 485 486 if "depends" not in self.pkg_def: 487 return "" 488 return Package.df_copycmds_set(self.pkg_def["depends"]) 489 490 @staticmethod 491 def df_copycmds_set(pkgs: Iterable[str]) -> str: 492 """Formulate the Dockerfile snippet necessary to COPY a set of 493 packages into a Docker stage. 494 """ 495 496 copy_cmds = "" 497 498 # Sort the packages for consistency. 499 for p in sorted(pkgs): 500 tag = Package.packages[p]["__tag"] 501 copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 502 # Workaround for upstream docker bug and multiple COPY cmds 503 # https://github.com/moby/moby/issues/37965 504 copy_cmds += "RUN true\n" 505 506 return copy_cmds 507 508 def _df_build(self) -> str: 509 """Formulate the Dockerfile snippet necessary to download, build, and 510 install a package into a Docker stage. 511 """ 512 513 # Download and extract source. 514 result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 515 516 # Handle 'custom_post_dl' commands. 517 custom_post_dl = self.pkg_def.get("custom_post_dl") 518 if custom_post_dl: 519 result += " && ".join(custom_post_dl) + " && " 520 521 # Build and install package based on 'build_type'. 522 build_type = self.pkg_def["build_type"] 523 if build_type == "autoconf": 524 result += self._cmd_build_autoconf() 525 elif build_type == "cmake": 526 result += self._cmd_build_cmake() 527 elif build_type == "custom": 528 result += self._cmd_build_custom() 529 elif build_type == "make": 530 result += self._cmd_build_make() 531 elif build_type == "meson": 532 result += self._cmd_build_meson() 533 else: 534 raise NotImplementedError( 535 f"Unhandled build type for {self.package}: {build_type}" 536 ) 537 538 # Handle 'custom_post_install' commands. 539 custom_post_install = self.pkg_def.get("custom_post_install") 540 if custom_post_install: 541 result += " && " + " && ".join(custom_post_install) 542 543 return result 544 545 def _cmd_build_autoconf(self) -> str: 546 options = " ".join(self.pkg_def.get("config_flags", [])) 547 env = " ".join(self.pkg_def.get("config_env", [])) 548 result = "./bootstrap.sh && " 549 result += f"{env} ./configure {configure_flags} {options} && " 550 result += f"make -j{proc_count} && make install" 551 return result 552 553 def _cmd_build_cmake(self) -> str: 554 options = " ".join(self.pkg_def.get("config_flags", [])) 555 env = " ".join(self.pkg_def.get("config_env", [])) 556 result = "mkdir builddir && cd builddir && " 557 result += f"{env} cmake {cmake_flags} {options} .. && " 558 result += "cmake --build . --target all && " 559 result += "cmake --build . --target install && " 560 result += "cd .." 561 return result 562 563 def _cmd_build_custom(self) -> str: 564 return " && ".join(self.pkg_def.get("build_steps", [])) 565 566 def _cmd_build_make(self) -> str: 567 return f"make -j{proc_count} && make install" 568 569 def _cmd_build_meson(self) -> str: 570 options = " ".join(self.pkg_def.get("config_flags", [])) 571 env = " ".join(self.pkg_def.get("config_env", [])) 572 result = f"{env} meson setup builddir {meson_flags} {options} && " 573 result += "ninja -C builddir && ninja -C builddir install" 574 return result 575 576 577class Docker: 578 """Class to assist with Docker interactions. All methods are static.""" 579 580 @staticmethod 581 def timestamp() -> str: 582 """Generate a timestamp for today using the ISO week.""" 583 today = date.today().isocalendar() 584 return f"{today[0]}-W{today[1]:02}" 585 586 @staticmethod 587 def tagname(pkgname: Optional[str], dockerfile: str) -> str: 588 """Generate a tag name for a package using a hash of the Dockerfile.""" 589 result = docker_image_name 590 if pkgname: 591 result += "-" + pkgname 592 593 result += ":" + Docker.timestamp() 594 result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 595 596 return result 597 598 @staticmethod 599 def build(pkg: str, tag: str, dockerfile: str) -> None: 600 """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 601 602 # If we're not forcing builds, check if it already exists and skip. 603 if not force_build: 604 if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 605 print( 606 f"Image {tag} already exists. Skipping.", file=sys.stderr 607 ) 608 return 609 610 # Build it. 611 # Capture the output of the 'docker build' command and send it to 612 # stderr (prefixed with the package name). This allows us to see 613 # progress but not pollute stdout. Later on we output the final 614 # docker tag to stdout and we want to keep that pristine. 615 # 616 # Other unusual flags: 617 # --no-cache: Bypass the Docker cache if 'force_build'. 618 # --force-rm: Clean up Docker processes if they fail. 619 docker.build( 620 proxy_args, 621 "--network=host", 622 "--force-rm", 623 "--no-cache=true" if force_build else "--no-cache=false", 624 "-t", 625 tag, 626 "-", 627 _in=dockerfile, 628 _out=( 629 lambda line: print( 630 pkg + ":", line, end="", file=sys.stderr, flush=True 631 ) 632 ), 633 _err_to_out=True, 634 ) 635 636 637# Read a bunch of environment variables. 638docker_image_name = os.environ.get( 639 "DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test" 640) 641force_build = os.environ.get("FORCE_DOCKER_BUILD") 642is_automated_ci_build = os.environ.get("BUILD_URL", False) 643distro = os.environ.get("DISTRO", "ubuntu:noble") 644branch = os.environ.get("BRANCH", "master") 645ubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 646http_proxy = os.environ.get("http_proxy") 647 648gerrit_project = os.environ.get("GERRIT_PROJECT") 649gerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 650 651# Ensure appropriate docker build output to see progress and identify 652# any issues 653os.environ["BUILDKIT_PROGRESS"] = "plain" 654 655# Set up some common variables. 656username = os.environ.get("USER", "root") 657homedir = os.environ.get("HOME", "/root") 658gid = os.getgid() 659uid = os.getuid() 660 661# Use well-known constants if user is root 662if username == "root": 663 homedir = "/root" 664 gid = 0 665 uid = 0 666 667# Determine the architecture for Docker. 668arch = uname("-m").strip() 669if arch == "ppc64le": 670 docker_base = "ppc64le/" 671elif arch == "x86_64": 672 docker_base = "" 673elif arch == "aarch64": 674 docker_base = "arm64v8/" 675else: 676 print( 677 f"Unsupported system architecture({arch}) found for docker image", 678 file=sys.stderr, 679 ) 680 sys.exit(1) 681 682# Special flags if setting up a deb mirror. 683mirror = "" 684if "ubuntu" in distro and ubuntu_mirror: 685 mirror = f""" 686RUN echo "deb {ubuntu_mirror} \ 687 $(. /etc/os-release && echo $VERSION_CODENAME) \ 688 main restricted universe multiverse" > /etc/apt/sources.list && \\ 689 echo "deb {ubuntu_mirror} \ 690 $(. /etc/os-release && echo $VERSION_CODENAME)-updates \ 691 main restricted universe multiverse" >> /etc/apt/sources.list && \\ 692 echo "deb {ubuntu_mirror} \ 693 $(. /etc/os-release && echo $VERSION_CODENAME)-security \ 694 main restricted universe multiverse" >> /etc/apt/sources.list && \\ 695 echo "deb {ubuntu_mirror} \ 696 $(. /etc/os-release && echo $VERSION_CODENAME)-proposed \ 697 main restricted universe multiverse" >> /etc/apt/sources.list && \\ 698 echo "deb {ubuntu_mirror} \ 699 $(. /etc/os-release && echo $VERSION_CODENAME)-backports \ 700 main restricted universe multiverse" >> /etc/apt/sources.list 701""" 702 703# Special flags for proxying. 704proxy_cmd = "" 705proxy_keyserver = "" 706proxy_args = [] 707if http_proxy: 708 proxy_cmd = f""" 709RUN echo "[http]" >> {homedir}/.gitconfig && \ 710 echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 711""" 712 proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 713 714 proxy_args.extend( 715 [ 716 "--build-arg", 717 f"http_proxy={http_proxy}", 718 "--build-arg", 719 f"https_proxy={http_proxy}", 720 ] 721 ) 722 723# Create base Dockerfile. 724dockerfile_base = f""" 725FROM {docker_base}{distro} 726 727{mirror} 728 729ENV DEBIAN_FRONTEND noninteractive 730 731ENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/" 732 733# Sometimes the ubuntu key expires and we need a way to force an execution 734# of the apt-get commands for the dbgsym-keyring. When this happens we see 735# an error like: "Release: The following signatures were invalid:" 736# Insert a bogus echo that we can change here when we get this error to force 737# the update. 738RUN echo "ubuntu keyserver rev as of 2021-04-21" 739 740# We need the keys to be imported for dbgsym repos 741# New releases have a package, older ones fall back to manual fetching 742# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 743# Known issue with gpg to get keys via proxy - 744# https://bugs.launchpad.net/ubuntu/+source/gnupg2/+bug/1788190, hence using 745# curl to get keys. 746RUN apt-get update && apt-get dist-upgrade -yy && \ 747 ( apt-get install -yy gpgv ubuntu-dbgsym-keyring || \ 748 ( apt-get install -yy dirmngr curl && \ 749 curl -sSL \ 750 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622' \ 751 | apt-key add - )) 752 753# Parse the current repo list into a debug repo list 754RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' \ 755 /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 756 757# Remove non-existent debug repos 758RUN sed -i '/-\\(backports\\|security\\) /d' /etc/apt/sources.list.d/debug.list 759 760RUN cat /etc/apt/sources.list.d/debug.list 761 762RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 763 abi-compliance-checker \ 764 abi-dumper \ 765 autoconf \ 766 autoconf-archive \ 767 bison \ 768 clang-17 \ 769 clang-format-17 \ 770 clang-tidy-17 \ 771 clang-tools-17 \ 772 cmake \ 773 curl \ 774 dbus \ 775 device-tree-compiler \ 776 flex \ 777 g++-13 \ 778 gcc-13 \ 779 git \ 780 iproute2 \ 781 iputils-ping \ 782 libaudit-dev \ 783 libc6-dbg \ 784 libc6-dev \ 785 libconfig++-dev \ 786 libcryptsetup-dev \ 787 libdbus-1-dev \ 788 libevdev-dev \ 789 libgpiod-dev \ 790 libi2c-dev \ 791 libjpeg-dev \ 792 libjson-perl \ 793 libldap2-dev \ 794 libmimetic-dev \ 795 libnl-3-dev \ 796 libnl-genl-3-dev \ 797 libpam0g-dev \ 798 libpciaccess-dev \ 799 libperlio-gzip-perl \ 800 libpng-dev \ 801 libprotobuf-dev \ 802 libsnmp-dev \ 803 libssl-dev \ 804 libsystemd-dev \ 805 libtool \ 806 liburing-dev \ 807 libxml2-utils \ 808 libxml-simple-perl \ 809 ninja-build \ 810 npm \ 811 pkg-config \ 812 protobuf-compiler \ 813 python3 \ 814 python3-dev\ 815 python3-git \ 816 python3-mako \ 817 python3-pip \ 818 python3-protobuf \ 819 python3-setuptools \ 820 python3-socks \ 821 python3-yaml \ 822 rsync \ 823 shellcheck \ 824 socat \ 825 sudo \ 826 systemd \ 827 valgrind \ 828 vim \ 829 wget \ 830 xxd 831 832RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 13 \ 833 --slave /usr/bin/g++ g++ /usr/bin/g++-13 \ 834 --slave /usr/bin/gcov gcov /usr/bin/gcov-13 \ 835 --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-13 \ 836 --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-13 837RUN update-alternatives --remove cpp /usr/bin/cpp && \ 838 update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-13 13 839 840RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 1000 \ 841 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-17 \ 842 --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-17 \ 843 --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-17 \ 844 --slave /usr/bin/run-clang-tidy run-clang-tidy.py \ 845 /usr/bin/run-clang-tidy-17 \ 846 --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-17 847 848""" 849 850if is_automated_ci_build: 851 dockerfile_base += f""" 852# Run an arbitrary command to pollute the docker cache regularly force us 853# to re-run `apt-get update` daily. 854RUN echo {Docker.timestamp()} 855RUN apt-get update && apt-get dist-upgrade -yy 856 857""" 858 859dockerfile_base += """ 860RUN pip3 install --break-system-packages \ 861 beautysh \ 862 black \ 863 codespell \ 864 flake8 \ 865 gcovr \ 866 gitlint \ 867 inflection \ 868 isort \ 869 jsonschema \ 870 meson==1.3.0 \ 871 requests 872 873RUN npm install -g \ 874 eslint@v8.56.0 eslint-plugin-json@v3.1.0 \ 875 markdownlint-cli@latest \ 876 prettier@latest 877""" 878 879# Build the base and stage docker images. 880docker_base_img_name = Docker.tagname("base", dockerfile_base) 881Docker.build("base", docker_base_img_name, dockerfile_base) 882Package.generate_all() 883 884# Create the final Dockerfile. 885dockerfile = f""" 886# Build the final output image 887FROM {docker_base_img_name} 888{Package.df_all_copycmds()} 889 890# Some of our infrastructure still relies on the presence of this file 891# even though it is no longer needed to rebuild the docker environment 892# NOTE: The file is sorted to ensure the ordering is stable. 893RUN echo '{Package.depcache()}' > /tmp/depcache 894 895# Ensure the group, user, and home directory are created (or rename them if 896# they already exist). 897RUN if grep -q ":{gid}:" /etc/group ; then \ 898 groupmod -n {username} $(awk -F : '{{ if ($3 == {gid}) {{ print $1 }} }}' /etc/group) ; \ 899 else \ 900 groupadd -f -g {gid} {username} ; \ 901 fi 902RUN mkdir -p "{os.path.dirname(homedir)}" 903RUN if grep -q ":{uid}:" /etc/passwd ; then \ 904 usermod -l {username} -d {homedir} -m $(awk -F : '{{ if ($3 == {uid}) {{ print $1 }} }}' /etc/passwd) ; \ 905 else \ 906 useradd -d {homedir} -m -u {uid} -g {gid} {username} ; \ 907 fi 908RUN sed -i '1iDefaults umask=000' /etc/sudoers 909RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 910 911# Ensure user has ability to write to /usr/local for different tool 912# and data installs 913RUN chown -R {username}:{username} /usr/local/share 914 915# Update library cache 916RUN ldconfig 917 918{proxy_cmd} 919 920RUN /bin/bash 921""" 922 923# Do the final docker build 924docker_final_img_name = Docker.tagname(None, dockerfile) 925Docker.build("final", docker_final_img_name, dockerfile) 926 927# Print the tag of the final image. 928print(docker_final_img_name) 929