1#!/usr/bin/env python3 2# 3# Build the required docker image to run package unit tests 4# 5# Script Variables: 6# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 7# default is openbmc/ubuntu-unit-test 8# DISTRO: <optional, the distro to build a docker image against> 9# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 10# images to be rebuilt rather than reusing caches.> 11# BUILD_URL: <optional, used to detect running under CI context 12# (ex. Jenkins)> 13# BRANCH: <optional, branch to build from each of the openbmc/ 14# repositories> 15# default is master, which will be used if input branch not 16# provided or not found 17# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 18# default ones in /etc/apt/sources.list> 19# default is empty, and no mirror is used. 20# http_proxy The HTTP address of the proxy server to connect to. 21# Default: "", proxy is not setup if this is not set 22 23import os 24import sys 25import threading 26from datetime import date 27from hashlib import sha256 28 29# typing.Dict is used for type-hints. 30from typing import Any, Callable, Dict, Iterable, Optional # noqa: F401 31 32from sh import docker, git, nproc, uname # type: ignore 33 34try: 35 # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 36 from typing import TypedDict 37except Exception: 38 39 class TypedDict(dict): # type: ignore 40 # We need to do this to eat the 'total' argument. 41 def __init_subclass__(cls, **kwargs: Any) -> None: 42 super().__init_subclass__() 43 44 45# Declare some variables used in package definitions. 46prefix = "/usr/local" 47proc_count = nproc().strip() 48 49 50class PackageDef(TypedDict, total=False): 51 """Package Definition for packages dictionary.""" 52 53 # rev [optional]: Revision of package to use. 54 rev: str 55 # url [optional]: lambda function to create URL: (package, rev) -> url. 56 url: Callable[[str, str], str] 57 # depends [optional]: List of package dependencies. 58 depends: Iterable[str] 59 # build_type [required]: Build type used for package. 60 # Currently supported: autoconf, cmake, custom, make, meson 61 build_type: str 62 # build_steps [optional]: Steps to run for 'custom' build_type. 63 build_steps: Iterable[str] 64 # config_flags [optional]: List of options to pass configuration tool. 65 config_flags: Iterable[str] 66 # config_env [optional]: List of environment variables to set for config. 67 config_env: Iterable[str] 68 # custom_post_dl [optional]: List of steps to run after download, but 69 # before config / build / install. 70 custom_post_dl: Iterable[str] 71 # custom_post_install [optional]: List of steps to run after install. 72 custom_post_install: Iterable[str] 73 74 # __tag [private]: Generated Docker tag name for package stage. 75 __tag: str 76 # __package [private]: Package object associated with this package. 77 __package: Any # Type is Package, but not defined yet. 78 79 80# Packages to include in image. 81packages = { 82 "boost": PackageDef( 83 rev="1.82.0", 84 url=( 85 lambda pkg, rev: f"https://boostorg.jfrog.io/artifactory/main/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.gz" # noqa: E501 86 ), 87 build_type="custom", 88 build_steps=[ 89 ( 90 "curl --remote-name" 91 " https://github.com/williamspatrick/beast/commit/98f8b1fbd059a35754c2c7b2841769cf8d021272.patch" 92 " && patch -p2 <" 93 " 98f8b1fbd059a35754c2c7b2841769cf8d021272.patch &&" 94 " ./bootstrap.sh" 95 f" --prefix={prefix} --with-libraries=context,coroutine" 96 ), 97 "./b2", 98 f"./b2 install --prefix={prefix}", 99 ], 100 ), 101 "USCiLab/cereal": PackageDef( 102 rev="v1.3.2", 103 build_type="custom", 104 build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 105 ), 106 "danmar/cppcheck": PackageDef( 107 rev="2.9", 108 build_type="cmake", 109 ), 110 "CLIUtils/CLI11": PackageDef( 111 rev="v2.3.2", 112 build_type="cmake", 113 config_flags=[ 114 "-DBUILD_TESTING=OFF", 115 "-DCLI11_BUILD_DOCS=OFF", 116 "-DCLI11_BUILD_EXAMPLES=OFF", 117 ], 118 ), 119 "fmtlib/fmt": PackageDef( 120 rev="10.0.0", 121 build_type="cmake", 122 config_flags=[ 123 "-DFMT_DOC=OFF", 124 "-DFMT_TEST=OFF", 125 ], 126 ), 127 "Naios/function2": PackageDef( 128 rev="4.2.1", 129 build_type="custom", 130 build_steps=[ 131 f"mkdir {prefix}/include/function2", 132 f"cp include/function2/function2.hpp {prefix}/include/function2/", 133 ], 134 ), 135 # release-1.13.0 136 "google/googletest": PackageDef( 137 rev="b796f7d44681514f58a683a3a71ff17c94edb0c1", 138 build_type="cmake", 139 config_env=["CXXFLAGS=-std=c++20"], 140 config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 141 ), 142 "nghttp2/nghttp2": PackageDef( 143 rev="v1.54.0", 144 build_type="cmake", 145 config_env=["CXXFLAGS=-std=c++20"], 146 config_flags=[ 147 "-DENABLE_LIB_ONLY=ON", 148 "-DENABLE_STATIC_LIB=ON", 149 ], 150 ), 151 "nlohmann/json": PackageDef( 152 rev="v3.11.2", 153 build_type="cmake", 154 config_flags=["-DJSON_BuildTests=OFF"], 155 custom_post_install=[ 156 ( 157 f"ln -s {prefix}/include/nlohmann/json.hpp" 158 f" {prefix}/include/json.hpp" 159 ), 160 ], 161 ), 162 "json-c/json-c": PackageDef( 163 rev="json-c-0.16-20220414", 164 build_type="cmake", 165 ), 166 # Snapshot from 2019-05-24 167 "linux-test-project/lcov": PackageDef( 168 rev="v1.15", 169 build_type="make", 170 ), 171 "LibVNC/libvncserver": PackageDef( 172 rev="LibVNCServer-0.9.13", 173 build_type="cmake", 174 ), 175 "leethomason/tinyxml2": PackageDef( 176 rev="9.0.0", 177 build_type="cmake", 178 ), 179 "tristanpenman/valijson": PackageDef( 180 rev="v0.7", 181 build_type="cmake", 182 config_flags=[ 183 "-Dvalijson_BUILD_TESTS=0", 184 "-Dvalijson_INSTALL_HEADERS=1", 185 ], 186 ), 187 "open-power/pdbg": PackageDef(build_type="autoconf"), 188 "openbmc/gpioplus": PackageDef( 189 depends=["openbmc/stdplus"], 190 build_type="meson", 191 config_flags=[ 192 "-Dexamples=false", 193 "-Dtests=disabled", 194 ], 195 ), 196 "openbmc/phosphor-dbus-interfaces": PackageDef( 197 depends=["openbmc/sdbusplus"], 198 build_type="meson", 199 config_flags=["-Dgenerate_md=false"], 200 ), 201 "openbmc/phosphor-logging": PackageDef( 202 depends=[ 203 "USCiLab/cereal", 204 "openbmc/phosphor-dbus-interfaces", 205 "openbmc/sdbusplus", 206 "openbmc/sdeventplus", 207 ], 208 build_type="meson", 209 config_flags=[ 210 "-Dlibonly=true", 211 "-Dtests=disabled", 212 f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml", 213 ], 214 ), 215 "openbmc/phosphor-objmgr": PackageDef( 216 depends=[ 217 "CLIUtils/CLI11", 218 "boost", 219 "leethomason/tinyxml2", 220 "openbmc/phosphor-dbus-interfaces", 221 "openbmc/phosphor-logging", 222 "openbmc/sdbusplus", 223 ], 224 build_type="meson", 225 config_flags=[ 226 "-Dtests=disabled", 227 ], 228 ), 229 "openbmc/libpeci": PackageDef( 230 build_type="meson", 231 config_flags=[ 232 "-Draw-peci=disabled", 233 ], 234 ), 235 "openbmc/libpldm": PackageDef( 236 build_type="meson", 237 config_flags=[ 238 "-Dabi=deprecated,stable", 239 "-Doem-ibm=enabled", 240 "-Dtests=disabled", 241 ], 242 ), 243 "openbmc/sdbusplus": PackageDef( 244 build_type="meson", 245 custom_post_dl=[ 246 "cd tools", 247 f"./setup.py install --root=/ --prefix={prefix}", 248 "cd ..", 249 ], 250 config_flags=[ 251 "-Dexamples=disabled", 252 "-Dtests=disabled", 253 ], 254 ), 255 "openbmc/sdeventplus": PackageDef( 256 depends=[ 257 "Naios/function2", 258 "openbmc/stdplus", 259 ], 260 build_type="meson", 261 config_flags=[ 262 "-Dexamples=false", 263 "-Dtests=disabled", 264 ], 265 ), 266 "openbmc/stdplus": PackageDef( 267 depends=[ 268 "fmtlib/fmt", 269 "google/googletest", 270 "Naios/function2", 271 ], 272 build_type="meson", 273 config_flags=[ 274 "-Dexamples=false", 275 "-Dtests=disabled", 276 "-Dgtest=enabled", 277 ], 278 ), 279} # type: Dict[str, PackageDef] 280 281# Define common flags used for builds 282configure_flags = " ".join( 283 [ 284 f"--prefix={prefix}", 285 ] 286) 287cmake_flags = " ".join( 288 [ 289 "-DBUILD_SHARED_LIBS=ON", 290 "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 291 f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 292 "-GNinja", 293 "-DCMAKE_MAKE_PROGRAM=ninja", 294 ] 295) 296meson_flags = " ".join( 297 [ 298 "--wrap-mode=nodownload", 299 f"-Dprefix={prefix}", 300 ] 301) 302 303 304class Package(threading.Thread): 305 """Class used to build the Docker stages for each package. 306 307 Generally, this class should not be instantiated directly but through 308 Package.generate_all(). 309 """ 310 311 # Copy the packages dictionary. 312 packages = packages.copy() 313 314 # Lock used for thread-safety. 315 lock = threading.Lock() 316 317 def __init__(self, pkg: str): 318 """pkg - The name of this package (ex. foo/bar )""" 319 super(Package, self).__init__() 320 321 self.package = pkg 322 self.exception = None # type: Optional[Exception] 323 324 # Reference to this package's 325 self.pkg_def = Package.packages[pkg] 326 self.pkg_def["__package"] = self 327 328 def run(self) -> None: 329 """Thread 'run' function. Builds the Docker stage.""" 330 331 # In case this package has no rev, fetch it from Github. 332 self._update_rev() 333 334 # Find all the Package objects that this package depends on. 335 # This section is locked because we are looking into another 336 # package's PackageDef dict, which could be being modified. 337 Package.lock.acquire() 338 deps: Iterable[Package] = [ 339 Package.packages[deppkg]["__package"] 340 for deppkg in self.pkg_def.get("depends", []) 341 ] 342 Package.lock.release() 343 344 # Wait until all the depends finish building. We need them complete 345 # for the "COPY" commands. 346 for deppkg in deps: 347 deppkg.join() 348 349 # Generate this package's Dockerfile. 350 dockerfile = f""" 351FROM {docker_base_img_name} 352{self._df_copycmds()} 353{self._df_build()} 354""" 355 356 # Generate the resulting tag name and save it to the PackageDef. 357 # This section is locked because we are modifying the PackageDef, 358 # which can be accessed by other threads. 359 Package.lock.acquire() 360 tag = Docker.tagname(self._stagename(), dockerfile) 361 self.pkg_def["__tag"] = tag 362 Package.lock.release() 363 364 # Do the build / save any exceptions. 365 try: 366 Docker.build(self.package, tag, dockerfile) 367 except Exception as e: 368 self.exception = e 369 370 @classmethod 371 def generate_all(cls) -> None: 372 """Ensure a Docker stage is created for all defined packages. 373 374 These are done in parallel but with appropriate blocking per 375 package 'depends' specifications. 376 """ 377 378 # Create a Package for each defined package. 379 pkg_threads = [Package(p) for p in cls.packages.keys()] 380 381 # Start building them all. 382 # This section is locked because threads depend on each other, 383 # based on the packages, and they cannot 'join' on a thread 384 # which is not yet started. Adding a lock here allows all the 385 # threads to start before they 'join' their dependencies. 386 Package.lock.acquire() 387 for t in pkg_threads: 388 t.start() 389 Package.lock.release() 390 391 # Wait for completion. 392 for t in pkg_threads: 393 t.join() 394 # Check if the thread saved off its own exception. 395 if t.exception: 396 print(f"Package {t.package} failed!", file=sys.stderr) 397 raise t.exception 398 399 @staticmethod 400 def df_all_copycmds() -> str: 401 """Formulate the Dockerfile snippet necessary to copy all packages 402 into the final image. 403 """ 404 return Package.df_copycmds_set(Package.packages.keys()) 405 406 @classmethod 407 def depcache(cls) -> str: 408 """Create the contents of the '/tmp/depcache'. 409 This file is a comma-separated list of "<pkg>:<rev>". 410 """ 411 412 # This needs to be sorted for consistency. 413 depcache = "" 414 for pkg in sorted(cls.packages.keys()): 415 depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 416 return depcache 417 418 def _update_rev(self) -> None: 419 """Look up the HEAD for missing a static rev.""" 420 421 if "rev" in self.pkg_def: 422 return 423 424 # Check if Jenkins/Gerrit gave us a revision and use it. 425 if gerrit_project == self.package and gerrit_rev: 426 print( 427 f"Found Gerrit revision for {self.package}: {gerrit_rev}", 428 file=sys.stderr, 429 ) 430 self.pkg_def["rev"] = gerrit_rev 431 return 432 433 # Ask Github for all the branches. 434 lookup = git( 435 "ls-remote", "--heads", f"https://github.com/{self.package}" 436 ) 437 438 # Find the branch matching {branch} (or fallback to master). 439 # This section is locked because we are modifying the PackageDef. 440 Package.lock.acquire() 441 for line in lookup.split("\n"): 442 if f"refs/heads/{branch}" in line: 443 self.pkg_def["rev"] = line.split()[0] 444 elif ( 445 "refs/heads/master" in line or "refs/heads/main" in line 446 ) and "rev" not in self.pkg_def: 447 self.pkg_def["rev"] = line.split()[0] 448 Package.lock.release() 449 450 def _stagename(self) -> str: 451 """Create a name for the Docker stage associated with this pkg.""" 452 return self.package.replace("/", "-").lower() 453 454 def _url(self) -> str: 455 """Get the URL for this package.""" 456 rev = self.pkg_def["rev"] 457 458 # If the lambda exists, call it. 459 if "url" in self.pkg_def: 460 return self.pkg_def["url"](self.package, rev) 461 462 # Default to the github archive URL. 463 return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 464 465 def _cmd_download(self) -> str: 466 """Formulate the command necessary to download and unpack to source.""" 467 468 url = self._url() 469 if ".tar." not in url: 470 raise NotImplementedError( 471 f"Unhandled download type for {self.package}: {url}" 472 ) 473 474 cmd = f"curl -L {url} | tar -x" 475 476 if url.endswith(".bz2"): 477 cmd += "j" 478 elif url.endswith(".gz"): 479 cmd += "z" 480 else: 481 raise NotImplementedError( 482 f"Unknown tar flags needed for {self.package}: {url}" 483 ) 484 485 return cmd 486 487 def _cmd_cd_srcdir(self) -> str: 488 """Formulate the command necessary to 'cd' into the source dir.""" 489 return f"cd {self.package.split('/')[-1]}*" 490 491 def _df_copycmds(self) -> str: 492 """Formulate the dockerfile snippet necessary to COPY all depends.""" 493 494 if "depends" not in self.pkg_def: 495 return "" 496 return Package.df_copycmds_set(self.pkg_def["depends"]) 497 498 @staticmethod 499 def df_copycmds_set(pkgs: Iterable[str]) -> str: 500 """Formulate the Dockerfile snippet necessary to COPY a set of 501 packages into a Docker stage. 502 """ 503 504 copy_cmds = "" 505 506 # Sort the packages for consistency. 507 for p in sorted(pkgs): 508 tag = Package.packages[p]["__tag"] 509 copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 510 # Workaround for upstream docker bug and multiple COPY cmds 511 # https://github.com/moby/moby/issues/37965 512 copy_cmds += "RUN true\n" 513 514 return copy_cmds 515 516 def _df_build(self) -> str: 517 """Formulate the Dockerfile snippet necessary to download, build, and 518 install a package into a Docker stage. 519 """ 520 521 # Download and extract source. 522 result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 523 524 # Handle 'custom_post_dl' commands. 525 custom_post_dl = self.pkg_def.get("custom_post_dl") 526 if custom_post_dl: 527 result += " && ".join(custom_post_dl) + " && " 528 529 # Build and install package based on 'build_type'. 530 build_type = self.pkg_def["build_type"] 531 if build_type == "autoconf": 532 result += self._cmd_build_autoconf() 533 elif build_type == "cmake": 534 result += self._cmd_build_cmake() 535 elif build_type == "custom": 536 result += self._cmd_build_custom() 537 elif build_type == "make": 538 result += self._cmd_build_make() 539 elif build_type == "meson": 540 result += self._cmd_build_meson() 541 else: 542 raise NotImplementedError( 543 f"Unhandled build type for {self.package}: {build_type}" 544 ) 545 546 # Handle 'custom_post_install' commands. 547 custom_post_install = self.pkg_def.get("custom_post_install") 548 if custom_post_install: 549 result += " && " + " && ".join(custom_post_install) 550 551 return result 552 553 def _cmd_build_autoconf(self) -> str: 554 options = " ".join(self.pkg_def.get("config_flags", [])) 555 env = " ".join(self.pkg_def.get("config_env", [])) 556 result = "./bootstrap.sh && " 557 result += f"{env} ./configure {configure_flags} {options} && " 558 result += f"make -j{proc_count} && make install" 559 return result 560 561 def _cmd_build_cmake(self) -> str: 562 options = " ".join(self.pkg_def.get("config_flags", [])) 563 env = " ".join(self.pkg_def.get("config_env", [])) 564 result = "mkdir builddir && cd builddir && " 565 result += f"{env} cmake {cmake_flags} {options} .. && " 566 result += "cmake --build . --target all && " 567 result += "cmake --build . --target install && " 568 result += "cd .." 569 return result 570 571 def _cmd_build_custom(self) -> str: 572 return " && ".join(self.pkg_def.get("build_steps", [])) 573 574 def _cmd_build_make(self) -> str: 575 return f"make -j{proc_count} && make install" 576 577 def _cmd_build_meson(self) -> str: 578 options = " ".join(self.pkg_def.get("config_flags", [])) 579 env = " ".join(self.pkg_def.get("config_env", [])) 580 result = f"{env} meson setup builddir {meson_flags} {options} && " 581 result += "ninja -C builddir && ninja -C builddir install" 582 return result 583 584 585class Docker: 586 """Class to assist with Docker interactions. All methods are static.""" 587 588 @staticmethod 589 def timestamp() -> str: 590 """Generate a timestamp for today using the ISO week.""" 591 today = date.today().isocalendar() 592 return f"{today[0]}-W{today[1]:02}" 593 594 @staticmethod 595 def tagname(pkgname: Optional[str], dockerfile: str) -> str: 596 """Generate a tag name for a package using a hash of the Dockerfile.""" 597 result = docker_image_name 598 if pkgname: 599 result += "-" + pkgname 600 601 result += ":" + Docker.timestamp() 602 result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 603 604 return result 605 606 @staticmethod 607 def build(pkg: str, tag: str, dockerfile: str) -> None: 608 """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 609 610 # If we're not forcing builds, check if it already exists and skip. 611 if not force_build: 612 if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 613 print( 614 f"Image {tag} already exists. Skipping.", file=sys.stderr 615 ) 616 return 617 618 # Build it. 619 # Capture the output of the 'docker build' command and send it to 620 # stderr (prefixed with the package name). This allows us to see 621 # progress but not polute stdout. Later on we output the final 622 # docker tag to stdout and we want to keep that pristine. 623 # 624 # Other unusual flags: 625 # --no-cache: Bypass the Docker cache if 'force_build'. 626 # --force-rm: Clean up Docker processes if they fail. 627 docker.build( 628 proxy_args, 629 "--network=host", 630 "--force-rm", 631 "--no-cache=true" if force_build else "--no-cache=false", 632 "-t", 633 tag, 634 "-", 635 _in=dockerfile, 636 _out=( 637 lambda line: print( 638 pkg + ":", line, end="", file=sys.stderr, flush=True 639 ) 640 ), 641 ) 642 643 644# Read a bunch of environment variables. 645docker_image_name = os.environ.get( 646 "DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test" 647) 648force_build = os.environ.get("FORCE_DOCKER_BUILD") 649is_automated_ci_build = os.environ.get("BUILD_URL", False) 650distro = os.environ.get("DISTRO", "ubuntu:mantic") 651branch = os.environ.get("BRANCH", "master") 652ubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 653http_proxy = os.environ.get("http_proxy") 654 655gerrit_project = os.environ.get("GERRIT_PROJECT") 656gerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 657 658# Ensure appropriate docker build output to see progress and identify 659# any issues 660os.environ["BUILDKIT_PROGRESS"] = "plain" 661 662# Set up some common variables. 663username = os.environ.get("USER", "root") 664homedir = os.environ.get("HOME", "/root") 665gid = os.getgid() 666uid = os.getuid() 667 668# Use well-known constants if user is root 669if username == "root": 670 homedir = "/root" 671 gid = 0 672 uid = 0 673 674# Determine the architecture for Docker. 675arch = uname("-m").strip() 676if arch == "ppc64le": 677 docker_base = "ppc64le/" 678elif arch == "x86_64": 679 docker_base = "" 680elif arch == "aarch64": 681 docker_base = "arm64v8/" 682else: 683 print( 684 f"Unsupported system architecture({arch}) found for docker image", 685 file=sys.stderr, 686 ) 687 sys.exit(1) 688 689# Special flags if setting up a deb mirror. 690mirror = "" 691if "ubuntu" in distro and ubuntu_mirror: 692 mirror = f""" 693RUN echo "deb {ubuntu_mirror} \ 694 $(. /etc/os-release && echo $VERSION_CODENAME) \ 695 main restricted universe multiverse" > /etc/apt/sources.list && \\ 696 echo "deb {ubuntu_mirror} \ 697 $(. /etc/os-release && echo $VERSION_CODENAME)-updates \ 698 main restricted universe multiverse" >> /etc/apt/sources.list && \\ 699 echo "deb {ubuntu_mirror} \ 700 $(. /etc/os-release && echo $VERSION_CODENAME)-security \ 701 main restricted universe multiverse" >> /etc/apt/sources.list && \\ 702 echo "deb {ubuntu_mirror} \ 703 $(. /etc/os-release && echo $VERSION_CODENAME)-proposed \ 704 main restricted universe multiverse" >> /etc/apt/sources.list && \\ 705 echo "deb {ubuntu_mirror} \ 706 $(. /etc/os-release && echo $VERSION_CODENAME)-backports \ 707 main restricted universe multiverse" >> /etc/apt/sources.list 708""" 709 710# Special flags for proxying. 711proxy_cmd = "" 712proxy_keyserver = "" 713proxy_args = [] 714if http_proxy: 715 proxy_cmd = f""" 716RUN echo "[http]" >> {homedir}/.gitconfig && \ 717 echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 718""" 719 proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 720 721 proxy_args.extend( 722 [ 723 "--build-arg", 724 f"http_proxy={http_proxy}", 725 "--build-arg", 726 f"https_proxy={http_proxy}", 727 ] 728 ) 729 730# Create base Dockerfile. 731dockerfile_base = f""" 732FROM {docker_base}{distro} 733 734{mirror} 735 736ENV DEBIAN_FRONTEND noninteractive 737 738ENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/" 739 740# Sometimes the ubuntu key expires and we need a way to force an execution 741# of the apt-get commands for the dbgsym-keyring. When this happens we see 742# an error like: "Release: The following signatures were invalid:" 743# Insert a bogus echo that we can change here when we get this error to force 744# the update. 745RUN echo "ubuntu keyserver rev as of 2021-04-21" 746 747# We need the keys to be imported for dbgsym repos 748# New releases have a package, older ones fall back to manual fetching 749# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 750# Known issue with gpg to get keys via proxy - 751# https://bugs.launchpad.net/ubuntu/+source/gnupg2/+bug/1788190, hence using 752# curl to get keys. 753RUN apt-get update && apt-get dist-upgrade -yy && \ 754 ( apt-get install -yy gpgv ubuntu-dbgsym-keyring || \ 755 ( apt-get install -yy dirmngr curl && \ 756 curl -sSL \ 757 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622' \ 758 | apt-key add - )) 759 760# Parse the current repo list into a debug repo list 761RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' \ 762 /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 763 764# Remove non-existent debug repos 765RUN sed -i '/-\\(backports\\|security\\) /d' /etc/apt/sources.list.d/debug.list 766 767RUN cat /etc/apt/sources.list.d/debug.list 768 769RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 770 abi-compliance-checker \ 771 abi-dumper \ 772 autoconf \ 773 autoconf-archive \ 774 bison \ 775 clang-16 \ 776 clang-format-16 \ 777 clang-tidy-16 \ 778 clang-tools-16 \ 779 cmake \ 780 curl \ 781 dbus \ 782 device-tree-compiler \ 783 flex \ 784 g++-13 \ 785 gcc-13 \ 786 git \ 787 iproute2 \ 788 iputils-ping \ 789 libaudit-dev \ 790 libc6-dbg \ 791 libc6-dev \ 792 libconfig++-dev \ 793 libcryptsetup-dev \ 794 libdbus-1-dev \ 795 libevdev-dev \ 796 libgpiod-dev \ 797 libi2c-dev \ 798 libjpeg-dev \ 799 libjson-perl \ 800 libldap2-dev \ 801 libmimetic-dev \ 802 libnl-3-dev \ 803 libnl-genl-3-dev \ 804 libpam0g-dev \ 805 libpciaccess-dev \ 806 libperlio-gzip-perl \ 807 libpng-dev \ 808 libprotobuf-dev \ 809 libsnmp-dev \ 810 libssl-dev \ 811 libsystemd-dev \ 812 libtool \ 813 liburing-dev \ 814 libxml2-utils \ 815 libxml-simple-perl \ 816 ninja-build \ 817 npm \ 818 pkg-config \ 819 protobuf-compiler \ 820 python3 \ 821 python3-dev\ 822 python3-git \ 823 python3-mako \ 824 python3-pip \ 825 python3-setuptools \ 826 python3-socks \ 827 python3-yaml \ 828 rsync \ 829 shellcheck \ 830 sudo \ 831 systemd \ 832 valgrind \ 833 valgrind-dbg \ 834 vim \ 835 wget \ 836 xxd 837 838RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 13 \ 839 --slave /usr/bin/g++ g++ /usr/bin/g++-13 \ 840 --slave /usr/bin/gcov gcov /usr/bin/gcov-13 \ 841 --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-13 \ 842 --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-13 843RUN update-alternatives --remove cpp /usr/bin/cpp && \ 844 update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-13 13 845 846RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-16 1000 \ 847 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-16 \ 848 --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-16 \ 849 --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-16 \ 850 --slave /usr/bin/run-clang-tidy run-clang-tidy.py \ 851 /usr/bin/run-clang-tidy-16 \ 852 --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-16 853 854""" 855 856if is_automated_ci_build: 857 dockerfile_base += f""" 858# Run an arbitrary command to polute the docker cache regularly force us 859# to re-run `apt-get update` daily. 860RUN echo {Docker.timestamp()} 861RUN apt-get update && apt-get dist-upgrade -yy 862 863""" 864 865dockerfile_base += """ 866RUN pip3 install --break-system-packages \ 867 beautysh \ 868 black \ 869 codespell \ 870 flake8 \ 871 gitlint \ 872 inflection \ 873 isort \ 874 jsonschema \ 875 protobuf \ 876 requests 877 878## Patch meson to fix Clang and C++23 on meson 1.1.1 879RUN pip3 install --break-system-packages \ 880 https://github.com/williamspatrick/meson/archive/meson-1.1.1-plus-clang-cpp23.tar.gz 881 882RUN npm install -g \ 883 eslint@latest eslint-plugin-json@latest \ 884 markdownlint-cli@latest \ 885 prettier@latest 886""" 887 888# Build the base and stage docker images. 889docker_base_img_name = Docker.tagname("base", dockerfile_base) 890Docker.build("base", docker_base_img_name, dockerfile_base) 891Package.generate_all() 892 893# Create the final Dockerfile. 894dockerfile = f""" 895# Build the final output image 896FROM {docker_base_img_name} 897{Package.df_all_copycmds()} 898 899# Some of our infrastructure still relies on the presence of this file 900# even though it is no longer needed to rebuild the docker environment 901# NOTE: The file is sorted to ensure the ordering is stable. 902RUN echo '{Package.depcache()}' > /tmp/depcache 903 904# Ensure the group, user, and home directory are created (or rename them if 905# they already exist). 906RUN if grep -q ":{gid}:" /etc/group ; then \ 907 groupmod -n {username} $(awk -F : '{{ if ($3 == {gid}) {{ print $1 }} }}' /etc/group) ; \ 908 else \ 909 groupadd -f -g {gid} {username} ; \ 910 fi 911RUN mkdir -p "{os.path.dirname(homedir)}" 912RUN if grep -q ":{uid}:" /etc/passwd ; then \ 913 usermod -l {username} -d {homedir} -m $(awk -F : '{{ if ($3 == {uid}) {{ print $1 }} }}' /etc/passwd) ; \ 914 else \ 915 useradd -d {homedir} -m -u {uid} -g {gid} {username} ; \ 916 fi 917RUN sed -i '1iDefaults umask=000' /etc/sudoers 918RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 919 920# Ensure user has ability to write to /usr/local for different tool 921# and data installs 922RUN chown -R {username}:{username} /usr/local/share 923 924{proxy_cmd} 925 926RUN /bin/bash 927""" 928 929# Do the final docker build 930docker_final_img_name = Docker.tagname(None, dockerfile) 931Docker.build("final", docker_final_img_name, dockerfile) 932 933# Print the tag of the final image. 934print(docker_final_img_name) 935