102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1050837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1150837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1250837436SPatrick Williams# (ex. Jenkins)> 1302871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1402871c91SPatrick Williams# repositories> 1502871c91SPatrick Williams# default is master, which will be used if input branch not 1602871c91SPatrick Williams# provided or not found 1702871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1802871c91SPatrick Williams# default ones in /etc/apt/sources.list> 1902871c91SPatrick Williams# default is empty, and no mirror is used. 2002871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2102871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2202871c91SPatrick Williams 2302871c91SPatrick Williamsimport os 2402871c91SPatrick Williamsimport sys 25b16f3e20SPatrick Williamsimport threading 26a18d9c57SPatrick Williamsfrom datetime import date 27a18d9c57SPatrick Williamsfrom hashlib import sha256 28ee3c9eebSPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 29ee3c9eebSPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional 3002871c91SPatrick Williams 31ee3c9eebSPatrick Williamstry: 32ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 33ee3c9eebSPatrick Williams from typing import TypedDict 34ee3c9eebSPatrick Williamsexcept: 35ee3c9eebSPatrick Williams 36ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 37ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 38ee3c9eebSPatrick Williams def __init_subclass__(cls, **kwargs): 39ee3c9eebSPatrick Williams super().__init_subclass__() 40ee3c9eebSPatrick Williams 41ee3c9eebSPatrick Williams 42ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 43aae36d18SPatrick Williamsprefix = "/usr/local" 4402871c91SPatrick Williamsproc_count = nproc().strip() 4502871c91SPatrick Williams 46ee3c9eebSPatrick Williams 47ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 48ee3c9eebSPatrick Williams """ Package Definition for packages dictionary. """ 49ee3c9eebSPatrick Williams 50ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 51ee3c9eebSPatrick Williams rev: str 52ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 53ee3c9eebSPatrick Williams url: Callable[[str, str], str] 54ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 55ee3c9eebSPatrick Williams depends: Iterable[str] 56ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 57ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 58ee3c9eebSPatrick Williams build_type: str 59ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 60ee3c9eebSPatrick Williams build_steps: Iterable[str] 61ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 62ee3c9eebSPatrick Williams config_flags: Iterable[str] 63ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 64ee3c9eebSPatrick Williams config_env: Iterable[str] 65ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 66ee3c9eebSPatrick Williams # before config / build / install. 67ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 686bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 696bce2ca1SPatrick Williams custom_post_install: Iterable[str] 70ee3c9eebSPatrick Williams 71ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 72ee3c9eebSPatrick Williams __tag: str 73ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 74ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 75ee3c9eebSPatrick Williams 7602871c91SPatrick Williams 777204324cSPatrick Williams# Packages to include in image. 787204324cSPatrick Williamspackages = { 799a85a04eSPatrick Williams # Install OpenSSL 3.x. 809a85a04eSPatrick Williams # Generally we want to rely on the version of OpenSSL from the OS, but v3.x 819a85a04eSPatrick Williams # was a major API change. It is included in Yocto but not Ubuntu until 829a85a04eSPatrick Williams # 22.04. Install it manually so that our CI can target the OpenSSL 3.x 839a85a04eSPatrick Williams # APIs. 849a85a04eSPatrick Williams "openssl/openssl": PackageDef( 859a85a04eSPatrick Williams rev="openssl-3.0.1", 869a85a04eSPatrick Williams build_type="custom", 879a85a04eSPatrick Williams build_steps=[ 889a85a04eSPatrick Williams f"./Configure --prefix={prefix} --libdir=lib", 899a85a04eSPatrick Williams f"make -j{proc_count}", 909a85a04eSPatrick Williams f"make -j{proc_count} install" 919a85a04eSPatrick Williams ], 929a85a04eSPatrick Williams ), 93ee3c9eebSPatrick Williams "boost": PackageDef( 94ef7bca47SEd Tanous rev="1.78.0", 95ee3c9eebSPatrick Williams url=( 965f2549eaSPatrick Williams lambda pkg, rev: f"https://downloads.yoctoproject.org/mirror/sources/{pkg}_{rev.replace('.', '_')}.tar.bz2" 972abc4a48SPatrick Williams ), 98ee3c9eebSPatrick Williams build_type="custom", 99ee3c9eebSPatrick Williams build_steps=[ 100aae36d18SPatrick Williams f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 101aae36d18SPatrick Williams "./b2", 102aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 103aae36d18SPatrick Williams ], 104ee3c9eebSPatrick Williams ), 105ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 1063f8b5294SPatrick Williams rev="3e4d1b84cab4891368d2179a61a7ba06a5693e7f", 107ee3c9eebSPatrick Williams build_type="custom", 108ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 109ee3c9eebSPatrick Williams ), 110ee3c9eebSPatrick Williams "catchorg/Catch2": PackageDef( 11196b43455SWilliam A. Kennington III rev="v2.13.6", 112ee3c9eebSPatrick Williams build_type="cmake", 113ee3c9eebSPatrick Williams config_flags=["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"], 114ee3c9eebSPatrick Williams ), 115ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 116ee3c9eebSPatrick Williams rev="v1.9.1", 117ee3c9eebSPatrick Williams build_type="cmake", 118ee3c9eebSPatrick Williams config_flags=[ 119aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 120aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 121aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 122aae36d18SPatrick Williams ], 123ee3c9eebSPatrick Williams ), 124ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 12563075013SWilly Tu rev="8.1.1", 126ee3c9eebSPatrick Williams build_type="cmake", 127ee3c9eebSPatrick Williams config_flags=[ 128aae36d18SPatrick Williams "-DFMT_DOC=OFF", 129aae36d18SPatrick Williams "-DFMT_TEST=OFF", 130aae36d18SPatrick Williams ], 131ee3c9eebSPatrick Williams ), 132ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 13396b43455SWilliam A. Kennington III rev="4.1.0", 134ee3c9eebSPatrick Williams build_type="custom", 135ee3c9eebSPatrick Williams build_steps=[ 136aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 137aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 138aae36d18SPatrick Williams ], 139ee3c9eebSPatrick Williams ), 1405202d8eeSMichael Shen # Release 2021-06-12 141ee3c9eebSPatrick Williams "google/googletest": PackageDef( 142*acfdee5dSWilly Tu rev="9e712372214d75bb30ec2847a44bf124d48096f3", 143ee3c9eebSPatrick Williams build_type="cmake", 1444dd32c02SWilliam A. Kennington III config_env=["CXXFLAGS=-std=c++20"], 145ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 146ee3c9eebSPatrick Williams ), 14702871c91SPatrick Williams # Release 2020-08-06 148ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 1497d1b2a1dSEd Tanous rev="v3.10.4", 1506bce2ca1SPatrick Williams build_type="cmake", 1516bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1526bce2ca1SPatrick Williams custom_post_install=[ 153aae36d18SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 154aae36d18SPatrick Williams ], 155ee3c9eebSPatrick Williams ), 15602871c91SPatrick Williams # Snapshot from 2019-05-24 157ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 158ee3c9eebSPatrick Williams rev="v1.15", 159ee3c9eebSPatrick Williams build_type="make", 160ee3c9eebSPatrick Williams ), 1610eedeedaSPatrick Williams # dev-5.8 2021-01-11 162ee3c9eebSPatrick Williams "openbmc/linux": PackageDef( 163ee3c9eebSPatrick Williams rev="3cc95ae40716e56f81b69615781f54c78079042d", 164ee3c9eebSPatrick Williams build_type="custom", 165ee3c9eebSPatrick Williams build_steps=[ 166aae36d18SPatrick Williams f"make -j{proc_count} defconfig", 167aae36d18SPatrick Williams f"make INSTALL_HDR_PATH={prefix} headers_install", 168aae36d18SPatrick Williams ], 169ee3c9eebSPatrick Williams ), 1700eedeedaSPatrick Williams # Snapshot from 2020-06-13 171ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 172ee3c9eebSPatrick Williams rev="LibVNCServer-0.9.13", 173ee3c9eebSPatrick Williams build_type="cmake", 174ee3c9eebSPatrick Williams ), 175ee3c9eebSPatrick Williams "martinmoene/span-lite": PackageDef( 17696b43455SWilliam A. Kennington III rev="v0.9.2", 177ee3c9eebSPatrick Williams build_type="cmake", 178ee3c9eebSPatrick Williams config_flags=[ 179aae36d18SPatrick Williams "-DSPAN_LITE_OPT_BUILD_TESTS=OFF", 180aae36d18SPatrick Williams ], 181ee3c9eebSPatrick Williams ), 1820eedeedaSPatrick Williams # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_8.0.0.bb 183ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 184ee3c9eebSPatrick Williams rev="8.0.0", 185ee3c9eebSPatrick Williams build_type="cmake", 186ee3c9eebSPatrick Williams ), 18702871c91SPatrick Williams # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 188ee3c9eebSPatrick Williams "CPPAlliance/url": PackageDef( 189ab640cdaSEd Tanous rev="d740a92d38e3a8f4d5b2153f53b82f1c98e312ab", 190eed466e3SEd Tanous build_type="custom", 191eed466e3SEd Tanous build_steps=[f"cp -a include/** {prefix}/include/"], 192ee3c9eebSPatrick Williams ), 19368992895SPatrick Williams # version from meta-openembedded/meta-oe/dynamic-layers/networking-layer/recipes-devools/valijson/valijson_0.6.bb 194ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 19568992895SPatrick Williams rev="v0.6", 196ee3c9eebSPatrick Williams build_type="cmake", 197ee3c9eebSPatrick Williams config_flags=[ 1980eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1990eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 200aae36d18SPatrick Williams ], 201ee3c9eebSPatrick Williams ), 20202871c91SPatrick Williams # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb 203ee3c9eebSPatrick Williams "nlohmann/fifo_map": PackageDef( 204ee3c9eebSPatrick Williams rev="0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9", 205ee3c9eebSPatrick Williams build_type="custom", 206ee3c9eebSPatrick Williams build_steps=[f"cp src/fifo_map.hpp {prefix}/include/"], 207ee3c9eebSPatrick Williams ), 20851224161SPatrick Williams # version from meta-openembedded/meta-oe/recipes-devtools/unifex/unifex_git.bb 20951224161SPatrick Williams "facebookexperimental/libunifex": PackageDef( 21051224161SPatrick Williams rev="9df21c58d34ce8a1cd3b15c3a7347495e29417a0", 21151224161SPatrick Williams build_type="cmake", 21251224161SPatrick Williams config_flags=[ 21351224161SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 21451224161SPatrick Williams "-DBUILD_TESTING=OFF", 21551224161SPatrick Williams "-DCMAKE_CXX_STANDARD=20", 21651224161SPatrick Williams "-DUNIFEX_BUILD_EXAMPLES=OFF", 21751224161SPatrick Williams ], 21851224161SPatrick Williams ), 219ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 220ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 221ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 222ee3c9eebSPatrick Williams build_type="meson", 223ee3c9eebSPatrick Williams config_flags=[ 224aae36d18SPatrick Williams "-Dexamples=false", 225aae36d18SPatrick Williams "-Dtests=disabled", 226aae36d18SPatrick Williams ], 227ee3c9eebSPatrick Williams ), 228ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 229ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 230ee3c9eebSPatrick Williams build_type="meson", 231ee3c9eebSPatrick Williams ), 232ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 233ee3c9eebSPatrick Williams depends=[ 23483394610SPatrick Williams "USCiLab/cereal", 23583394610SPatrick Williams "nlohmann/fifo_map", 23683394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 23783394610SPatrick Williams "openbmc/sdbusplus", 23883394610SPatrick Williams "openbmc/sdeventplus", 239aae36d18SPatrick Williams ], 240f79ce4c4SPatrick Williams build_type="meson", 241ee3c9eebSPatrick Williams config_flags=[ 242f79ce4c4SPatrick Williams f"-Dyaml_dir={prefix}/share/phosphor-dbus-yaml/yaml", 243aae36d18SPatrick Williams ], 244ee3c9eebSPatrick Williams ), 245ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 246ee3c9eebSPatrick Williams depends=[ 24783394610SPatrick Williams "boost", 24883394610SPatrick Williams "leethomason/tinyxml2", 24983394610SPatrick Williams "openbmc/phosphor-logging", 25083394610SPatrick Williams "openbmc/sdbusplus", 251aae36d18SPatrick Williams ], 2521197e359SBrad Bishop build_type="meson", 2531197e359SBrad Bishop config_flags=[ 2541197e359SBrad Bishop "-Dtests=disabled", 2551197e359SBrad Bishop ], 256ee3c9eebSPatrick Williams ), 257ee3c9eebSPatrick Williams "openbmc/pldm": PackageDef( 258ee3c9eebSPatrick Williams depends=[ 25983394610SPatrick Williams "CLIUtils/CLI11", 26083394610SPatrick Williams "boost", 26183394610SPatrick Williams "nlohmann/json", 26283394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 26383394610SPatrick Williams "openbmc/phosphor-logging", 26483394610SPatrick Williams "openbmc/sdbusplus", 26583394610SPatrick Williams "openbmc/sdeventplus", 266aae36d18SPatrick Williams ], 267ee3c9eebSPatrick Williams build_type="meson", 268ee3c9eebSPatrick Williams config_flags=[ 269aae36d18SPatrick Williams "-Dlibpldm-only=enabled", 270aae36d18SPatrick Williams "-Doem-ibm=enabled", 271aae36d18SPatrick Williams "-Dtests=disabled", 272aae36d18SPatrick Williams ], 273ee3c9eebSPatrick Williams ), 274ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 27551224161SPatrick Williams depends=[ 27651224161SPatrick Williams "facebookexperimental/libunifex", 27751224161SPatrick Williams ], 278ee3c9eebSPatrick Williams build_type="meson", 279ee3c9eebSPatrick Williams custom_post_dl=[ 280aae36d18SPatrick Williams "cd tools", 281aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 282aae36d18SPatrick Williams "cd ..", 283aae36d18SPatrick Williams ], 284ee3c9eebSPatrick Williams config_flags=[ 285aae36d18SPatrick Williams "-Dexamples=disabled", 286aae36d18SPatrick Williams "-Dtests=disabled", 287aae36d18SPatrick Williams ], 288b16f3e20SPatrick Williams ), 289ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 290ee3c9eebSPatrick Williams depends=["Naios/function2", "openbmc/stdplus"], 291ee3c9eebSPatrick Williams build_type="meson", 292ee3c9eebSPatrick Williams config_flags=[ 293ee3c9eebSPatrick Williams "-Dexamples=false", 294ee3c9eebSPatrick Williams "-Dtests=disabled", 295ee3c9eebSPatrick Williams ], 296ee3c9eebSPatrick Williams ), 297ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 298ee3c9eebSPatrick Williams depends=["fmtlib/fmt", "martinmoene/span-lite"], 299ee3c9eebSPatrick Williams build_type="meson", 300ee3c9eebSPatrick Williams config_flags=[ 301ee3c9eebSPatrick Williams "-Dexamples=false", 302ee3c9eebSPatrick Williams "-Dtests=disabled", 303ee3c9eebSPatrick Williams ], 304ee3c9eebSPatrick Williams ), 305ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 30602871c91SPatrick Williams 30702871c91SPatrick Williams# Define common flags used for builds 30802871c91SPatrick Williamsconfigure_flags = " ".join( 30902871c91SPatrick Williams [ 31002871c91SPatrick Williams f"--prefix={prefix}", 31102871c91SPatrick Williams ] 31202871c91SPatrick Williams) 31302871c91SPatrick Williamscmake_flags = " ".join( 31402871c91SPatrick Williams [ 31502871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 3160f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 31702871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 3180f2086b3SPatrick Williams "-GNinja", 3190f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 32002871c91SPatrick Williams ] 32102871c91SPatrick Williams) 32202871c91SPatrick Williamsmeson_flags = " ".join( 32302871c91SPatrick Williams [ 32402871c91SPatrick Williams "--wrap-mode=nodownload", 32502871c91SPatrick Williams f"-Dprefix={prefix}", 32602871c91SPatrick Williams ] 32702871c91SPatrick Williams) 32802871c91SPatrick Williams 329ee3c9eebSPatrick Williams 330ee3c9eebSPatrick Williamsclass Package(threading.Thread): 331ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 332ee3c9eebSPatrick Williams 333ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 334ee3c9eebSPatrick Williams Package.generate_all(). 335ee3c9eebSPatrick Williams """ 336ee3c9eebSPatrick Williams 337ee3c9eebSPatrick Williams # Copy the packages dictionary. 338ee3c9eebSPatrick Williams packages = packages.copy() 339ee3c9eebSPatrick Williams 340ee3c9eebSPatrick Williams # Lock used for thread-safety. 341ee3c9eebSPatrick Williams lock = threading.Lock() 342ee3c9eebSPatrick Williams 343ee3c9eebSPatrick Williams def __init__(self, pkg: str): 344ee3c9eebSPatrick Williams """ pkg - The name of this package (ex. foo/bar ) """ 345ee3c9eebSPatrick Williams super(Package, self).__init__() 346ee3c9eebSPatrick Williams 347ee3c9eebSPatrick Williams self.package = pkg 348ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 349ee3c9eebSPatrick Williams 350ee3c9eebSPatrick Williams # Reference to this package's 351ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 352ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 353ee3c9eebSPatrick Williams 354ee3c9eebSPatrick Williams def run(self) -> None: 355ee3c9eebSPatrick Williams """ Thread 'run' function. Builds the Docker stage. """ 356ee3c9eebSPatrick Williams 357ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 358ee3c9eebSPatrick Williams self._update_rev() 359ee3c9eebSPatrick Williams 360ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 361ee3c9eebSPatrick Williams # This section is locked because we are looking into another 362ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 363ee3c9eebSPatrick Williams Package.lock.acquire() 364ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 365ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 366ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 367ee3c9eebSPatrick Williams ] 368ee3c9eebSPatrick Williams Package.lock.release() 369ee3c9eebSPatrick Williams 370ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 371ee3c9eebSPatrick Williams # for the "COPY" commands. 372ee3c9eebSPatrick Williams for deppkg in deps: 373ee3c9eebSPatrick Williams deppkg.join() 374ee3c9eebSPatrick Williams 375ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 376ee3c9eebSPatrick Williams dockerfile = f""" 377ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 378ee3c9eebSPatrick Williams{self._df_copycmds()} 379ee3c9eebSPatrick Williams{self._df_build()} 380ee3c9eebSPatrick Williams""" 381ee3c9eebSPatrick Williams 382ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 383ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 384ee3c9eebSPatrick Williams # which can be accessed by other threads. 385ee3c9eebSPatrick Williams Package.lock.acquire() 386ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 387ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 388ee3c9eebSPatrick Williams Package.lock.release() 389ee3c9eebSPatrick Williams 390ee3c9eebSPatrick Williams # Do the build / save any exceptions. 391ee3c9eebSPatrick Williams try: 392ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 393ee3c9eebSPatrick Williams except Exception as e: 394ee3c9eebSPatrick Williams self.exception = e 395ee3c9eebSPatrick Williams 396ee3c9eebSPatrick Williams @classmethod 397ee3c9eebSPatrick Williams def generate_all(cls) -> None: 398ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 399ee3c9eebSPatrick Williams 400ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 401ee3c9eebSPatrick Williams package 'depends' specifications. 402ee3c9eebSPatrick Williams """ 403ee3c9eebSPatrick Williams 404ee3c9eebSPatrick Williams # Create a Package for each defined package. 405ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 406ee3c9eebSPatrick Williams 407ee3c9eebSPatrick Williams # Start building them all. 4086dbd7807SPatrick Williams # This section is locked because threads depend on each other, 4096dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 4106dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 4116dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 4126dbd7807SPatrick Williams Package.lock.acquire() 413ee3c9eebSPatrick Williams for t in pkg_threads: 414ee3c9eebSPatrick Williams t.start() 4156dbd7807SPatrick Williams Package.lock.release() 416ee3c9eebSPatrick Williams 417ee3c9eebSPatrick Williams # Wait for completion. 418ee3c9eebSPatrick Williams for t in pkg_threads: 419ee3c9eebSPatrick Williams t.join() 420ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 421ee3c9eebSPatrick Williams if t.exception: 422ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 423ee3c9eebSPatrick Williams raise t.exception 424ee3c9eebSPatrick Williams 425ee3c9eebSPatrick Williams @staticmethod 426ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 427ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 428ee3c9eebSPatrick Williams into the final image. 429ee3c9eebSPatrick Williams """ 430ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 431ee3c9eebSPatrick Williams 432ee3c9eebSPatrick Williams @classmethod 433ee3c9eebSPatrick Williams def depcache(cls) -> str: 434ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 435ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 436ee3c9eebSPatrick Williams """ 437ee3c9eebSPatrick Williams 438ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 439ee3c9eebSPatrick Williams depcache = "" 440ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 441ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 442ee3c9eebSPatrick Williams return depcache 443ee3c9eebSPatrick Williams 444ee3c9eebSPatrick Williams def _update_rev(self) -> None: 445ee3c9eebSPatrick Williams """ Look up the HEAD for missing a static rev. """ 446ee3c9eebSPatrick Williams 447ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 448ee3c9eebSPatrick Williams return 449ee3c9eebSPatrick Williams 45065b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 45165b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 45265b21fb9SPatrick Williams print( 45365b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 45465b21fb9SPatrick Williams file=sys.stderr, 45565b21fb9SPatrick Williams ) 45665b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 45765b21fb9SPatrick Williams return 45865b21fb9SPatrick Williams 459ee3c9eebSPatrick Williams # Ask Github for all the branches. 460ee3c9eebSPatrick Williams lookup = git("ls-remote", "--heads", f"https://github.com/{self.package}") 461ee3c9eebSPatrick Williams 462ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 463ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 464ee3c9eebSPatrick Williams Package.lock.acquire() 465ee3c9eebSPatrick Williams for line in lookup.split("\n"): 466ee3c9eebSPatrick Williams if f"refs/heads/{branch}" in line: 467ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 468ee3c9eebSPatrick Williams elif f"refs/heads/master" in line and "rev" not in self.pkg_def: 469ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 470ee3c9eebSPatrick Williams Package.lock.release() 471ee3c9eebSPatrick Williams 472ee3c9eebSPatrick Williams def _stagename(self) -> str: 473ee3c9eebSPatrick Williams """ Create a name for the Docker stage associated with this pkg. """ 474ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 475ee3c9eebSPatrick Williams 476ee3c9eebSPatrick Williams def _url(self) -> str: 477ee3c9eebSPatrick Williams """ Get the URL for this package. """ 478ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 479ee3c9eebSPatrick Williams 480ee3c9eebSPatrick Williams # If the lambda exists, call it. 481ee3c9eebSPatrick Williams if "url" in self.pkg_def: 482ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 483ee3c9eebSPatrick Williams 484ee3c9eebSPatrick Williams # Default to the github archive URL. 485ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 486ee3c9eebSPatrick Williams 487ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 488ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 489ee3c9eebSPatrick Williams 490ee3c9eebSPatrick Williams url = self._url() 491ee3c9eebSPatrick Williams if ".tar." not in url: 492ee3c9eebSPatrick Williams raise NotImplementedError( 493ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 494ee3c9eebSPatrick Williams ) 495ee3c9eebSPatrick Williams 496ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 497ee3c9eebSPatrick Williams 498ee3c9eebSPatrick Williams if url.endswith(".bz2"): 499ee3c9eebSPatrick Williams cmd += "j" 500ee3c9eebSPatrick Williams elif url.endswith(".gz"): 501ee3c9eebSPatrick Williams cmd += "z" 502ee3c9eebSPatrick Williams else: 503ee3c9eebSPatrick Williams raise NotImplementedError( 504ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 505ee3c9eebSPatrick Williams ) 506ee3c9eebSPatrick Williams 507ee3c9eebSPatrick Williams return cmd 508ee3c9eebSPatrick Williams 509ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 510ee3c9eebSPatrick Williams """ Formulate the command necessary to 'cd' into the source dir. """ 511ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 512ee3c9eebSPatrick Williams 513ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 514ee3c9eebSPatrick Williams """ Formulate the dockerfile snippet necessary to COPY all depends. """ 515ee3c9eebSPatrick Williams 516ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 517ee3c9eebSPatrick Williams return "" 518ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 519ee3c9eebSPatrick Williams 520ee3c9eebSPatrick Williams @staticmethod 521ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 522ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 523ee3c9eebSPatrick Williams packages into a Docker stage. 524ee3c9eebSPatrick Williams """ 525ee3c9eebSPatrick Williams 526ee3c9eebSPatrick Williams copy_cmds = "" 527ee3c9eebSPatrick Williams 528ee3c9eebSPatrick Williams # Sort the packages for consistency. 529ee3c9eebSPatrick Williams for p in sorted(pkgs): 530ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 531ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 532ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 533ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 534ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 535ee3c9eebSPatrick Williams 536ee3c9eebSPatrick Williams return copy_cmds 537ee3c9eebSPatrick Williams 538ee3c9eebSPatrick Williams def _df_build(self) -> str: 539ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 540ee3c9eebSPatrick Williams install a package into a Docker stage. 541ee3c9eebSPatrick Williams """ 542ee3c9eebSPatrick Williams 543ee3c9eebSPatrick Williams # Download and extract source. 544ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 545ee3c9eebSPatrick Williams 546ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 547ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 548ee3c9eebSPatrick Williams if custom_post_dl: 549ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 550ee3c9eebSPatrick Williams 551ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 552ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 553ee3c9eebSPatrick Williams if build_type == "autoconf": 554ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 555ee3c9eebSPatrick Williams elif build_type == "cmake": 556ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 557ee3c9eebSPatrick Williams elif build_type == "custom": 558ee3c9eebSPatrick Williams result += self._cmd_build_custom() 559ee3c9eebSPatrick Williams elif build_type == "make": 560ee3c9eebSPatrick Williams result += self._cmd_build_make() 561ee3c9eebSPatrick Williams elif build_type == "meson": 562ee3c9eebSPatrick Williams result += self._cmd_build_meson() 563ee3c9eebSPatrick Williams else: 564ee3c9eebSPatrick Williams raise NotImplementedError( 565ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 566ee3c9eebSPatrick Williams ) 567ee3c9eebSPatrick Williams 5686bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5696bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5706bce2ca1SPatrick Williams if custom_post_install: 5716bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5726bce2ca1SPatrick Williams 573ee3c9eebSPatrick Williams return result 574ee3c9eebSPatrick Williams 575ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 576ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 577ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 578ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 579ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 580ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 581ee3c9eebSPatrick Williams return result 582ee3c9eebSPatrick Williams 583ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 584ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 585ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 586ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 587ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 588ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 589ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 590ee3c9eebSPatrick Williams result += "cd .." 591ee3c9eebSPatrick Williams return result 592ee3c9eebSPatrick Williams 593ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 594ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 595ee3c9eebSPatrick Williams 596ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 597ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 598ee3c9eebSPatrick Williams 599ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 600ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 601ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 602ee3c9eebSPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 603ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 604ee3c9eebSPatrick Williams return result 605ee3c9eebSPatrick Williams 606ee3c9eebSPatrick Williams 607ee3c9eebSPatrick Williamsclass Docker: 608ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 609ee3c9eebSPatrick Williams 610ee3c9eebSPatrick Williams @staticmethod 611ee3c9eebSPatrick Williams def timestamp() -> str: 612ee3c9eebSPatrick Williams """ Generate a timestamp for today using the ISO week. """ 613ee3c9eebSPatrick Williams today = date.today().isocalendar() 614ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 615ee3c9eebSPatrick Williams 616ee3c9eebSPatrick Williams @staticmethod 617ee3c9eebSPatrick Williams def tagname(pkgname: str, dockerfile: str) -> str: 618ee3c9eebSPatrick Williams """ Generate a tag name for a package using a hash of the Dockerfile. """ 619ee3c9eebSPatrick Williams result = docker_image_name 620ee3c9eebSPatrick Williams if pkgname: 621ee3c9eebSPatrick Williams result += "-" + pkgname 622ee3c9eebSPatrick Williams 623ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 624ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 625ee3c9eebSPatrick Williams 626ee3c9eebSPatrick Williams return result 627ee3c9eebSPatrick Williams 628ee3c9eebSPatrick Williams @staticmethod 629ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 630ee3c9eebSPatrick Williams """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 631ee3c9eebSPatrick Williams 632ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 633ee3c9eebSPatrick Williams if not force_build: 634ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 635ee3c9eebSPatrick Williams print(f"Image {tag} already exists. Skipping.", file=sys.stderr) 636ee3c9eebSPatrick Williams return 637ee3c9eebSPatrick Williams 638ee3c9eebSPatrick Williams # Build it. 639ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 640ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 641ee3c9eebSPatrick Williams # progress but not polute stdout. Later on we output the final 642ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 643ee3c9eebSPatrick Williams # 644ee3c9eebSPatrick Williams # Other unusual flags: 645ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 646ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 647ee3c9eebSPatrick Williams docker.build( 648ee3c9eebSPatrick Williams proxy_args, 649ee3c9eebSPatrick Williams "--network=host", 650ee3c9eebSPatrick Williams "--force-rm", 651ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 652ee3c9eebSPatrick Williams "-t", 653ee3c9eebSPatrick Williams tag, 654ee3c9eebSPatrick Williams "-", 655ee3c9eebSPatrick Williams _in=dockerfile, 656ee3c9eebSPatrick Williams _out=( 657ee3c9eebSPatrick Williams lambda line: print( 658ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 659ee3c9eebSPatrick Williams ) 660ee3c9eebSPatrick Williams ), 661ee3c9eebSPatrick Williams ) 662ee3c9eebSPatrick Williams 663ee3c9eebSPatrick Williams 664ee3c9eebSPatrick Williams# Read a bunch of environment variables. 665ee3c9eebSPatrick Williamsdocker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 666ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 667ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 6683b6bfe42SPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:impish") 669ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 670ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 671ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 672ee3c9eebSPatrick Williams 67365b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 67465b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 67565b21fb9SPatrick Williams 676ee3c9eebSPatrick Williams# Set up some common variables. 677ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 678ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 679ee3c9eebSPatrick Williamsgid = os.getgid() 680ee3c9eebSPatrick Williamsuid = os.getuid() 681ee3c9eebSPatrick Williams 6826825a018SJosh Lehan# Use well-known constants if user is root 6836825a018SJosh Lehanif username == "root": 6846825a018SJosh Lehan homedir = "/root" 6856825a018SJosh Lehan gid = 0 6866825a018SJosh Lehan uid = 0 6876825a018SJosh Lehan 688ee3c9eebSPatrick Williams# Determine the architecture for Docker. 689ee3c9eebSPatrick Williamsarch = uname("-m").strip() 690ee3c9eebSPatrick Williamsif arch == "ppc64le": 691ee3c9eebSPatrick Williams docker_base = "ppc64le/" 692ee3c9eebSPatrick Williamselif arch == "x86_64": 693ee3c9eebSPatrick Williams docker_base = "" 694051b05b7SThang Q. Nguyenelif arch == "aarch64": 695f98f1a8dSThang Q. Nguyen docker_base = "arm64v8/" 696ee3c9eebSPatrick Williamselse: 697ee3c9eebSPatrick Williams print( 698ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 699ee3c9eebSPatrick Williams file=sys.stderr, 700ee3c9eebSPatrick Williams ) 701ee3c9eebSPatrick Williams sys.exit(1) 702ee3c9eebSPatrick Williams 70302871c91SPatrick Williams# Special flags if setting up a deb mirror. 70402871c91SPatrick Williamsmirror = "" 70502871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 70602871c91SPatrick Williams mirror = f""" 70702871c91SPatrick WilliamsRUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 70802871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 70902871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 71002871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 71102871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 71202871c91SPatrick Williams""" 71302871c91SPatrick Williams 71402871c91SPatrick Williams# Special flags for proxying. 71502871c91SPatrick Williamsproxy_cmd = "" 71634ec77e8SAdrian Ambrożewiczproxy_keyserver = "" 71702871c91SPatrick Williamsproxy_args = [] 71802871c91SPatrick Williamsif http_proxy: 71902871c91SPatrick Williams proxy_cmd = f""" 72002871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 72102871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 72202871c91SPatrick Williams""" 72334ec77e8SAdrian Ambrożewicz proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 72434ec77e8SAdrian Ambrożewicz 72502871c91SPatrick Williams proxy_args.extend( 72602871c91SPatrick Williams [ 72702871c91SPatrick Williams "--build-arg", 72802871c91SPatrick Williams f"http_proxy={http_proxy}", 72902871c91SPatrick Williams "--build-arg", 730d461cd6aSLei YU f"https_proxy={http_proxy}", 73102871c91SPatrick Williams ] 73202871c91SPatrick Williams ) 73302871c91SPatrick Williams 734ee3c9eebSPatrick Williams# Create base Dockerfile. 735a18d9c57SPatrick Williamsdockerfile_base = f""" 736a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 73702871c91SPatrick Williams 73802871c91SPatrick Williams{mirror} 73902871c91SPatrick Williams 74002871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 74102871c91SPatrick Williams 742abace2cbSAndrew GeisslerENV PYTHONPATH "/usr/local/lib/python3.9/site-packages/" 74302871c91SPatrick Williams 744bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 745bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 746bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 747bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 748bb16ac14SPatrick Williams# the update. 749bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 750bb16ac14SPatrick Williams 75102871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 75202871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 75302871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 75450837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 755f79ce4c4SPatrick Williams ( apt-get install gpgv ubuntu-dbgsym-keyring || \ 75650837436SPatrick Williams ( apt-get install -yy dirmngr && \ 75750837436SPatrick Williams apt-key adv --keyserver keyserver.ubuntu.com \ 75834ec77e8SAdrian Ambrożewicz {proxy_keyserver} \ 75950837436SPatrick Williams --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 76002871c91SPatrick Williams 76102871c91SPatrick Williams# Parse the current repo list into a debug repo list 76202871c91SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 76302871c91SPatrick Williams 76402871c91SPatrick Williams# Remove non-existent debug repos 76502871c91SPatrick WilliamsRUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 76602871c91SPatrick Williams 76702871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 76802871c91SPatrick Williams 76902871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 770b84d59dcSWilliam A. Kennington III gcc-11 \ 771b84d59dcSWilliam A. Kennington III g++-11 \ 77202871c91SPatrick Williams libc6-dbg \ 77302871c91SPatrick Williams libc6-dev \ 77402871c91SPatrick Williams libtool \ 77502871c91SPatrick Williams bison \ 77602871c91SPatrick Williams libdbus-1-dev \ 77702871c91SPatrick Williams flex \ 77802871c91SPatrick Williams cmake \ 77902871c91SPatrick Williams python3 \ 78002871c91SPatrick Williams python3-dev\ 78102871c91SPatrick Williams python3-yaml \ 78202871c91SPatrick Williams python3-mako \ 78302871c91SPatrick Williams python3-pip \ 78402871c91SPatrick Williams python3-setuptools \ 78502871c91SPatrick Williams python3-git \ 78602871c91SPatrick Williams python3-socks \ 78702871c91SPatrick Williams pkg-config \ 78802871c91SPatrick Williams autoconf \ 78902871c91SPatrick Williams autoconf-archive \ 79002871c91SPatrick Williams libsystemd-dev \ 79102871c91SPatrick Williams systemd \ 79202871c91SPatrick Williams libssl-dev \ 79302871c91SPatrick Williams libevdev-dev \ 79402871c91SPatrick Williams libjpeg-dev \ 79502871c91SPatrick Williams libpng-dev \ 79602871c91SPatrick Williams ninja-build \ 79702871c91SPatrick Williams sudo \ 79802871c91SPatrick Williams curl \ 79902871c91SPatrick Williams git \ 80002871c91SPatrick Williams dbus \ 80102871c91SPatrick Williams iputils-ping \ 802f89c8508SPatrick Williams clang-13 \ 803f89c8508SPatrick Williams clang-format-13 \ 804f89c8508SPatrick Williams clang-tidy-13 \ 805f89c8508SPatrick Williams clang-tools-13 \ 80602871c91SPatrick Williams shellcheck \ 80702871c91SPatrick Williams npm \ 80802871c91SPatrick Williams iproute2 \ 80902871c91SPatrick Williams libnl-3-dev \ 81002871c91SPatrick Williams libnl-genl-3-dev \ 81102871c91SPatrick Williams libconfig++-dev \ 81202871c91SPatrick Williams libsnmp-dev \ 81302871c91SPatrick Williams valgrind \ 81402871c91SPatrick Williams valgrind-dbg \ 81502871c91SPatrick Williams libpam0g-dev \ 81602871c91SPatrick Williams xxd \ 81702871c91SPatrick Williams libi2c-dev \ 81802871c91SPatrick Williams wget \ 81902871c91SPatrick Williams libldap2-dev \ 82002871c91SPatrick Williams libprotobuf-dev \ 821dafe7529SWilliam A. Kennington III liburing-dev \ 822dafe7529SWilliam A. Kennington III liburing1-dbgsym \ 82302871c91SPatrick Williams libperlio-gzip-perl \ 82402871c91SPatrick Williams libjson-perl \ 82502871c91SPatrick Williams protobuf-compiler \ 82602871c91SPatrick Williams libgpiod-dev \ 82702871c91SPatrick Williams device-tree-compiler \ 82802871c91SPatrick Williams cppcheck \ 82902871c91SPatrick Williams libpciaccess-dev \ 83002871c91SPatrick Williams libmimetic-dev \ 83102871c91SPatrick Williams libxml2-utils \ 8320eedeedaSPatrick Williams libxml-simple-perl \ 8339adf68d6SJohn Wedig rsync \ 8349adf68d6SJohn Wedig libcryptsetup-dev 83502871c91SPatrick Williams 83687111bb7SManojkiran EdaRUN npm install -g eslint@latest eslint-plugin-json@latest 83787111bb7SManojkiran Eda 838b84d59dcSWilliam A. Kennington IIIRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 11 \ 839b84d59dcSWilliam A. Kennington III --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ 840b84d59dcSWilliam A. Kennington III --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ 841b84d59dcSWilliam A. Kennington III --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-11 \ 842b84d59dcSWilliam A. Kennington III --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-11 84302871c91SPatrick Williams 844f89c8508SPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 1000 \ 845f89c8508SPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-13 \ 846f89c8508SPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-13 \ 847f89c8508SPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-13 \ 848f89c8508SPatrick Williams --slave /usr/bin/run-clang-tidy run-clang-tidy.py /usr/bin/run-clang-tidy-13 \ 849f89c8508SPatrick Williams --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-13 85002871c91SPatrick Williams 85150837436SPatrick Williams""" 85250837436SPatrick Williams 85350837436SPatrick Williamsif is_automated_ci_build: 85450837436SPatrick Williams dockerfile_base += f""" 85550837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 85650837436SPatrick Williams# to re-run `apt-get update` daily. 857ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 85850837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 85950837436SPatrick Williams 86050837436SPatrick Williams""" 86150837436SPatrick Williams 86250837436SPatrick Williamsdockerfile_base += f""" 86302871c91SPatrick WilliamsRUN pip3 install inflection 86402871c91SPatrick WilliamsRUN pip3 install pycodestyle 86502871c91SPatrick WilliamsRUN pip3 install jsonschema 866b3e88fb6SPatrick WilliamsRUN pip3 install meson==0.58.1 86702871c91SPatrick WilliamsRUN pip3 install protobuf 868e6f120aaSManojkiran EdaRUN pip3 install codespell 869ca8c4a8bSEd TanousRUN pip3 install requests 870a18d9c57SPatrick Williams""" 87102871c91SPatrick Williams 872ee3c9eebSPatrick Williams# Build the base and stage docker images. 873ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 874ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 875ee3c9eebSPatrick WilliamsPackage.generate_all() 87602871c91SPatrick Williams 877ee3c9eebSPatrick Williams# Create the final Dockerfile. 878a18d9c57SPatrick Williamsdockerfile = f""" 87902871c91SPatrick Williams# Build the final output image 880a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 881ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 88202871c91SPatrick Williams 88302871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 88402871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 88502871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 886ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 88702871c91SPatrick Williams 88802871c91SPatrick Williams# Final configuration for the workspace 8896825a018SJosh LehanRUN grep -q {gid} /etc/group || groupadd -f -g {gid} {username} 89002871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 89102871c91SPatrick WilliamsRUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 89202871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 89302871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 89402871c91SPatrick Williams 895305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 896305a9a5dSAndrew Geissler# and data installs 8977bb00b13SAndrew GeisslerRUN chown -R {username}:{username} /usr/local/share 898305a9a5dSAndrew Geissler 89902871c91SPatrick Williams{proxy_cmd} 90002871c91SPatrick Williams 90102871c91SPatrick WilliamsRUN /bin/bash 90202871c91SPatrick Williams""" 90302871c91SPatrick Williams 904a18d9c57SPatrick Williams# Do the final docker build 905ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 906ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 907ee3c9eebSPatrick Williams 90800536fbeSPatrick Williams# Print the tag of the final image. 90900536fbeSPatrick Williamsprint(docker_final_img_name) 910