102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1050837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1150837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1250837436SPatrick Williams# (ex. Jenkins)> 1302871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1402871c91SPatrick Williams# repositories> 1502871c91SPatrick Williams# default is master, which will be used if input branch not 1602871c91SPatrick Williams# provided or not found 1702871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1802871c91SPatrick Williams# default ones in /etc/apt/sources.list> 1902871c91SPatrick Williams# default is empty, and no mirror is used. 2002871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2102871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2202871c91SPatrick Williams 2302871c91SPatrick Williamsimport os 2402871c91SPatrick Williamsimport sys 25b16f3e20SPatrick Williamsimport threading 26a18d9c57SPatrick Williamsfrom datetime import date 27a18d9c57SPatrick Williamsfrom hashlib import sha256 28ee3c9eebSPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 29ee3c9eebSPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional 3002871c91SPatrick Williams 31ee3c9eebSPatrick Williamstry: 32ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 33ee3c9eebSPatrick Williams from typing import TypedDict 34ee3c9eebSPatrick Williamsexcept: 35ee3c9eebSPatrick Williams 36ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 37ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 38ee3c9eebSPatrick Williams def __init_subclass__(cls, **kwargs): 39ee3c9eebSPatrick Williams super().__init_subclass__() 40ee3c9eebSPatrick Williams 41ee3c9eebSPatrick Williams 42ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 43aae36d18SPatrick Williamsprefix = "/usr/local" 4402871c91SPatrick Williamsproc_count = nproc().strip() 4502871c91SPatrick Williams 46ee3c9eebSPatrick Williams 47ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 48ee3c9eebSPatrick Williams """ Package Definition for packages dictionary. """ 49ee3c9eebSPatrick Williams 50ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 51ee3c9eebSPatrick Williams rev: str 52ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 53ee3c9eebSPatrick Williams url: Callable[[str, str], str] 54ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 55ee3c9eebSPatrick Williams depends: Iterable[str] 56ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 57ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 58ee3c9eebSPatrick Williams build_type: str 59ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 60ee3c9eebSPatrick Williams build_steps: Iterable[str] 61ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 62ee3c9eebSPatrick Williams config_flags: Iterable[str] 63ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 64ee3c9eebSPatrick Williams config_env: Iterable[str] 65ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 66ee3c9eebSPatrick Williams # before config / build / install. 67ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 686bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 696bce2ca1SPatrick Williams custom_post_install: Iterable[str] 70ee3c9eebSPatrick Williams 71ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 72ee3c9eebSPatrick Williams __tag: str 73ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 74ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 75ee3c9eebSPatrick Williams 7602871c91SPatrick Williams 777204324cSPatrick Williams# Packages to include in image. 787204324cSPatrick Williamspackages = { 79ee3c9eebSPatrick Williams "boost": PackageDef( 806c55c580SEd Tanous rev="1.79.0", 81ee3c9eebSPatrick Williams url=( 825f2549eaSPatrick Williams lambda pkg, rev: f"https://downloads.yoctoproject.org/mirror/sources/{pkg}_{rev.replace('.', '_')}.tar.bz2" 832abc4a48SPatrick Williams ), 84ee3c9eebSPatrick Williams build_type="custom", 85ee3c9eebSPatrick Williams build_steps=[ 86aae36d18SPatrick Williams f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 87aae36d18SPatrick Williams "./b2", 88aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 89aae36d18SPatrick Williams ], 90ee3c9eebSPatrick Williams ), 91ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 923f8b5294SPatrick Williams rev="3e4d1b84cab4891368d2179a61a7ba06a5693e7f", 93ee3c9eebSPatrick Williams build_type="custom", 94ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 95ee3c9eebSPatrick Williams ), 96ee3c9eebSPatrick Williams "catchorg/Catch2": PackageDef( 9796b43455SWilliam A. Kennington III rev="v2.13.6", 98ee3c9eebSPatrick Williams build_type="cmake", 99ee3c9eebSPatrick Williams config_flags=["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"], 100ee3c9eebSPatrick Williams ), 101c7198558SEd Tanous "danmar/cppcheck": PackageDef( 102c7198558SEd Tanous rev="27578e9c4c1f90c62b6938867735a054082e178e", 103c7198558SEd Tanous build_type="cmake", 104c7198558SEd Tanous ), 105ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 106ee3c9eebSPatrick Williams rev="v1.9.1", 107ee3c9eebSPatrick Williams build_type="cmake", 108ee3c9eebSPatrick Williams config_flags=[ 109aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 110aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 111aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 112aae36d18SPatrick Williams ], 113ee3c9eebSPatrick Williams ), 114ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 11563075013SWilly Tu rev="8.1.1", 116ee3c9eebSPatrick Williams build_type="cmake", 117ee3c9eebSPatrick Williams config_flags=[ 118aae36d18SPatrick Williams "-DFMT_DOC=OFF", 119aae36d18SPatrick Williams "-DFMT_TEST=OFF", 120aae36d18SPatrick Williams ], 121ee3c9eebSPatrick Williams ), 122ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 12396b43455SWilliam A. Kennington III rev="4.1.0", 124ee3c9eebSPatrick Williams build_type="custom", 125ee3c9eebSPatrick Williams build_steps=[ 126aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 127aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 128aae36d18SPatrick Williams ], 129ee3c9eebSPatrick Williams ), 1305202d8eeSMichael Shen # Release 2021-06-12 131ee3c9eebSPatrick Williams "google/googletest": PackageDef( 132acfdee5dSWilly Tu rev="9e712372214d75bb30ec2847a44bf124d48096f3", 133ee3c9eebSPatrick Williams build_type="cmake", 1344dd32c02SWilliam A. Kennington III config_env=["CXXFLAGS=-std=c++20"], 135ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 136ee3c9eebSPatrick Williams ), 13702871c91SPatrick Williams # Release 2020-08-06 138ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 1397d1b2a1dSEd Tanous rev="v3.10.4", 1406bce2ca1SPatrick Williams build_type="cmake", 1416bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1426bce2ca1SPatrick Williams custom_post_install=[ 143aae36d18SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 144aae36d18SPatrick Williams ], 145ee3c9eebSPatrick Williams ), 14602871c91SPatrick Williams # Snapshot from 2019-05-24 147ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 148ee3c9eebSPatrick Williams rev="v1.15", 149ee3c9eebSPatrick Williams build_type="make", 150ee3c9eebSPatrick Williams ), 1510eedeedaSPatrick Williams # dev-5.8 2021-01-11 152ee3c9eebSPatrick Williams "openbmc/linux": PackageDef( 153ee3c9eebSPatrick Williams rev="3cc95ae40716e56f81b69615781f54c78079042d", 154ee3c9eebSPatrick Williams build_type="custom", 155ee3c9eebSPatrick Williams build_steps=[ 156aae36d18SPatrick Williams f"make -j{proc_count} defconfig", 157aae36d18SPatrick Williams f"make INSTALL_HDR_PATH={prefix} headers_install", 158aae36d18SPatrick Williams ], 159ee3c9eebSPatrick Williams ), 1600eedeedaSPatrick Williams # Snapshot from 2020-06-13 161ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 162ee3c9eebSPatrick Williams rev="LibVNCServer-0.9.13", 163ee3c9eebSPatrick Williams build_type="cmake", 164ee3c9eebSPatrick Williams ), 1650eedeedaSPatrick Williams # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_8.0.0.bb 166ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 167ee3c9eebSPatrick Williams rev="8.0.0", 168ee3c9eebSPatrick Williams build_type="cmake", 169ee3c9eebSPatrick Williams ), 17002871c91SPatrick Williams # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 171ee3c9eebSPatrick Williams "CPPAlliance/url": PackageDef( 172ab640cdaSEd Tanous rev="d740a92d38e3a8f4d5b2153f53b82f1c98e312ab", 173eed466e3SEd Tanous build_type="custom", 174eed466e3SEd Tanous build_steps=[f"cp -a include/** {prefix}/include/"], 175ee3c9eebSPatrick Williams ), 17668992895SPatrick Williams # version from meta-openembedded/meta-oe/dynamic-layers/networking-layer/recipes-devools/valijson/valijson_0.6.bb 177ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 17868992895SPatrick Williams rev="v0.6", 179ee3c9eebSPatrick Williams build_type="cmake", 180ee3c9eebSPatrick Williams config_flags=[ 1810eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1820eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 183aae36d18SPatrick Williams ], 184ee3c9eebSPatrick Williams ), 18551224161SPatrick Williams # version from meta-openembedded/meta-oe/recipes-devtools/unifex/unifex_git.bb 18651224161SPatrick Williams "facebookexperimental/libunifex": PackageDef( 18751224161SPatrick Williams rev="9df21c58d34ce8a1cd3b15c3a7347495e29417a0", 18851224161SPatrick Williams build_type="cmake", 18951224161SPatrick Williams config_flags=[ 19051224161SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 19151224161SPatrick Williams "-DBUILD_TESTING=OFF", 19251224161SPatrick Williams "-DCMAKE_CXX_STANDARD=20", 19351224161SPatrick Williams "-DUNIFEX_BUILD_EXAMPLES=OFF", 19451224161SPatrick Williams ], 19551224161SPatrick Williams ), 196ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 197ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 198ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 199ee3c9eebSPatrick Williams build_type="meson", 200ee3c9eebSPatrick Williams config_flags=[ 201aae36d18SPatrick Williams "-Dexamples=false", 202aae36d18SPatrick Williams "-Dtests=disabled", 203aae36d18SPatrick Williams ], 204ee3c9eebSPatrick Williams ), 205ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 206ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 207ee3c9eebSPatrick Williams build_type="meson", 2084fe87776SWilliam A. Kennington III config_flags=["-Dgenerate_md=false"], 209ee3c9eebSPatrick Williams ), 210ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 211ee3c9eebSPatrick Williams depends=[ 21283394610SPatrick Williams "USCiLab/cereal", 21383394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 21483394610SPatrick Williams "openbmc/sdbusplus", 21583394610SPatrick Williams "openbmc/sdeventplus", 216aae36d18SPatrick Williams ], 217f79ce4c4SPatrick Williams build_type="meson", 218ee3c9eebSPatrick Williams config_flags=[ 2195eabdae9SPatrick Williams f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml", 220aae36d18SPatrick Williams ], 221ee3c9eebSPatrick Williams ), 222ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 223ee3c9eebSPatrick Williams depends=[ 22483394610SPatrick Williams "boost", 22583394610SPatrick Williams "leethomason/tinyxml2", 22683394610SPatrick Williams "openbmc/phosphor-logging", 22783394610SPatrick Williams "openbmc/sdbusplus", 228aae36d18SPatrick Williams ], 2291197e359SBrad Bishop build_type="meson", 2301197e359SBrad Bishop config_flags=[ 2311197e359SBrad Bishop "-Dtests=disabled", 2321197e359SBrad Bishop ], 233ee3c9eebSPatrick Williams ), 234ee3c9eebSPatrick Williams "openbmc/pldm": PackageDef( 235ee3c9eebSPatrick Williams depends=[ 23683394610SPatrick Williams "CLIUtils/CLI11", 23783394610SPatrick Williams "boost", 23883394610SPatrick Williams "nlohmann/json", 23983394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 24083394610SPatrick Williams "openbmc/phosphor-logging", 24183394610SPatrick Williams "openbmc/sdbusplus", 24283394610SPatrick Williams "openbmc/sdeventplus", 243aae36d18SPatrick Williams ], 244ee3c9eebSPatrick Williams build_type="meson", 245ee3c9eebSPatrick Williams config_flags=[ 246aae36d18SPatrick Williams "-Dlibpldm-only=enabled", 247aae36d18SPatrick Williams "-Doem-ibm=enabled", 248aae36d18SPatrick Williams "-Dtests=disabled", 249aae36d18SPatrick Williams ], 250ee3c9eebSPatrick Williams ), 251ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 25251224161SPatrick Williams depends=[ 25351224161SPatrick Williams "facebookexperimental/libunifex", 25451224161SPatrick Williams ], 255ee3c9eebSPatrick Williams build_type="meson", 256ee3c9eebSPatrick Williams custom_post_dl=[ 257aae36d18SPatrick Williams "cd tools", 258aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 259aae36d18SPatrick Williams "cd ..", 260aae36d18SPatrick Williams ], 261ee3c9eebSPatrick Williams config_flags=[ 262aae36d18SPatrick Williams "-Dexamples=disabled", 263aae36d18SPatrick Williams "-Dtests=disabled", 264aae36d18SPatrick Williams ], 265b16f3e20SPatrick Williams ), 266ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 267ee3c9eebSPatrick Williams depends=["Naios/function2", "openbmc/stdplus"], 268ee3c9eebSPatrick Williams build_type="meson", 269ee3c9eebSPatrick Williams config_flags=[ 270ee3c9eebSPatrick Williams "-Dexamples=false", 271ee3c9eebSPatrick Williams "-Dtests=disabled", 272ee3c9eebSPatrick Williams ], 273ee3c9eebSPatrick Williams ), 274ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 275ce5f495aSWilliam A. Kennington III depends=["fmtlib/fmt", "google/googletest"], 276ee3c9eebSPatrick Williams build_type="meson", 277ee3c9eebSPatrick Williams config_flags=[ 278ee3c9eebSPatrick Williams "-Dexamples=false", 279ee3c9eebSPatrick Williams "-Dtests=disabled", 280ee3c9eebSPatrick Williams ], 281ee3c9eebSPatrick Williams ), 282ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 28302871c91SPatrick Williams 28402871c91SPatrick Williams# Define common flags used for builds 28502871c91SPatrick Williamsconfigure_flags = " ".join( 28602871c91SPatrick Williams [ 28702871c91SPatrick Williams f"--prefix={prefix}", 28802871c91SPatrick Williams ] 28902871c91SPatrick Williams) 29002871c91SPatrick Williamscmake_flags = " ".join( 29102871c91SPatrick Williams [ 29202871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 2930f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 29402871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 2950f2086b3SPatrick Williams "-GNinja", 2960f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 29702871c91SPatrick Williams ] 29802871c91SPatrick Williams) 29902871c91SPatrick Williamsmeson_flags = " ".join( 30002871c91SPatrick Williams [ 30102871c91SPatrick Williams "--wrap-mode=nodownload", 30202871c91SPatrick Williams f"-Dprefix={prefix}", 30302871c91SPatrick Williams ] 30402871c91SPatrick Williams) 30502871c91SPatrick Williams 306ee3c9eebSPatrick Williams 307ee3c9eebSPatrick Williamsclass Package(threading.Thread): 308ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 309ee3c9eebSPatrick Williams 310ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 311ee3c9eebSPatrick Williams Package.generate_all(). 312ee3c9eebSPatrick Williams """ 313ee3c9eebSPatrick Williams 314ee3c9eebSPatrick Williams # Copy the packages dictionary. 315ee3c9eebSPatrick Williams packages = packages.copy() 316ee3c9eebSPatrick Williams 317ee3c9eebSPatrick Williams # Lock used for thread-safety. 318ee3c9eebSPatrick Williams lock = threading.Lock() 319ee3c9eebSPatrick Williams 320ee3c9eebSPatrick Williams def __init__(self, pkg: str): 321ee3c9eebSPatrick Williams """ pkg - The name of this package (ex. foo/bar ) """ 322ee3c9eebSPatrick Williams super(Package, self).__init__() 323ee3c9eebSPatrick Williams 324ee3c9eebSPatrick Williams self.package = pkg 325ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 326ee3c9eebSPatrick Williams 327ee3c9eebSPatrick Williams # Reference to this package's 328ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 329ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 330ee3c9eebSPatrick Williams 331ee3c9eebSPatrick Williams def run(self) -> None: 332ee3c9eebSPatrick Williams """ Thread 'run' function. Builds the Docker stage. """ 333ee3c9eebSPatrick Williams 334ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 335ee3c9eebSPatrick Williams self._update_rev() 336ee3c9eebSPatrick Williams 337ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 338ee3c9eebSPatrick Williams # This section is locked because we are looking into another 339ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 340ee3c9eebSPatrick Williams Package.lock.acquire() 341ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 342ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 343ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 344ee3c9eebSPatrick Williams ] 345ee3c9eebSPatrick Williams Package.lock.release() 346ee3c9eebSPatrick Williams 347ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 348ee3c9eebSPatrick Williams # for the "COPY" commands. 349ee3c9eebSPatrick Williams for deppkg in deps: 350ee3c9eebSPatrick Williams deppkg.join() 351ee3c9eebSPatrick Williams 352ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 353ee3c9eebSPatrick Williams dockerfile = f""" 354ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 355ee3c9eebSPatrick Williams{self._df_copycmds()} 356ee3c9eebSPatrick Williams{self._df_build()} 357ee3c9eebSPatrick Williams""" 358ee3c9eebSPatrick Williams 359ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 360ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 361ee3c9eebSPatrick Williams # which can be accessed by other threads. 362ee3c9eebSPatrick Williams Package.lock.acquire() 363ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 364ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 365ee3c9eebSPatrick Williams Package.lock.release() 366ee3c9eebSPatrick Williams 367ee3c9eebSPatrick Williams # Do the build / save any exceptions. 368ee3c9eebSPatrick Williams try: 369ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 370ee3c9eebSPatrick Williams except Exception as e: 371ee3c9eebSPatrick Williams self.exception = e 372ee3c9eebSPatrick Williams 373ee3c9eebSPatrick Williams @classmethod 374ee3c9eebSPatrick Williams def generate_all(cls) -> None: 375ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 376ee3c9eebSPatrick Williams 377ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 378ee3c9eebSPatrick Williams package 'depends' specifications. 379ee3c9eebSPatrick Williams """ 380ee3c9eebSPatrick Williams 381ee3c9eebSPatrick Williams # Create a Package for each defined package. 382ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 383ee3c9eebSPatrick Williams 384ee3c9eebSPatrick Williams # Start building them all. 3856dbd7807SPatrick Williams # This section is locked because threads depend on each other, 3866dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 3876dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 3886dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 3896dbd7807SPatrick Williams Package.lock.acquire() 390ee3c9eebSPatrick Williams for t in pkg_threads: 391ee3c9eebSPatrick Williams t.start() 3926dbd7807SPatrick Williams Package.lock.release() 393ee3c9eebSPatrick Williams 394ee3c9eebSPatrick Williams # Wait for completion. 395ee3c9eebSPatrick Williams for t in pkg_threads: 396ee3c9eebSPatrick Williams t.join() 397ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 398ee3c9eebSPatrick Williams if t.exception: 399ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 400ee3c9eebSPatrick Williams raise t.exception 401ee3c9eebSPatrick Williams 402ee3c9eebSPatrick Williams @staticmethod 403ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 404ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 405ee3c9eebSPatrick Williams into the final image. 406ee3c9eebSPatrick Williams """ 407ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 408ee3c9eebSPatrick Williams 409ee3c9eebSPatrick Williams @classmethod 410ee3c9eebSPatrick Williams def depcache(cls) -> str: 411ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 412ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 413ee3c9eebSPatrick Williams """ 414ee3c9eebSPatrick Williams 415ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 416ee3c9eebSPatrick Williams depcache = "" 417ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 418ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 419ee3c9eebSPatrick Williams return depcache 420ee3c9eebSPatrick Williams 421ee3c9eebSPatrick Williams def _update_rev(self) -> None: 422ee3c9eebSPatrick Williams """ Look up the HEAD for missing a static rev. """ 423ee3c9eebSPatrick Williams 424ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 425ee3c9eebSPatrick Williams return 426ee3c9eebSPatrick Williams 42765b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 42865b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 42965b21fb9SPatrick Williams print( 43065b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 43165b21fb9SPatrick Williams file=sys.stderr, 43265b21fb9SPatrick Williams ) 43365b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 43465b21fb9SPatrick Williams return 43565b21fb9SPatrick Williams 436ee3c9eebSPatrick Williams # Ask Github for all the branches. 437ee3c9eebSPatrick Williams lookup = git("ls-remote", "--heads", f"https://github.com/{self.package}") 438ee3c9eebSPatrick Williams 439ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 440ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 441ee3c9eebSPatrick Williams Package.lock.acquire() 442ee3c9eebSPatrick Williams for line in lookup.split("\n"): 443ee3c9eebSPatrick Williams if f"refs/heads/{branch}" in line: 444ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 445ee3c9eebSPatrick Williams elif f"refs/heads/master" in line and "rev" not in self.pkg_def: 446ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 447ee3c9eebSPatrick Williams Package.lock.release() 448ee3c9eebSPatrick Williams 449ee3c9eebSPatrick Williams def _stagename(self) -> str: 450ee3c9eebSPatrick Williams """ Create a name for the Docker stage associated with this pkg. """ 451ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 452ee3c9eebSPatrick Williams 453ee3c9eebSPatrick Williams def _url(self) -> str: 454ee3c9eebSPatrick Williams """ Get the URL for this package. """ 455ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 456ee3c9eebSPatrick Williams 457ee3c9eebSPatrick Williams # If the lambda exists, call it. 458ee3c9eebSPatrick Williams if "url" in self.pkg_def: 459ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 460ee3c9eebSPatrick Williams 461ee3c9eebSPatrick Williams # Default to the github archive URL. 462ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 463ee3c9eebSPatrick Williams 464ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 465ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 466ee3c9eebSPatrick Williams 467ee3c9eebSPatrick Williams url = self._url() 468ee3c9eebSPatrick Williams if ".tar." not in url: 469ee3c9eebSPatrick Williams raise NotImplementedError( 470ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 471ee3c9eebSPatrick Williams ) 472ee3c9eebSPatrick Williams 473ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 474ee3c9eebSPatrick Williams 475ee3c9eebSPatrick Williams if url.endswith(".bz2"): 476ee3c9eebSPatrick Williams cmd += "j" 477ee3c9eebSPatrick Williams elif url.endswith(".gz"): 478ee3c9eebSPatrick Williams cmd += "z" 479ee3c9eebSPatrick Williams else: 480ee3c9eebSPatrick Williams raise NotImplementedError( 481ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 482ee3c9eebSPatrick Williams ) 483ee3c9eebSPatrick Williams 484ee3c9eebSPatrick Williams return cmd 485ee3c9eebSPatrick Williams 486ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 487ee3c9eebSPatrick Williams """ Formulate the command necessary to 'cd' into the source dir. """ 488ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 489ee3c9eebSPatrick Williams 490ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 491ee3c9eebSPatrick Williams """ Formulate the dockerfile snippet necessary to COPY all depends. """ 492ee3c9eebSPatrick Williams 493ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 494ee3c9eebSPatrick Williams return "" 495ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 496ee3c9eebSPatrick Williams 497ee3c9eebSPatrick Williams @staticmethod 498ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 499ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 500ee3c9eebSPatrick Williams packages into a Docker stage. 501ee3c9eebSPatrick Williams """ 502ee3c9eebSPatrick Williams 503ee3c9eebSPatrick Williams copy_cmds = "" 504ee3c9eebSPatrick Williams 505ee3c9eebSPatrick Williams # Sort the packages for consistency. 506ee3c9eebSPatrick Williams for p in sorted(pkgs): 507ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 508ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 509ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 510ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 511ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 512ee3c9eebSPatrick Williams 513ee3c9eebSPatrick Williams return copy_cmds 514ee3c9eebSPatrick Williams 515ee3c9eebSPatrick Williams def _df_build(self) -> str: 516ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 517ee3c9eebSPatrick Williams install a package into a Docker stage. 518ee3c9eebSPatrick Williams """ 519ee3c9eebSPatrick Williams 520ee3c9eebSPatrick Williams # Download and extract source. 521ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 522ee3c9eebSPatrick Williams 523ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 524ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 525ee3c9eebSPatrick Williams if custom_post_dl: 526ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 527ee3c9eebSPatrick Williams 528ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 529ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 530ee3c9eebSPatrick Williams if build_type == "autoconf": 531ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 532ee3c9eebSPatrick Williams elif build_type == "cmake": 533ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 534ee3c9eebSPatrick Williams elif build_type == "custom": 535ee3c9eebSPatrick Williams result += self._cmd_build_custom() 536ee3c9eebSPatrick Williams elif build_type == "make": 537ee3c9eebSPatrick Williams result += self._cmd_build_make() 538ee3c9eebSPatrick Williams elif build_type == "meson": 539ee3c9eebSPatrick Williams result += self._cmd_build_meson() 540ee3c9eebSPatrick Williams else: 541ee3c9eebSPatrick Williams raise NotImplementedError( 542ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 543ee3c9eebSPatrick Williams ) 544ee3c9eebSPatrick Williams 5456bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5466bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5476bce2ca1SPatrick Williams if custom_post_install: 5486bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5496bce2ca1SPatrick Williams 550ee3c9eebSPatrick Williams return result 551ee3c9eebSPatrick Williams 552ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 553ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 554ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 555ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 556ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 557ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 558ee3c9eebSPatrick Williams return result 559ee3c9eebSPatrick Williams 560ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 561ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 562ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 563ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 564ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 565ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 566ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 567ee3c9eebSPatrick Williams result += "cd .." 568ee3c9eebSPatrick Williams return result 569ee3c9eebSPatrick Williams 570ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 571ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 572ee3c9eebSPatrick Williams 573ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 574ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 575ee3c9eebSPatrick Williams 576ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 577ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 578ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 579ee3c9eebSPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 580ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 581ee3c9eebSPatrick Williams return result 582ee3c9eebSPatrick Williams 583ee3c9eebSPatrick Williams 584ee3c9eebSPatrick Williamsclass Docker: 585ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 586ee3c9eebSPatrick Williams 587ee3c9eebSPatrick Williams @staticmethod 588ee3c9eebSPatrick Williams def timestamp() -> str: 589ee3c9eebSPatrick Williams """ Generate a timestamp for today using the ISO week. """ 590ee3c9eebSPatrick Williams today = date.today().isocalendar() 591ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 592ee3c9eebSPatrick Williams 593ee3c9eebSPatrick Williams @staticmethod 594ee3c9eebSPatrick Williams def tagname(pkgname: str, dockerfile: str) -> str: 595ee3c9eebSPatrick Williams """ Generate a tag name for a package using a hash of the Dockerfile. """ 596ee3c9eebSPatrick Williams result = docker_image_name 597ee3c9eebSPatrick Williams if pkgname: 598ee3c9eebSPatrick Williams result += "-" + pkgname 599ee3c9eebSPatrick Williams 600ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 601ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 602ee3c9eebSPatrick Williams 603ee3c9eebSPatrick Williams return result 604ee3c9eebSPatrick Williams 605ee3c9eebSPatrick Williams @staticmethod 606ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 607ee3c9eebSPatrick Williams """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 608ee3c9eebSPatrick Williams 609ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 610ee3c9eebSPatrick Williams if not force_build: 611ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 612ee3c9eebSPatrick Williams print(f"Image {tag} already exists. Skipping.", file=sys.stderr) 613ee3c9eebSPatrick Williams return 614ee3c9eebSPatrick Williams 615ee3c9eebSPatrick Williams # Build it. 616ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 617ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 618ee3c9eebSPatrick Williams # progress but not polute stdout. Later on we output the final 619ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 620ee3c9eebSPatrick Williams # 621ee3c9eebSPatrick Williams # Other unusual flags: 622ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 623ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 624ee3c9eebSPatrick Williams docker.build( 625ee3c9eebSPatrick Williams proxy_args, 626ee3c9eebSPatrick Williams "--network=host", 627ee3c9eebSPatrick Williams "--force-rm", 628ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 629ee3c9eebSPatrick Williams "-t", 630ee3c9eebSPatrick Williams tag, 631ee3c9eebSPatrick Williams "-", 632ee3c9eebSPatrick Williams _in=dockerfile, 633ee3c9eebSPatrick Williams _out=( 634ee3c9eebSPatrick Williams lambda line: print( 635ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 636ee3c9eebSPatrick Williams ) 637ee3c9eebSPatrick Williams ), 638ee3c9eebSPatrick Williams ) 639ee3c9eebSPatrick Williams 640ee3c9eebSPatrick Williams 641ee3c9eebSPatrick Williams# Read a bunch of environment variables. 642ee3c9eebSPatrick Williamsdocker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 643ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 644ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 6458949d3c3SPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:jammy") 646ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 647ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 648ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 649ee3c9eebSPatrick Williams 65065b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 65165b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 65265b21fb9SPatrick Williams 653ee3c9eebSPatrick Williams# Set up some common variables. 654ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 655ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 656ee3c9eebSPatrick Williamsgid = os.getgid() 657ee3c9eebSPatrick Williamsuid = os.getuid() 658ee3c9eebSPatrick Williams 6596825a018SJosh Lehan# Use well-known constants if user is root 6606825a018SJosh Lehanif username == "root": 6616825a018SJosh Lehan homedir = "/root" 6626825a018SJosh Lehan gid = 0 6636825a018SJosh Lehan uid = 0 6646825a018SJosh Lehan 665ee3c9eebSPatrick Williams# Determine the architecture for Docker. 666ee3c9eebSPatrick Williamsarch = uname("-m").strip() 667ee3c9eebSPatrick Williamsif arch == "ppc64le": 668ee3c9eebSPatrick Williams docker_base = "ppc64le/" 669ee3c9eebSPatrick Williamselif arch == "x86_64": 670ee3c9eebSPatrick Williams docker_base = "" 671051b05b7SThang Q. Nguyenelif arch == "aarch64": 672f98f1a8dSThang Q. Nguyen docker_base = "arm64v8/" 673ee3c9eebSPatrick Williamselse: 674ee3c9eebSPatrick Williams print( 675ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 676ee3c9eebSPatrick Williams file=sys.stderr, 677ee3c9eebSPatrick Williams ) 678ee3c9eebSPatrick Williams sys.exit(1) 679ee3c9eebSPatrick Williams 68002871c91SPatrick Williams# Special flags if setting up a deb mirror. 68102871c91SPatrick Williamsmirror = "" 68202871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 68302871c91SPatrick Williams mirror = f""" 68402871c91SPatrick WilliamsRUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 68502871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 68602871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 68702871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 68802871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 68902871c91SPatrick Williams""" 69002871c91SPatrick Williams 69102871c91SPatrick Williams# Special flags for proxying. 69202871c91SPatrick Williamsproxy_cmd = "" 69334ec77e8SAdrian Ambrożewiczproxy_keyserver = "" 69402871c91SPatrick Williamsproxy_args = [] 69502871c91SPatrick Williamsif http_proxy: 69602871c91SPatrick Williams proxy_cmd = f""" 69702871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 69802871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 69902871c91SPatrick Williams""" 70034ec77e8SAdrian Ambrożewicz proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 70134ec77e8SAdrian Ambrożewicz 70202871c91SPatrick Williams proxy_args.extend( 70302871c91SPatrick Williams [ 70402871c91SPatrick Williams "--build-arg", 70502871c91SPatrick Williams f"http_proxy={http_proxy}", 70602871c91SPatrick Williams "--build-arg", 707d461cd6aSLei YU f"https_proxy={http_proxy}", 70802871c91SPatrick Williams ] 70902871c91SPatrick Williams ) 71002871c91SPatrick Williams 711ee3c9eebSPatrick Williams# Create base Dockerfile. 712a18d9c57SPatrick Williamsdockerfile_base = f""" 713a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 71402871c91SPatrick Williams 71502871c91SPatrick Williams{mirror} 71602871c91SPatrick Williams 71702871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 71802871c91SPatrick Williams 7198949d3c3SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/" 72002871c91SPatrick Williams 721bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 722bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 723bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 724bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 725bb16ac14SPatrick Williams# the update. 726bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 727bb16ac14SPatrick Williams 72802871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 72902871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 73002871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 73150837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 732f79ce4c4SPatrick Williams ( apt-get install gpgv ubuntu-dbgsym-keyring || \ 73350837436SPatrick Williams ( apt-get install -yy dirmngr && \ 73450837436SPatrick Williams apt-key adv --keyserver keyserver.ubuntu.com \ 73534ec77e8SAdrian Ambrożewicz {proxy_keyserver} \ 73650837436SPatrick Williams --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 73702871c91SPatrick Williams 73802871c91SPatrick Williams# Parse the current repo list into a debug repo list 73902871c91SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 74002871c91SPatrick Williams 74102871c91SPatrick Williams# Remove non-existent debug repos 74202871c91SPatrick WilliamsRUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 74302871c91SPatrick Williams 74402871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 74502871c91SPatrick Williams 74602871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 747b84d59dcSWilliam A. Kennington III gcc-11 \ 748b84d59dcSWilliam A. Kennington III g++-11 \ 74902871c91SPatrick Williams libc6-dbg \ 75002871c91SPatrick Williams libc6-dev \ 75102871c91SPatrick Williams libtool \ 75202871c91SPatrick Williams bison \ 75302871c91SPatrick Williams libdbus-1-dev \ 75402871c91SPatrick Williams flex \ 75502871c91SPatrick Williams cmake \ 75602871c91SPatrick Williams python3 \ 75702871c91SPatrick Williams python3-dev\ 75802871c91SPatrick Williams python3-yaml \ 75902871c91SPatrick Williams python3-mako \ 76002871c91SPatrick Williams python3-pip \ 76102871c91SPatrick Williams python3-setuptools \ 76202871c91SPatrick Williams python3-git \ 76302871c91SPatrick Williams python3-socks \ 76402871c91SPatrick Williams pkg-config \ 76502871c91SPatrick Williams autoconf \ 76602871c91SPatrick Williams autoconf-archive \ 76702871c91SPatrick Williams libsystemd-dev \ 76802871c91SPatrick Williams systemd \ 76902871c91SPatrick Williams libssl-dev \ 77002871c91SPatrick Williams libevdev-dev \ 77102871c91SPatrick Williams libjpeg-dev \ 77202871c91SPatrick Williams libpng-dev \ 77302871c91SPatrick Williams ninja-build \ 77402871c91SPatrick Williams sudo \ 77502871c91SPatrick Williams curl \ 77602871c91SPatrick Williams git \ 77702871c91SPatrick Williams dbus \ 77802871c91SPatrick Williams iputils-ping \ 7790ff14b2aSJian Zhang clang-14 \ 7800ff14b2aSJian Zhang clang-format-14 \ 7810ff14b2aSJian Zhang clang-tidy-14 \ 7820ff14b2aSJian Zhang clang-tools-14 \ 78302871c91SPatrick Williams shellcheck \ 78402871c91SPatrick Williams npm \ 78502871c91SPatrick Williams iproute2 \ 78602871c91SPatrick Williams libnl-3-dev \ 78702871c91SPatrick Williams libnl-genl-3-dev \ 78802871c91SPatrick Williams libconfig++-dev \ 78902871c91SPatrick Williams libsnmp-dev \ 79002871c91SPatrick Williams valgrind \ 79102871c91SPatrick Williams valgrind-dbg \ 79202871c91SPatrick Williams libpam0g-dev \ 79302871c91SPatrick Williams xxd \ 79402871c91SPatrick Williams libi2c-dev \ 79502871c91SPatrick Williams wget \ 79602871c91SPatrick Williams libldap2-dev \ 79702871c91SPatrick Williams libprotobuf-dev \ 798dafe7529SWilliam A. Kennington III liburing-dev \ 7998949d3c3SPatrick Williams liburing2-dbgsym \ 80002871c91SPatrick Williams libperlio-gzip-perl \ 80102871c91SPatrick Williams libjson-perl \ 80202871c91SPatrick Williams protobuf-compiler \ 80302871c91SPatrick Williams libgpiod-dev \ 80402871c91SPatrick Williams device-tree-compiler \ 80502871c91SPatrick Williams libpciaccess-dev \ 80602871c91SPatrick Williams libmimetic-dev \ 80702871c91SPatrick Williams libxml2-utils \ 8080eedeedaSPatrick Williams libxml-simple-perl \ 8099adf68d6SJohn Wedig rsync \ 8109adf68d6SJohn Wedig libcryptsetup-dev 81102871c91SPatrick Williams 8123ea4cd7eSPatrick Williams# Apply autoconf-archive-v2022.02.11 file ax_cxx_compile_stdcxx for C++20. 8133ea4cd7eSPatrick WilliamsRUN curl "http://git.savannah.gnu.org/gitweb/?p=autoconf-archive.git;a=blob_plain;f=m4/ax_cxx_compile_stdcxx.m4;hb=3311b6bdeff883c6a13952594a9dcb60bce6ba80" \ 8143ea4cd7eSPatrick Williams > /usr/share/aclocal/ax_cxx_compile_stdcxx.m4 8153ea4cd7eSPatrick Williams 81687111bb7SManojkiran EdaRUN npm install -g eslint@latest eslint-plugin-json@latest 81787111bb7SManojkiran Eda 818b84d59dcSWilliam A. Kennington IIIRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 11 \ 819b84d59dcSWilliam A. Kennington III --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ 820b84d59dcSWilliam A. Kennington III --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ 821b84d59dcSWilliam A. Kennington III --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-11 \ 822b84d59dcSWilliam A. Kennington III --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-11 82302871c91SPatrick Williams 8240ff14b2aSJian ZhangRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 1000 \ 8250ff14b2aSJian Zhang --slave /usr/bin/clang++ clang++ /usr/bin/clang++-14 \ 8260ff14b2aSJian Zhang --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-14 \ 8270ff14b2aSJian Zhang --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-14 \ 8280ff14b2aSJian Zhang --slave /usr/bin/run-clang-tidy run-clang-tidy.py /usr/bin/run-clang-tidy-14 \ 8290ff14b2aSJian Zhang --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-14 83002871c91SPatrick Williams 83150837436SPatrick Williams""" 83250837436SPatrick Williams 83350837436SPatrick Williamsif is_automated_ci_build: 83450837436SPatrick Williams dockerfile_base += f""" 83550837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 83650837436SPatrick Williams# to re-run `apt-get update` daily. 837ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 83850837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 83950837436SPatrick Williams 84050837436SPatrick Williams""" 84150837436SPatrick Williams 84250837436SPatrick Williamsdockerfile_base += f""" 84302871c91SPatrick WilliamsRUN pip3 install inflection 84402871c91SPatrick WilliamsRUN pip3 install pycodestyle 84502871c91SPatrick WilliamsRUN pip3 install jsonschema 846e86aa1e7SPatrick WilliamsRUN pip3 install meson==0.61.3 84702871c91SPatrick WilliamsRUN pip3 install protobuf 848e6f120aaSManojkiran EdaRUN pip3 install codespell 849ca8c4a8bSEd TanousRUN pip3 install requests 850a18d9c57SPatrick Williams""" 85102871c91SPatrick Williams 852*fb9948a3SEd Tanous# Note, we use sha1s here because the newest gitlint release doesn't include 853*fb9948a3SEd Tanous# some features we need. Next time they release, we can rely on a direct 854*fb9948a3SEd Tanous# release tag 855*fb9948a3SEd Tanousdockerfile_base += f""" 856*fb9948a3SEd TanousRUN pip3 install git+https://github.com/jorisroovers/gitlint.git@8ede310d62d5794efa7518b235f899f8a8ad6a68\#subdirectory=gitlint-core 857*fb9948a3SEd TanousRUN pip3 install git+https://github.com/jorisroovers/gitlint.git@8ede310d62d5794efa7518b235f899f8a8ad6a68 858*fb9948a3SEd Tanous""" 859*fb9948a3SEd Tanous 860ee3c9eebSPatrick Williams# Build the base and stage docker images. 861ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 862ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 863ee3c9eebSPatrick WilliamsPackage.generate_all() 86402871c91SPatrick Williams 865ee3c9eebSPatrick Williams# Create the final Dockerfile. 866a18d9c57SPatrick Williamsdockerfile = f""" 86702871c91SPatrick Williams# Build the final output image 868a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 869ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 87002871c91SPatrick Williams 87102871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 87202871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 87302871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 874ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 87502871c91SPatrick Williams 87602871c91SPatrick Williams# Final configuration for the workspace 8776825a018SJosh LehanRUN grep -q {gid} /etc/group || groupadd -f -g {gid} {username} 87802871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 87902871c91SPatrick WilliamsRUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 88002871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 88102871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 88202871c91SPatrick Williams 883305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 884305a9a5dSAndrew Geissler# and data installs 8857bb00b13SAndrew GeisslerRUN chown -R {username}:{username} /usr/local/share 886305a9a5dSAndrew Geissler 88702871c91SPatrick Williams{proxy_cmd} 88802871c91SPatrick Williams 88902871c91SPatrick WilliamsRUN /bin/bash 89002871c91SPatrick Williams""" 89102871c91SPatrick Williams 892a18d9c57SPatrick Williams# Do the final docker build 893ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 894ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 895ee3c9eebSPatrick Williams 89600536fbeSPatrick Williams# Print the tag of the final image. 89700536fbeSPatrick Williamsprint(docker_final_img_name) 898