102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1050837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1150837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1250837436SPatrick Williams# (ex. Jenkins)> 1302871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1402871c91SPatrick Williams# repositories> 1502871c91SPatrick Williams# default is master, which will be used if input branch not 1602871c91SPatrick Williams# provided or not found 1702871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1802871c91SPatrick Williams# default ones in /etc/apt/sources.list> 1902871c91SPatrick Williams# default is empty, and no mirror is used. 2002871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2102871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2202871c91SPatrick Williams 2302871c91SPatrick Williamsimport os 2402871c91SPatrick Williamsimport sys 25b16f3e20SPatrick Williamsimport threading 26a18d9c57SPatrick Williamsfrom datetime import date 27a18d9c57SPatrick Williamsfrom hashlib import sha256 28ee3c9eebSPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 29ee3c9eebSPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional 3002871c91SPatrick Williams 31ee3c9eebSPatrick Williamstry: 32ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 33ee3c9eebSPatrick Williams from typing import TypedDict 34ee3c9eebSPatrick Williamsexcept: 35ee3c9eebSPatrick Williams 36ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 37ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 38ee3c9eebSPatrick Williams def __init_subclass__(cls, **kwargs): 39ee3c9eebSPatrick Williams super().__init_subclass__() 40ee3c9eebSPatrick Williams 41ee3c9eebSPatrick Williams 42ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 43aae36d18SPatrick Williamsprefix = "/usr/local" 4402871c91SPatrick Williamsproc_count = nproc().strip() 4502871c91SPatrick Williams 46ee3c9eebSPatrick Williams 47ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 48ee3c9eebSPatrick Williams """ Package Definition for packages dictionary. """ 49ee3c9eebSPatrick Williams 50ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 51ee3c9eebSPatrick Williams rev: str 52ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 53ee3c9eebSPatrick Williams url: Callable[[str, str], str] 54ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 55ee3c9eebSPatrick Williams depends: Iterable[str] 56ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 57ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 58ee3c9eebSPatrick Williams build_type: str 59ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 60ee3c9eebSPatrick Williams build_steps: Iterable[str] 61ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 62ee3c9eebSPatrick Williams config_flags: Iterable[str] 63ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 64ee3c9eebSPatrick Williams config_env: Iterable[str] 65ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 66ee3c9eebSPatrick Williams # before config / build / install. 67ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 686bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 696bce2ca1SPatrick Williams custom_post_install: Iterable[str] 70ee3c9eebSPatrick Williams 71ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 72ee3c9eebSPatrick Williams __tag: str 73ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 74ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 75ee3c9eebSPatrick Williams 7602871c91SPatrick Williams 777204324cSPatrick Williams# Packages to include in image. 787204324cSPatrick Williamspackages = { 79ee3c9eebSPatrick Williams "boost": PackageDef( 80de3ee762SEd Tanous rev="1.77.0", 81ee3c9eebSPatrick Williams url=( 825f2549eaSPatrick Williams lambda pkg, rev: f"https://downloads.yoctoproject.org/mirror/sources/{pkg}_{rev.replace('.', '_')}.tar.bz2" 832abc4a48SPatrick Williams ), 84ee3c9eebSPatrick Williams build_type="custom", 85ee3c9eebSPatrick Williams build_steps=[ 86aae36d18SPatrick Williams f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 87aae36d18SPatrick Williams "./b2", 88aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 89aae36d18SPatrick Williams ], 90ee3c9eebSPatrick Williams ), 91ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 923f8b5294SPatrick Williams rev="3e4d1b84cab4891368d2179a61a7ba06a5693e7f", 93ee3c9eebSPatrick Williams build_type="custom", 94ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 95ee3c9eebSPatrick Williams ), 96ee3c9eebSPatrick Williams "catchorg/Catch2": PackageDef( 9796b43455SWilliam A. Kennington III rev="v2.13.6", 98ee3c9eebSPatrick Williams build_type="cmake", 99ee3c9eebSPatrick Williams config_flags=["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"], 100ee3c9eebSPatrick Williams ), 101ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 102ee3c9eebSPatrick Williams rev="v1.9.1", 103ee3c9eebSPatrick Williams build_type="cmake", 104ee3c9eebSPatrick Williams config_flags=[ 105aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 106aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 107aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 108aae36d18SPatrick Williams ], 109ee3c9eebSPatrick Williams ), 110ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 111ee3c9eebSPatrick Williams rev="7.1.3", 112ee3c9eebSPatrick Williams build_type="cmake", 113ee3c9eebSPatrick Williams config_flags=[ 114aae36d18SPatrick Williams "-DFMT_DOC=OFF", 115aae36d18SPatrick Williams "-DFMT_TEST=OFF", 116aae36d18SPatrick Williams ], 117ee3c9eebSPatrick Williams ), 118ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 11996b43455SWilliam A. Kennington III rev="4.1.0", 120ee3c9eebSPatrick Williams build_type="custom", 121ee3c9eebSPatrick Williams build_steps=[ 122aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 123aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 124aae36d18SPatrick Williams ], 125ee3c9eebSPatrick Williams ), 1264dd32c02SWilliam A. Kennington III # Snapshot from 2021-05-13 127ee3c9eebSPatrick Williams "google/googletest": PackageDef( 1284dd32c02SWilliam A. Kennington III rev="662fe38e44900c007eccb65a5d2ea19df7bd520e", 129ee3c9eebSPatrick Williams build_type="cmake", 1304dd32c02SWilliam A. Kennington III config_env=["CXXFLAGS=-std=c++20"], 131ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 132ee3c9eebSPatrick Williams ), 13302871c91SPatrick Williams # Release 2020-08-06 134ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 1357d1b2a1dSEd Tanous rev="v3.10.4", 1366bce2ca1SPatrick Williams build_type="cmake", 1376bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1386bce2ca1SPatrick Williams custom_post_install=[ 139aae36d18SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 140aae36d18SPatrick Williams ], 141ee3c9eebSPatrick Williams ), 14202871c91SPatrick Williams # Snapshot from 2019-05-24 143ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 144ee3c9eebSPatrick Williams rev="v1.15", 145ee3c9eebSPatrick Williams build_type="make", 146ee3c9eebSPatrick Williams ), 1470eedeedaSPatrick Williams # dev-5.8 2021-01-11 148ee3c9eebSPatrick Williams "openbmc/linux": PackageDef( 149ee3c9eebSPatrick Williams rev="3cc95ae40716e56f81b69615781f54c78079042d", 150ee3c9eebSPatrick Williams build_type="custom", 151ee3c9eebSPatrick Williams build_steps=[ 152aae36d18SPatrick Williams f"make -j{proc_count} defconfig", 153aae36d18SPatrick Williams f"make INSTALL_HDR_PATH={prefix} headers_install", 154aae36d18SPatrick Williams ], 155ee3c9eebSPatrick Williams ), 1560eedeedaSPatrick Williams # Snapshot from 2020-06-13 157ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 158ee3c9eebSPatrick Williams rev="LibVNCServer-0.9.13", 159ee3c9eebSPatrick Williams build_type="cmake", 160ee3c9eebSPatrick Williams ), 161ee3c9eebSPatrick Williams "martinmoene/span-lite": PackageDef( 16296b43455SWilliam A. Kennington III rev="v0.9.2", 163ee3c9eebSPatrick Williams build_type="cmake", 164ee3c9eebSPatrick Williams config_flags=[ 165aae36d18SPatrick Williams "-DSPAN_LITE_OPT_BUILD_TESTS=OFF", 166aae36d18SPatrick Williams ], 167ee3c9eebSPatrick Williams ), 1680eedeedaSPatrick Williams # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_8.0.0.bb 169ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 170ee3c9eebSPatrick Williams rev="8.0.0", 171ee3c9eebSPatrick Williams build_type="cmake", 172ee3c9eebSPatrick Williams ), 17302871c91SPatrick Williams # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 174ee3c9eebSPatrick Williams "CPPAlliance/url": PackageDef( 175eed466e3SEd Tanous rev="4f712ed69a04a344957d22efa5dc111b415b3aff", 176eed466e3SEd Tanous build_type="custom", 177eed466e3SEd Tanous build_steps=[f"cp -a include/** {prefix}/include/"], 178ee3c9eebSPatrick Williams ), 17968992895SPatrick Williams # version from meta-openembedded/meta-oe/dynamic-layers/networking-layer/recipes-devools/valijson/valijson_0.6.bb 180ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 18168992895SPatrick Williams rev="v0.6", 182ee3c9eebSPatrick Williams build_type="cmake", 183ee3c9eebSPatrick Williams config_flags=[ 1840eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1850eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 186aae36d18SPatrick Williams ], 187ee3c9eebSPatrick Williams ), 18802871c91SPatrick Williams # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb 189ee3c9eebSPatrick Williams "nlohmann/fifo_map": PackageDef( 190ee3c9eebSPatrick Williams rev="0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9", 191ee3c9eebSPatrick Williams build_type="custom", 192ee3c9eebSPatrick Williams build_steps=[f"cp src/fifo_map.hpp {prefix}/include/"], 193ee3c9eebSPatrick Williams ), 194*51224161SPatrick Williams # version from meta-openembedded/meta-oe/recipes-devtools/unifex/unifex_git.bb 195*51224161SPatrick Williams "facebookexperimental/libunifex": PackageDef( 196*51224161SPatrick Williams rev="9df21c58d34ce8a1cd3b15c3a7347495e29417a0", 197*51224161SPatrick Williams build_type="cmake", 198*51224161SPatrick Williams config_flags=[ 199*51224161SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 200*51224161SPatrick Williams "-DBUILD_TESTING=OFF", 201*51224161SPatrick Williams "-DCMAKE_CXX_STANDARD=20", 202*51224161SPatrick Williams "-DUNIFEX_BUILD_EXAMPLES=OFF", 203*51224161SPatrick Williams ], 204*51224161SPatrick Williams ), 205ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 206ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 207ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 208ee3c9eebSPatrick Williams build_type="meson", 209ee3c9eebSPatrick Williams config_flags=[ 210aae36d18SPatrick Williams "-Dexamples=false", 211aae36d18SPatrick Williams "-Dtests=disabled", 212aae36d18SPatrick Williams ], 213ee3c9eebSPatrick Williams ), 214ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 215ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 216ee3c9eebSPatrick Williams build_type="meson", 217ee3c9eebSPatrick Williams config_flags=[ 218aae36d18SPatrick Williams "-Ddata_com_ibm=true", 219aae36d18SPatrick Williams "-Ddata_org_open_power=true", 220aae36d18SPatrick Williams ], 221ee3c9eebSPatrick Williams ), 222ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 223ee3c9eebSPatrick Williams depends=[ 22483394610SPatrick Williams "USCiLab/cereal", 22583394610SPatrick Williams "nlohmann/fifo_map", 22683394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 22783394610SPatrick Williams "openbmc/sdbusplus", 22883394610SPatrick Williams "openbmc/sdeventplus", 229aae36d18SPatrick Williams ], 230f79ce4c4SPatrick Williams build_type="meson", 231ee3c9eebSPatrick Williams config_flags=[ 232f79ce4c4SPatrick Williams f"-Dyaml_dir={prefix}/share/phosphor-dbus-yaml/yaml", 233aae36d18SPatrick Williams ], 234ee3c9eebSPatrick Williams ), 235ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 236ee3c9eebSPatrick Williams depends=[ 23783394610SPatrick Williams "boost", 23883394610SPatrick Williams "leethomason/tinyxml2", 23983394610SPatrick Williams "openbmc/phosphor-logging", 24083394610SPatrick Williams "openbmc/sdbusplus", 241aae36d18SPatrick Williams ], 2421197e359SBrad Bishop build_type="meson", 2431197e359SBrad Bishop config_flags=[ 2441197e359SBrad Bishop "-Dtests=disabled", 2451197e359SBrad Bishop ], 246ee3c9eebSPatrick Williams ), 247ee3c9eebSPatrick Williams "openbmc/pldm": PackageDef( 248ee3c9eebSPatrick Williams depends=[ 24983394610SPatrick Williams "CLIUtils/CLI11", 25083394610SPatrick Williams "boost", 25183394610SPatrick Williams "nlohmann/json", 25283394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 25383394610SPatrick Williams "openbmc/phosphor-logging", 25483394610SPatrick Williams "openbmc/sdbusplus", 25583394610SPatrick Williams "openbmc/sdeventplus", 256aae36d18SPatrick Williams ], 257ee3c9eebSPatrick Williams build_type="meson", 258ee3c9eebSPatrick Williams config_flags=[ 259aae36d18SPatrick Williams "-Dlibpldm-only=enabled", 260aae36d18SPatrick Williams "-Doem-ibm=enabled", 261aae36d18SPatrick Williams "-Dtests=disabled", 262aae36d18SPatrick Williams ], 263ee3c9eebSPatrick Williams ), 264ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 265*51224161SPatrick Williams depends=[ 266*51224161SPatrick Williams "facebookexperimental/libunifex", 267*51224161SPatrick Williams ], 268ee3c9eebSPatrick Williams build_type="meson", 269ee3c9eebSPatrick Williams custom_post_dl=[ 270aae36d18SPatrick Williams "cd tools", 271aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 272aae36d18SPatrick Williams "cd ..", 273aae36d18SPatrick Williams ], 274ee3c9eebSPatrick Williams config_flags=[ 275aae36d18SPatrick Williams "-Dexamples=disabled", 276aae36d18SPatrick Williams "-Dtests=disabled", 277aae36d18SPatrick Williams ], 278b16f3e20SPatrick Williams ), 279ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 280ee3c9eebSPatrick Williams depends=["Naios/function2", "openbmc/stdplus"], 281ee3c9eebSPatrick Williams build_type="meson", 282ee3c9eebSPatrick Williams config_flags=[ 283ee3c9eebSPatrick Williams "-Dexamples=false", 284ee3c9eebSPatrick Williams "-Dtests=disabled", 285ee3c9eebSPatrick Williams ], 286ee3c9eebSPatrick Williams ), 287ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 288ee3c9eebSPatrick Williams depends=["fmtlib/fmt", "martinmoene/span-lite"], 289ee3c9eebSPatrick Williams build_type="meson", 290ee3c9eebSPatrick Williams config_flags=[ 291ee3c9eebSPatrick Williams "-Dexamples=false", 292ee3c9eebSPatrick Williams "-Dtests=disabled", 293ee3c9eebSPatrick Williams ], 294ee3c9eebSPatrick Williams ), 295ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 29602871c91SPatrick Williams 29702871c91SPatrick Williams# Define common flags used for builds 29802871c91SPatrick Williamsconfigure_flags = " ".join( 29902871c91SPatrick Williams [ 30002871c91SPatrick Williams f"--prefix={prefix}", 30102871c91SPatrick Williams ] 30202871c91SPatrick Williams) 30302871c91SPatrick Williamscmake_flags = " ".join( 30402871c91SPatrick Williams [ 30502871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 3060f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 30702871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 3080f2086b3SPatrick Williams "-GNinja", 3090f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 31002871c91SPatrick Williams ] 31102871c91SPatrick Williams) 31202871c91SPatrick Williamsmeson_flags = " ".join( 31302871c91SPatrick Williams [ 31402871c91SPatrick Williams "--wrap-mode=nodownload", 31502871c91SPatrick Williams f"-Dprefix={prefix}", 31602871c91SPatrick Williams ] 31702871c91SPatrick Williams) 31802871c91SPatrick Williams 319ee3c9eebSPatrick Williams 320ee3c9eebSPatrick Williamsclass Package(threading.Thread): 321ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 322ee3c9eebSPatrick Williams 323ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 324ee3c9eebSPatrick Williams Package.generate_all(). 325ee3c9eebSPatrick Williams """ 326ee3c9eebSPatrick Williams 327ee3c9eebSPatrick Williams # Copy the packages dictionary. 328ee3c9eebSPatrick Williams packages = packages.copy() 329ee3c9eebSPatrick Williams 330ee3c9eebSPatrick Williams # Lock used for thread-safety. 331ee3c9eebSPatrick Williams lock = threading.Lock() 332ee3c9eebSPatrick Williams 333ee3c9eebSPatrick Williams def __init__(self, pkg: str): 334ee3c9eebSPatrick Williams """ pkg - The name of this package (ex. foo/bar ) """ 335ee3c9eebSPatrick Williams super(Package, self).__init__() 336ee3c9eebSPatrick Williams 337ee3c9eebSPatrick Williams self.package = pkg 338ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 339ee3c9eebSPatrick Williams 340ee3c9eebSPatrick Williams # Reference to this package's 341ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 342ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 343ee3c9eebSPatrick Williams 344ee3c9eebSPatrick Williams def run(self) -> None: 345ee3c9eebSPatrick Williams """ Thread 'run' function. Builds the Docker stage. """ 346ee3c9eebSPatrick Williams 347ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 348ee3c9eebSPatrick Williams self._update_rev() 349ee3c9eebSPatrick Williams 350ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 351ee3c9eebSPatrick Williams # This section is locked because we are looking into another 352ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 353ee3c9eebSPatrick Williams Package.lock.acquire() 354ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 355ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 356ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 357ee3c9eebSPatrick Williams ] 358ee3c9eebSPatrick Williams Package.lock.release() 359ee3c9eebSPatrick Williams 360ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 361ee3c9eebSPatrick Williams # for the "COPY" commands. 362ee3c9eebSPatrick Williams for deppkg in deps: 363ee3c9eebSPatrick Williams deppkg.join() 364ee3c9eebSPatrick Williams 365ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 366ee3c9eebSPatrick Williams dockerfile = f""" 367ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 368ee3c9eebSPatrick Williams{self._df_copycmds()} 369ee3c9eebSPatrick Williams{self._df_build()} 370ee3c9eebSPatrick Williams""" 371ee3c9eebSPatrick Williams 372ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 373ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 374ee3c9eebSPatrick Williams # which can be accessed by other threads. 375ee3c9eebSPatrick Williams Package.lock.acquire() 376ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 377ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 378ee3c9eebSPatrick Williams Package.lock.release() 379ee3c9eebSPatrick Williams 380ee3c9eebSPatrick Williams # Do the build / save any exceptions. 381ee3c9eebSPatrick Williams try: 382ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 383ee3c9eebSPatrick Williams except Exception as e: 384ee3c9eebSPatrick Williams self.exception = e 385ee3c9eebSPatrick Williams 386ee3c9eebSPatrick Williams @classmethod 387ee3c9eebSPatrick Williams def generate_all(cls) -> None: 388ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 389ee3c9eebSPatrick Williams 390ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 391ee3c9eebSPatrick Williams package 'depends' specifications. 392ee3c9eebSPatrick Williams """ 393ee3c9eebSPatrick Williams 394ee3c9eebSPatrick Williams # Create a Package for each defined package. 395ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 396ee3c9eebSPatrick Williams 397ee3c9eebSPatrick Williams # Start building them all. 3986dbd7807SPatrick Williams # This section is locked because threads depend on each other, 3996dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 4006dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 4016dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 4026dbd7807SPatrick Williams Package.lock.acquire() 403ee3c9eebSPatrick Williams for t in pkg_threads: 404ee3c9eebSPatrick Williams t.start() 4056dbd7807SPatrick Williams Package.lock.release() 406ee3c9eebSPatrick Williams 407ee3c9eebSPatrick Williams # Wait for completion. 408ee3c9eebSPatrick Williams for t in pkg_threads: 409ee3c9eebSPatrick Williams t.join() 410ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 411ee3c9eebSPatrick Williams if t.exception: 412ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 413ee3c9eebSPatrick Williams raise t.exception 414ee3c9eebSPatrick Williams 415ee3c9eebSPatrick Williams @staticmethod 416ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 417ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 418ee3c9eebSPatrick Williams into the final image. 419ee3c9eebSPatrick Williams """ 420ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 421ee3c9eebSPatrick Williams 422ee3c9eebSPatrick Williams @classmethod 423ee3c9eebSPatrick Williams def depcache(cls) -> str: 424ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 425ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 426ee3c9eebSPatrick Williams """ 427ee3c9eebSPatrick Williams 428ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 429ee3c9eebSPatrick Williams depcache = "" 430ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 431ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 432ee3c9eebSPatrick Williams return depcache 433ee3c9eebSPatrick Williams 434ee3c9eebSPatrick Williams def _update_rev(self) -> None: 435ee3c9eebSPatrick Williams """ Look up the HEAD for missing a static rev. """ 436ee3c9eebSPatrick Williams 437ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 438ee3c9eebSPatrick Williams return 439ee3c9eebSPatrick Williams 44065b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 44165b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 44265b21fb9SPatrick Williams print( 44365b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 44465b21fb9SPatrick Williams file=sys.stderr, 44565b21fb9SPatrick Williams ) 44665b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 44765b21fb9SPatrick Williams return 44865b21fb9SPatrick Williams 449ee3c9eebSPatrick Williams # Ask Github for all the branches. 450ee3c9eebSPatrick Williams lookup = git("ls-remote", "--heads", f"https://github.com/{self.package}") 451ee3c9eebSPatrick Williams 452ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 453ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 454ee3c9eebSPatrick Williams Package.lock.acquire() 455ee3c9eebSPatrick Williams for line in lookup.split("\n"): 456ee3c9eebSPatrick Williams if f"refs/heads/{branch}" in line: 457ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 458ee3c9eebSPatrick Williams elif f"refs/heads/master" in line and "rev" not in self.pkg_def: 459ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 460ee3c9eebSPatrick Williams Package.lock.release() 461ee3c9eebSPatrick Williams 462ee3c9eebSPatrick Williams def _stagename(self) -> str: 463ee3c9eebSPatrick Williams """ Create a name for the Docker stage associated with this pkg. """ 464ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 465ee3c9eebSPatrick Williams 466ee3c9eebSPatrick Williams def _url(self) -> str: 467ee3c9eebSPatrick Williams """ Get the URL for this package. """ 468ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 469ee3c9eebSPatrick Williams 470ee3c9eebSPatrick Williams # If the lambda exists, call it. 471ee3c9eebSPatrick Williams if "url" in self.pkg_def: 472ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 473ee3c9eebSPatrick Williams 474ee3c9eebSPatrick Williams # Default to the github archive URL. 475ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 476ee3c9eebSPatrick Williams 477ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 478ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 479ee3c9eebSPatrick Williams 480ee3c9eebSPatrick Williams url = self._url() 481ee3c9eebSPatrick Williams if ".tar." not in url: 482ee3c9eebSPatrick Williams raise NotImplementedError( 483ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 484ee3c9eebSPatrick Williams ) 485ee3c9eebSPatrick Williams 486ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 487ee3c9eebSPatrick Williams 488ee3c9eebSPatrick Williams if url.endswith(".bz2"): 489ee3c9eebSPatrick Williams cmd += "j" 490ee3c9eebSPatrick Williams elif url.endswith(".gz"): 491ee3c9eebSPatrick Williams cmd += "z" 492ee3c9eebSPatrick Williams else: 493ee3c9eebSPatrick Williams raise NotImplementedError( 494ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 495ee3c9eebSPatrick Williams ) 496ee3c9eebSPatrick Williams 497ee3c9eebSPatrick Williams return cmd 498ee3c9eebSPatrick Williams 499ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 500ee3c9eebSPatrick Williams """ Formulate the command necessary to 'cd' into the source dir. """ 501ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 502ee3c9eebSPatrick Williams 503ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 504ee3c9eebSPatrick Williams """ Formulate the dockerfile snippet necessary to COPY all depends. """ 505ee3c9eebSPatrick Williams 506ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 507ee3c9eebSPatrick Williams return "" 508ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 509ee3c9eebSPatrick Williams 510ee3c9eebSPatrick Williams @staticmethod 511ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 512ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 513ee3c9eebSPatrick Williams packages into a Docker stage. 514ee3c9eebSPatrick Williams """ 515ee3c9eebSPatrick Williams 516ee3c9eebSPatrick Williams copy_cmds = "" 517ee3c9eebSPatrick Williams 518ee3c9eebSPatrick Williams # Sort the packages for consistency. 519ee3c9eebSPatrick Williams for p in sorted(pkgs): 520ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 521ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 522ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 523ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 524ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 525ee3c9eebSPatrick Williams 526ee3c9eebSPatrick Williams return copy_cmds 527ee3c9eebSPatrick Williams 528ee3c9eebSPatrick Williams def _df_build(self) -> str: 529ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 530ee3c9eebSPatrick Williams install a package into a Docker stage. 531ee3c9eebSPatrick Williams """ 532ee3c9eebSPatrick Williams 533ee3c9eebSPatrick Williams # Download and extract source. 534ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 535ee3c9eebSPatrick Williams 536ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 537ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 538ee3c9eebSPatrick Williams if custom_post_dl: 539ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 540ee3c9eebSPatrick Williams 541ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 542ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 543ee3c9eebSPatrick Williams if build_type == "autoconf": 544ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 545ee3c9eebSPatrick Williams elif build_type == "cmake": 546ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 547ee3c9eebSPatrick Williams elif build_type == "custom": 548ee3c9eebSPatrick Williams result += self._cmd_build_custom() 549ee3c9eebSPatrick Williams elif build_type == "make": 550ee3c9eebSPatrick Williams result += self._cmd_build_make() 551ee3c9eebSPatrick Williams elif build_type == "meson": 552ee3c9eebSPatrick Williams result += self._cmd_build_meson() 553ee3c9eebSPatrick Williams else: 554ee3c9eebSPatrick Williams raise NotImplementedError( 555ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 556ee3c9eebSPatrick Williams ) 557ee3c9eebSPatrick Williams 5586bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5596bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5606bce2ca1SPatrick Williams if custom_post_install: 5616bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5626bce2ca1SPatrick Williams 563ee3c9eebSPatrick Williams return result 564ee3c9eebSPatrick Williams 565ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 566ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 567ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 568ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 569ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 570ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 571ee3c9eebSPatrick Williams return result 572ee3c9eebSPatrick Williams 573ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 574ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 575ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 576ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 577ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 578ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 579ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 580ee3c9eebSPatrick Williams result += "cd .." 581ee3c9eebSPatrick Williams return result 582ee3c9eebSPatrick Williams 583ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 584ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 585ee3c9eebSPatrick Williams 586ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 587ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 588ee3c9eebSPatrick Williams 589ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 590ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 591ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 592ee3c9eebSPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 593ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 594ee3c9eebSPatrick Williams return result 595ee3c9eebSPatrick Williams 596ee3c9eebSPatrick Williams 597ee3c9eebSPatrick Williamsclass Docker: 598ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 599ee3c9eebSPatrick Williams 600ee3c9eebSPatrick Williams @staticmethod 601ee3c9eebSPatrick Williams def timestamp() -> str: 602ee3c9eebSPatrick Williams """ Generate a timestamp for today using the ISO week. """ 603ee3c9eebSPatrick Williams today = date.today().isocalendar() 604ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 605ee3c9eebSPatrick Williams 606ee3c9eebSPatrick Williams @staticmethod 607ee3c9eebSPatrick Williams def tagname(pkgname: str, dockerfile: str) -> str: 608ee3c9eebSPatrick Williams """ Generate a tag name for a package using a hash of the Dockerfile. """ 609ee3c9eebSPatrick Williams result = docker_image_name 610ee3c9eebSPatrick Williams if pkgname: 611ee3c9eebSPatrick Williams result += "-" + pkgname 612ee3c9eebSPatrick Williams 613ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 614ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 615ee3c9eebSPatrick Williams 616ee3c9eebSPatrick Williams return result 617ee3c9eebSPatrick Williams 618ee3c9eebSPatrick Williams @staticmethod 619ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 620ee3c9eebSPatrick Williams """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 621ee3c9eebSPatrick Williams 622ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 623ee3c9eebSPatrick Williams if not force_build: 624ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 625ee3c9eebSPatrick Williams print(f"Image {tag} already exists. Skipping.", file=sys.stderr) 626ee3c9eebSPatrick Williams return 627ee3c9eebSPatrick Williams 628ee3c9eebSPatrick Williams # Build it. 629ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 630ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 631ee3c9eebSPatrick Williams # progress but not polute stdout. Later on we output the final 632ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 633ee3c9eebSPatrick Williams # 634ee3c9eebSPatrick Williams # Other unusual flags: 635ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 636ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 637ee3c9eebSPatrick Williams docker.build( 638ee3c9eebSPatrick Williams proxy_args, 639ee3c9eebSPatrick Williams "--network=host", 640ee3c9eebSPatrick Williams "--force-rm", 641ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 642ee3c9eebSPatrick Williams "-t", 643ee3c9eebSPatrick Williams tag, 644ee3c9eebSPatrick Williams "-", 645ee3c9eebSPatrick Williams _in=dockerfile, 646ee3c9eebSPatrick Williams _out=( 647ee3c9eebSPatrick Williams lambda line: print( 648ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 649ee3c9eebSPatrick Williams ) 650ee3c9eebSPatrick Williams ), 651ee3c9eebSPatrick Williams ) 652ee3c9eebSPatrick Williams 653ee3c9eebSPatrick Williams 654ee3c9eebSPatrick Williams# Read a bunch of environment variables. 655ee3c9eebSPatrick Williamsdocker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 656ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 657ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 6583b6bfe42SPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:impish") 659ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 660ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 661ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 662ee3c9eebSPatrick Williams 66365b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 66465b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 66565b21fb9SPatrick Williams 666ee3c9eebSPatrick Williams# Set up some common variables. 667ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 668ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 669ee3c9eebSPatrick Williamsgid = os.getgid() 670ee3c9eebSPatrick Williamsuid = os.getuid() 671ee3c9eebSPatrick Williams 672ee3c9eebSPatrick Williams# Determine the architecture for Docker. 673ee3c9eebSPatrick Williamsarch = uname("-m").strip() 674ee3c9eebSPatrick Williamsif arch == "ppc64le": 675ee3c9eebSPatrick Williams docker_base = "ppc64le/" 676ee3c9eebSPatrick Williamselif arch == "x86_64": 677ee3c9eebSPatrick Williams docker_base = "" 678ee3c9eebSPatrick Williamselse: 679ee3c9eebSPatrick Williams print( 680ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 681ee3c9eebSPatrick Williams file=sys.stderr, 682ee3c9eebSPatrick Williams ) 683ee3c9eebSPatrick Williams sys.exit(1) 684ee3c9eebSPatrick Williams 68502871c91SPatrick Williams# Special flags if setting up a deb mirror. 68602871c91SPatrick Williamsmirror = "" 68702871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 68802871c91SPatrick Williams mirror = f""" 68902871c91SPatrick WilliamsRUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 69002871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 69102871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 69202871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 69302871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 69402871c91SPatrick Williams""" 69502871c91SPatrick Williams 69602871c91SPatrick Williams# Special flags for proxying. 69702871c91SPatrick Williamsproxy_cmd = "" 69834ec77e8SAdrian Ambrożewiczproxy_keyserver = "" 69902871c91SPatrick Williamsproxy_args = [] 70002871c91SPatrick Williamsif http_proxy: 70102871c91SPatrick Williams proxy_cmd = f""" 70202871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 70302871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 70402871c91SPatrick Williams""" 70534ec77e8SAdrian Ambrożewicz proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 70634ec77e8SAdrian Ambrożewicz 70702871c91SPatrick Williams proxy_args.extend( 70802871c91SPatrick Williams [ 70902871c91SPatrick Williams "--build-arg", 71002871c91SPatrick Williams f"http_proxy={http_proxy}", 71102871c91SPatrick Williams "--build-arg", 712d461cd6aSLei YU f"https_proxy={http_proxy}", 71302871c91SPatrick Williams ] 71402871c91SPatrick Williams ) 71502871c91SPatrick Williams 716ee3c9eebSPatrick Williams# Create base Dockerfile. 717a18d9c57SPatrick Williamsdockerfile_base = f""" 718a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 71902871c91SPatrick Williams 72002871c91SPatrick Williams{mirror} 72102871c91SPatrick Williams 72202871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 72302871c91SPatrick Williams 72402871c91SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/" 72502871c91SPatrick Williams 726bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 727bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 728bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 729bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 730bb16ac14SPatrick Williams# the update. 731bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 732bb16ac14SPatrick Williams 73302871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 73402871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 73502871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 73650837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 737f79ce4c4SPatrick Williams ( apt-get install gpgv ubuntu-dbgsym-keyring || \ 73850837436SPatrick Williams ( apt-get install -yy dirmngr && \ 73950837436SPatrick Williams apt-key adv --keyserver keyserver.ubuntu.com \ 74034ec77e8SAdrian Ambrożewicz {proxy_keyserver} \ 74150837436SPatrick Williams --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 74202871c91SPatrick Williams 74302871c91SPatrick Williams# Parse the current repo list into a debug repo list 74402871c91SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 74502871c91SPatrick Williams 74602871c91SPatrick Williams# Remove non-existent debug repos 74702871c91SPatrick WilliamsRUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 74802871c91SPatrick Williams 74902871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 75002871c91SPatrick Williams 75102871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 752b84d59dcSWilliam A. Kennington III gcc-11 \ 753b84d59dcSWilliam A. Kennington III g++-11 \ 75402871c91SPatrick Williams libc6-dbg \ 75502871c91SPatrick Williams libc6-dev \ 75602871c91SPatrick Williams libtool \ 75702871c91SPatrick Williams bison \ 75802871c91SPatrick Williams libdbus-1-dev \ 75902871c91SPatrick Williams flex \ 76002871c91SPatrick Williams cmake \ 76102871c91SPatrick Williams python3 \ 76202871c91SPatrick Williams python3-dev\ 76302871c91SPatrick Williams python3-yaml \ 76402871c91SPatrick Williams python3-mako \ 76502871c91SPatrick Williams python3-pip \ 76602871c91SPatrick Williams python3-setuptools \ 76702871c91SPatrick Williams python3-git \ 76802871c91SPatrick Williams python3-socks \ 76902871c91SPatrick Williams pkg-config \ 77002871c91SPatrick Williams autoconf \ 77102871c91SPatrick Williams autoconf-archive \ 77202871c91SPatrick Williams libsystemd-dev \ 77302871c91SPatrick Williams systemd \ 77402871c91SPatrick Williams libssl-dev \ 77502871c91SPatrick Williams libevdev-dev \ 77602871c91SPatrick Williams libjpeg-dev \ 77702871c91SPatrick Williams libpng-dev \ 77802871c91SPatrick Williams ninja-build \ 77902871c91SPatrick Williams sudo \ 78002871c91SPatrick Williams curl \ 78102871c91SPatrick Williams git \ 78202871c91SPatrick Williams dbus \ 78302871c91SPatrick Williams iputils-ping \ 784f89c8508SPatrick Williams clang-13 \ 785f89c8508SPatrick Williams clang-format-13 \ 786f89c8508SPatrick Williams clang-tidy-13 \ 787f89c8508SPatrick Williams clang-tools-13 \ 78802871c91SPatrick Williams shellcheck \ 78902871c91SPatrick Williams npm \ 79002871c91SPatrick Williams iproute2 \ 79102871c91SPatrick Williams libnl-3-dev \ 79202871c91SPatrick Williams libnl-genl-3-dev \ 79302871c91SPatrick Williams libconfig++-dev \ 79402871c91SPatrick Williams libsnmp-dev \ 79502871c91SPatrick Williams valgrind \ 79602871c91SPatrick Williams valgrind-dbg \ 79702871c91SPatrick Williams libpam0g-dev \ 79802871c91SPatrick Williams xxd \ 79902871c91SPatrick Williams libi2c-dev \ 80002871c91SPatrick Williams wget \ 80102871c91SPatrick Williams libldap2-dev \ 80202871c91SPatrick Williams libprotobuf-dev \ 803dafe7529SWilliam A. Kennington III liburing-dev \ 804dafe7529SWilliam A. Kennington III liburing1-dbgsym \ 80502871c91SPatrick Williams libperlio-gzip-perl \ 80602871c91SPatrick Williams libjson-perl \ 80702871c91SPatrick Williams protobuf-compiler \ 80802871c91SPatrick Williams libgpiod-dev \ 80902871c91SPatrick Williams device-tree-compiler \ 81002871c91SPatrick Williams cppcheck \ 81102871c91SPatrick Williams libpciaccess-dev \ 81202871c91SPatrick Williams libmimetic-dev \ 81302871c91SPatrick Williams libxml2-utils \ 8140eedeedaSPatrick Williams libxml-simple-perl \ 8159adf68d6SJohn Wedig rsync \ 8169adf68d6SJohn Wedig libcryptsetup-dev 81702871c91SPatrick Williams 81887111bb7SManojkiran EdaRUN npm install -g eslint@latest eslint-plugin-json@latest 81987111bb7SManojkiran Eda 820b84d59dcSWilliam A. Kennington IIIRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 11 \ 821b84d59dcSWilliam A. Kennington III --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ 822b84d59dcSWilliam A. Kennington III --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ 823b84d59dcSWilliam A. Kennington III --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-11 \ 824b84d59dcSWilliam A. Kennington III --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-11 82502871c91SPatrick Williams 826f89c8508SPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-13 1000 \ 827f89c8508SPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-13 \ 828f89c8508SPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-13 \ 829f89c8508SPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-13 \ 830f89c8508SPatrick Williams --slave /usr/bin/run-clang-tidy run-clang-tidy.py /usr/bin/run-clang-tidy-13 \ 831f89c8508SPatrick Williams --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-13 83202871c91SPatrick Williams 83350837436SPatrick Williams""" 83450837436SPatrick Williams 83550837436SPatrick Williamsif is_automated_ci_build: 83650837436SPatrick Williams dockerfile_base += f""" 83750837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 83850837436SPatrick Williams# to re-run `apt-get update` daily. 839ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 84050837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 84150837436SPatrick Williams 84250837436SPatrick Williams""" 84350837436SPatrick Williams 84450837436SPatrick Williamsdockerfile_base += f""" 84502871c91SPatrick WilliamsRUN pip3 install inflection 84602871c91SPatrick WilliamsRUN pip3 install pycodestyle 84702871c91SPatrick WilliamsRUN pip3 install jsonschema 848b3e88fb6SPatrick WilliamsRUN pip3 install meson==0.58.1 84902871c91SPatrick WilliamsRUN pip3 install protobuf 850e6f120aaSManojkiran EdaRUN pip3 install codespell 851a18d9c57SPatrick Williams""" 85202871c91SPatrick Williams 853ee3c9eebSPatrick Williams# Build the base and stage docker images. 854ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 855ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 856ee3c9eebSPatrick WilliamsPackage.generate_all() 85702871c91SPatrick Williams 858ee3c9eebSPatrick Williams# Create the final Dockerfile. 859a18d9c57SPatrick Williamsdockerfile = f""" 86002871c91SPatrick Williams# Build the final output image 861a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 862ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 86302871c91SPatrick Williams 86402871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 86502871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 86602871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 867ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 86802871c91SPatrick Williams 86902871c91SPatrick Williams# Final configuration for the workspace 87002871c91SPatrick WilliamsRUN grep -q {gid} /etc/group || groupadd -g {gid} {username} 87102871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 87202871c91SPatrick WilliamsRUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 87302871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 87402871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 87502871c91SPatrick Williams 876305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 877305a9a5dSAndrew Geissler# and data installs 8787bb00b13SAndrew GeisslerRUN chown -R {username}:{username} /usr/local/share 879305a9a5dSAndrew Geissler 88002871c91SPatrick Williams{proxy_cmd} 88102871c91SPatrick Williams 88202871c91SPatrick WilliamsRUN /bin/bash 88302871c91SPatrick Williams""" 88402871c91SPatrick Williams 885a18d9c57SPatrick Williams# Do the final docker build 886ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 887ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 888ee3c9eebSPatrick Williams 88900536fbeSPatrick Williams# Print the tag of the final image. 89000536fbeSPatrick Williamsprint(docker_final_img_name) 891