102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# default is ubuntu:focal 1050837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1150837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1250837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1350837436SPatrick Williams# (ex. Jenkins)> 1402871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1502871c91SPatrick Williams# repositories> 1602871c91SPatrick Williams# default is master, which will be used if input branch not 1702871c91SPatrick Williams# provided or not found 1802871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1902871c91SPatrick Williams# default ones in /etc/apt/sources.list> 2002871c91SPatrick Williams# default is empty, and no mirror is used. 2102871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2202871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2302871c91SPatrick Williams 2402871c91SPatrick Williamsimport os 2502871c91SPatrick Williamsimport sys 26b16f3e20SPatrick Williamsimport threading 27a18d9c57SPatrick Williamsfrom datetime import date 28a18d9c57SPatrick Williamsfrom hashlib import sha256 29*ee3c9eebSPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 30*ee3c9eebSPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional 3102871c91SPatrick Williams 32*ee3c9eebSPatrick Williamstry: 33*ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 34*ee3c9eebSPatrick Williams from typing import TypedDict 35*ee3c9eebSPatrick Williamsexcept: 36*ee3c9eebSPatrick Williams 37*ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 38*ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 39*ee3c9eebSPatrick Williams def __init_subclass__(cls, **kwargs): 40*ee3c9eebSPatrick Williams super().__init_subclass__() 41*ee3c9eebSPatrick Williams 42*ee3c9eebSPatrick Williams 43*ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 44aae36d18SPatrick Williamsprefix = "/usr/local" 4502871c91SPatrick Williamsproc_count = nproc().strip() 4602871c91SPatrick Williams 47*ee3c9eebSPatrick Williams 48*ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 49*ee3c9eebSPatrick Williams """ Package Definition for packages dictionary. """ 50*ee3c9eebSPatrick Williams 51*ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 52*ee3c9eebSPatrick Williams rev: str 53*ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 54*ee3c9eebSPatrick Williams url: Callable[[str, str], str] 55*ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 56*ee3c9eebSPatrick Williams depends: Iterable[str] 57*ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 58*ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 59*ee3c9eebSPatrick Williams build_type: str 60*ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 61*ee3c9eebSPatrick Williams build_steps: Iterable[str] 62*ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 63*ee3c9eebSPatrick Williams config_flags: Iterable[str] 64*ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 65*ee3c9eebSPatrick Williams config_env: Iterable[str] 66*ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 67*ee3c9eebSPatrick Williams # before config / build / install. 68*ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 69*ee3c9eebSPatrick Williams 70*ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 71*ee3c9eebSPatrick Williams __tag: str 72*ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 73*ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 74*ee3c9eebSPatrick Williams 7502871c91SPatrick Williams 767204324cSPatrick Williams# Packages to include in image. 777204324cSPatrick Williamspackages = { 78*ee3c9eebSPatrick Williams "boost": PackageDef( 79*ee3c9eebSPatrick Williams rev="1.75.0", 80*ee3c9eebSPatrick Williams url=( 812abc4a48SPatrick Williams lambda pkg, rev: f"https://dl.bintray.com/boostorg/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.bz2" 822abc4a48SPatrick Williams ), 83*ee3c9eebSPatrick Williams build_type="custom", 84*ee3c9eebSPatrick Williams build_steps=[ 85aae36d18SPatrick Williams f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 86aae36d18SPatrick Williams "./b2", 87aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 88aae36d18SPatrick Williams ], 89*ee3c9eebSPatrick Williams ), 90*ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 91*ee3c9eebSPatrick Williams rev="v1.3.0", 92*ee3c9eebSPatrick Williams build_type="custom", 93*ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 94*ee3c9eebSPatrick Williams ), 95*ee3c9eebSPatrick Williams "catchorg/Catch2": PackageDef( 96*ee3c9eebSPatrick Williams rev="v2.12.2", 97*ee3c9eebSPatrick Williams build_type="cmake", 98*ee3c9eebSPatrick Williams config_flags=["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"], 99*ee3c9eebSPatrick Williams ), 100*ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 101*ee3c9eebSPatrick Williams rev="v1.9.1", 102*ee3c9eebSPatrick Williams build_type="cmake", 103*ee3c9eebSPatrick Williams config_flags=[ 104aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 105aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 106aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 107aae36d18SPatrick Williams ], 108*ee3c9eebSPatrick Williams ), 109*ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 110*ee3c9eebSPatrick Williams rev="7.1.3", 111*ee3c9eebSPatrick Williams build_type="cmake", 112*ee3c9eebSPatrick Williams config_flags=[ 113aae36d18SPatrick Williams "-DFMT_DOC=OFF", 114aae36d18SPatrick Williams "-DFMT_TEST=OFF", 115aae36d18SPatrick Williams ], 116*ee3c9eebSPatrick Williams ), 11702871c91SPatrick Williams # Snapshot from 2020-01-03 118*ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 119*ee3c9eebSPatrick Williams rev="3a0746bf5f601dfed05330aefcb6854354fce07d", 120*ee3c9eebSPatrick Williams build_type="custom", 121*ee3c9eebSPatrick Williams build_steps=[ 122aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 123aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 124aae36d18SPatrick Williams ], 125*ee3c9eebSPatrick Williams ), 126*ee3c9eebSPatrick Williams "google/googletest": PackageDef( 127*ee3c9eebSPatrick Williams rev="release-1.10.0", 128*ee3c9eebSPatrick Williams build_type="cmake", 129*ee3c9eebSPatrick Williams config_env=["CXXFLAGS=-std=c++17"], 130*ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 131*ee3c9eebSPatrick Williams ), 13202871c91SPatrick Williams # Release 2020-08-06 133*ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 134*ee3c9eebSPatrick Williams rev="v3.9.1", 135*ee3c9eebSPatrick Williams build_type="custom", 136*ee3c9eebSPatrick Williams build_steps=[ 137aae36d18SPatrick Williams f"mkdir {prefix}/include/nlohmann", 1380eedeedaSPatrick Williams f"cp single_include/nlohmann/json.hpp {prefix}/include/nlohmann", 139aae36d18SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 140aae36d18SPatrick Williams ], 141*ee3c9eebSPatrick Williams ), 14202871c91SPatrick Williams # Snapshot from 2019-05-24 143*ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 144*ee3c9eebSPatrick Williams rev="v1.15", 145*ee3c9eebSPatrick Williams build_type="make", 146*ee3c9eebSPatrick Williams ), 1470eedeedaSPatrick Williams # dev-5.8 2021-01-11 148*ee3c9eebSPatrick Williams "openbmc/linux": PackageDef( 149*ee3c9eebSPatrick Williams rev="3cc95ae40716e56f81b69615781f54c78079042d", 150*ee3c9eebSPatrick Williams build_type="custom", 151*ee3c9eebSPatrick Williams build_steps=[ 152aae36d18SPatrick Williams f"make -j{proc_count} defconfig", 153aae36d18SPatrick Williams f"make INSTALL_HDR_PATH={prefix} headers_install", 154aae36d18SPatrick Williams ], 155*ee3c9eebSPatrick Williams ), 1560eedeedaSPatrick Williams # Snapshot from 2020-06-13 157*ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 158*ee3c9eebSPatrick Williams rev="LibVNCServer-0.9.13", 159*ee3c9eebSPatrick Williams build_type="cmake", 160*ee3c9eebSPatrick Williams ), 161*ee3c9eebSPatrick Williams "martinmoene/span-lite": PackageDef( 162*ee3c9eebSPatrick Williams rev="v0.8.1", 163*ee3c9eebSPatrick Williams build_type="cmake", 164*ee3c9eebSPatrick Williams config_flags=[ 165aae36d18SPatrick Williams "-DSPAN_LITE_OPT_BUILD_TESTS=OFF", 166aae36d18SPatrick Williams ], 167*ee3c9eebSPatrick Williams ), 1680eedeedaSPatrick Williams # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_8.0.0.bb 169*ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 170*ee3c9eebSPatrick Williams rev="8.0.0", 171*ee3c9eebSPatrick Williams build_type="cmake", 172*ee3c9eebSPatrick Williams ), 17302871c91SPatrick Williams # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 174*ee3c9eebSPatrick Williams "CPPAlliance/url": PackageDef( 175*ee3c9eebSPatrick Williams rev="a56ae0df6d3078319755fbaa67822b4fa7fd352b", 176*ee3c9eebSPatrick Williams build_type="cmake", 177*ee3c9eebSPatrick Williams config_flags=[ 178aae36d18SPatrick Williams "-DBOOST_URL_BUILD_EXAMPLES=OFF", 179aae36d18SPatrick Williams "-DBOOST_URL_BUILD_TESTS=OFF", 180aae36d18SPatrick Williams "-DBOOST_URL_STANDALONE=ON", 181aae36d18SPatrick Williams ], 182*ee3c9eebSPatrick Williams ), 1830eedeedaSPatrick Williams # version from ./meta-openembedded/meta-oe/dynamic-layers/networking-layer/recipes-devtools/valijson/valijson_0.3.bb 1840eedeedaSPatrick Williams # Snapshot from 2020-12-02 - fix for curlpp dependency 185*ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 186*ee3c9eebSPatrick Williams rev="8cc83c8be9c1c927f5da952b2333b30e5f0353be", 187*ee3c9eebSPatrick Williams build_type="cmake", 188*ee3c9eebSPatrick Williams config_flags=[ 1890eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1900eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 191aae36d18SPatrick Williams ], 192*ee3c9eebSPatrick Williams ), 19302871c91SPatrick Williams # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb 194*ee3c9eebSPatrick Williams "nlohmann/fifo_map": PackageDef( 195*ee3c9eebSPatrick Williams rev="0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9", 196*ee3c9eebSPatrick Williams build_type="custom", 197*ee3c9eebSPatrick Williams build_steps=[f"cp src/fifo_map.hpp {prefix}/include/"], 198*ee3c9eebSPatrick Williams ), 199*ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 200*ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 201*ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 202*ee3c9eebSPatrick Williams build_type="meson", 203*ee3c9eebSPatrick Williams config_flags=[ 204aae36d18SPatrick Williams "-Dexamples=false", 205aae36d18SPatrick Williams "-Dtests=disabled", 206aae36d18SPatrick Williams ], 207*ee3c9eebSPatrick Williams ), 208*ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 209*ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 210*ee3c9eebSPatrick Williams build_type="meson", 211*ee3c9eebSPatrick Williams config_flags=[ 212aae36d18SPatrick Williams "-Ddata_com_ibm=true", 213aae36d18SPatrick Williams "-Ddata_org_open_power=true", 214aae36d18SPatrick Williams ], 215*ee3c9eebSPatrick Williams ), 216*ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 217*ee3c9eebSPatrick Williams depends=[ 21883394610SPatrick Williams "USCiLab/cereal", 21983394610SPatrick Williams "nlohmann/fifo_map", 22083394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 22183394610SPatrick Williams "openbmc/sdbusplus", 22283394610SPatrick Williams "openbmc/sdeventplus", 223aae36d18SPatrick Williams ], 224*ee3c9eebSPatrick Williams build_type="autoconf", 225*ee3c9eebSPatrick Williams config_flags=[ 226aae36d18SPatrick Williams "--enable-metadata-processing", 227aae36d18SPatrick Williams f"YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml", 228aae36d18SPatrick Williams ], 229*ee3c9eebSPatrick Williams ), 230*ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 231*ee3c9eebSPatrick Williams depends=[ 23283394610SPatrick Williams "boost", 23383394610SPatrick Williams "leethomason/tinyxml2", 23483394610SPatrick Williams "openbmc/phosphor-logging", 23583394610SPatrick Williams "openbmc/sdbusplus", 236aae36d18SPatrick Williams ], 237*ee3c9eebSPatrick Williams build_type="autoconf", 238*ee3c9eebSPatrick Williams ), 239*ee3c9eebSPatrick Williams "openbmc/pldm": PackageDef( 240*ee3c9eebSPatrick Williams depends=[ 24183394610SPatrick Williams "CLIUtils/CLI11", 24283394610SPatrick Williams "boost", 24383394610SPatrick Williams "nlohmann/json", 24483394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 24583394610SPatrick Williams "openbmc/phosphor-logging", 24683394610SPatrick Williams "openbmc/sdbusplus", 24783394610SPatrick Williams "openbmc/sdeventplus", 248aae36d18SPatrick Williams ], 249*ee3c9eebSPatrick Williams build_type="meson", 250*ee3c9eebSPatrick Williams config_flags=[ 251aae36d18SPatrick Williams "-Dlibpldm-only=enabled", 252aae36d18SPatrick Williams "-Doem-ibm=enabled", 253aae36d18SPatrick Williams "-Dtests=disabled", 254aae36d18SPatrick Williams ], 255*ee3c9eebSPatrick Williams ), 256*ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 257*ee3c9eebSPatrick Williams build_type="meson", 258*ee3c9eebSPatrick Williams custom_post_dl=[ 259aae36d18SPatrick Williams "cd tools", 260aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 261aae36d18SPatrick Williams "cd ..", 262aae36d18SPatrick Williams ], 263*ee3c9eebSPatrick Williams config_flags=[ 264aae36d18SPatrick Williams "-Dexamples=disabled", 265aae36d18SPatrick Williams "-Dtests=disabled", 266aae36d18SPatrick Williams ], 267b16f3e20SPatrick Williams ), 268*ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 269*ee3c9eebSPatrick Williams depends=["Naios/function2", "openbmc/stdplus"], 270*ee3c9eebSPatrick Williams build_type="meson", 271*ee3c9eebSPatrick Williams config_flags=[ 272*ee3c9eebSPatrick Williams "-Dexamples=false", 273*ee3c9eebSPatrick Williams "-Dtests=disabled", 274*ee3c9eebSPatrick Williams ], 275*ee3c9eebSPatrick Williams ), 276*ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 277*ee3c9eebSPatrick Williams depends=["fmtlib/fmt", "martinmoene/span-lite"], 278*ee3c9eebSPatrick Williams build_type="meson", 279*ee3c9eebSPatrick Williams config_flags=[ 280*ee3c9eebSPatrick Williams "-Dexamples=false", 281*ee3c9eebSPatrick Williams "-Dtests=disabled", 282*ee3c9eebSPatrick Williams ], 283*ee3c9eebSPatrick Williams ), 284*ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 28502871c91SPatrick Williams 28602871c91SPatrick Williams# Define common flags used for builds 28702871c91SPatrick Williamsconfigure_flags = " ".join( 28802871c91SPatrick Williams [ 28902871c91SPatrick Williams f"--prefix={prefix}", 29002871c91SPatrick Williams ] 29102871c91SPatrick Williams) 29202871c91SPatrick Williamscmake_flags = " ".join( 29302871c91SPatrick Williams [ 29402871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 2950f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 29602871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 2970f2086b3SPatrick Williams "-GNinja", 2980f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 29902871c91SPatrick Williams ] 30002871c91SPatrick Williams) 30102871c91SPatrick Williamsmeson_flags = " ".join( 30202871c91SPatrick Williams [ 30302871c91SPatrick Williams "--wrap-mode=nodownload", 30402871c91SPatrick Williams f"-Dprefix={prefix}", 30502871c91SPatrick Williams ] 30602871c91SPatrick Williams) 30702871c91SPatrick Williams 308*ee3c9eebSPatrick Williams 309*ee3c9eebSPatrick Williamsclass Package(threading.Thread): 310*ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 311*ee3c9eebSPatrick Williams 312*ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 313*ee3c9eebSPatrick Williams Package.generate_all(). 314*ee3c9eebSPatrick Williams """ 315*ee3c9eebSPatrick Williams 316*ee3c9eebSPatrick Williams # Copy the packages dictionary. 317*ee3c9eebSPatrick Williams packages = packages.copy() 318*ee3c9eebSPatrick Williams 319*ee3c9eebSPatrick Williams # Lock used for thread-safety. 320*ee3c9eebSPatrick Williams lock = threading.Lock() 321*ee3c9eebSPatrick Williams 322*ee3c9eebSPatrick Williams def __init__(self, pkg: str): 323*ee3c9eebSPatrick Williams """ pkg - The name of this package (ex. foo/bar ) """ 324*ee3c9eebSPatrick Williams super(Package, self).__init__() 325*ee3c9eebSPatrick Williams 326*ee3c9eebSPatrick Williams self.package = pkg 327*ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 328*ee3c9eebSPatrick Williams 329*ee3c9eebSPatrick Williams # Reference to this package's 330*ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 331*ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 332*ee3c9eebSPatrick Williams 333*ee3c9eebSPatrick Williams def run(self) -> None: 334*ee3c9eebSPatrick Williams """ Thread 'run' function. Builds the Docker stage. """ 335*ee3c9eebSPatrick Williams 336*ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 337*ee3c9eebSPatrick Williams self._update_rev() 338*ee3c9eebSPatrick Williams 339*ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 340*ee3c9eebSPatrick Williams # This section is locked because we are looking into another 341*ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 342*ee3c9eebSPatrick Williams Package.lock.acquire() 343*ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 344*ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 345*ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 346*ee3c9eebSPatrick Williams ] 347*ee3c9eebSPatrick Williams Package.lock.release() 348*ee3c9eebSPatrick Williams 349*ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 350*ee3c9eebSPatrick Williams # for the "COPY" commands. 351*ee3c9eebSPatrick Williams for deppkg in deps: 352*ee3c9eebSPatrick Williams deppkg.join() 353*ee3c9eebSPatrick Williams 354*ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 355*ee3c9eebSPatrick Williams dockerfile = f""" 356*ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 357*ee3c9eebSPatrick Williams{self._df_copycmds()} 358*ee3c9eebSPatrick Williams{self._df_build()} 359*ee3c9eebSPatrick Williams""" 360*ee3c9eebSPatrick Williams 361*ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 362*ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 363*ee3c9eebSPatrick Williams # which can be accessed by other threads. 364*ee3c9eebSPatrick Williams Package.lock.acquire() 365*ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 366*ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 367*ee3c9eebSPatrick Williams Package.lock.release() 368*ee3c9eebSPatrick Williams 369*ee3c9eebSPatrick Williams # Do the build / save any exceptions. 370*ee3c9eebSPatrick Williams try: 371*ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 372*ee3c9eebSPatrick Williams except Exception as e: 373*ee3c9eebSPatrick Williams self.exception = e 374*ee3c9eebSPatrick Williams 375*ee3c9eebSPatrick Williams @classmethod 376*ee3c9eebSPatrick Williams def generate_all(cls) -> None: 377*ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 378*ee3c9eebSPatrick Williams 379*ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 380*ee3c9eebSPatrick Williams package 'depends' specifications. 381*ee3c9eebSPatrick Williams """ 382*ee3c9eebSPatrick Williams 383*ee3c9eebSPatrick Williams # Create a Package for each defined package. 384*ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 385*ee3c9eebSPatrick Williams 386*ee3c9eebSPatrick Williams # Start building them all. 387*ee3c9eebSPatrick Williams for t in pkg_threads: 388*ee3c9eebSPatrick Williams t.start() 389*ee3c9eebSPatrick Williams 390*ee3c9eebSPatrick Williams # Wait for completion. 391*ee3c9eebSPatrick Williams for t in pkg_threads: 392*ee3c9eebSPatrick Williams t.join() 393*ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 394*ee3c9eebSPatrick Williams if t.exception: 395*ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 396*ee3c9eebSPatrick Williams raise t.exception 397*ee3c9eebSPatrick Williams 398*ee3c9eebSPatrick Williams @staticmethod 399*ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 400*ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 401*ee3c9eebSPatrick Williams into the final image. 402*ee3c9eebSPatrick Williams """ 403*ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 404*ee3c9eebSPatrick Williams 405*ee3c9eebSPatrick Williams @classmethod 406*ee3c9eebSPatrick Williams def depcache(cls) -> str: 407*ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 408*ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 409*ee3c9eebSPatrick Williams """ 410*ee3c9eebSPatrick Williams 411*ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 412*ee3c9eebSPatrick Williams depcache = "" 413*ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 414*ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 415*ee3c9eebSPatrick Williams return depcache 416*ee3c9eebSPatrick Williams 417*ee3c9eebSPatrick Williams def _update_rev(self) -> None: 418*ee3c9eebSPatrick Williams """ Look up the HEAD for missing a static rev. """ 419*ee3c9eebSPatrick Williams 420*ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 421*ee3c9eebSPatrick Williams return 422*ee3c9eebSPatrick Williams 423*ee3c9eebSPatrick Williams # Ask Github for all the branches. 424*ee3c9eebSPatrick Williams lookup = git("ls-remote", "--heads", f"https://github.com/{self.package}") 425*ee3c9eebSPatrick Williams 426*ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 427*ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 428*ee3c9eebSPatrick Williams Package.lock.acquire() 429*ee3c9eebSPatrick Williams for line in lookup.split("\n"): 430*ee3c9eebSPatrick Williams if f"refs/heads/{branch}" in line: 431*ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 432*ee3c9eebSPatrick Williams elif f"refs/heads/master" in line and "rev" not in self.pkg_def: 433*ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 434*ee3c9eebSPatrick Williams Package.lock.release() 435*ee3c9eebSPatrick Williams 436*ee3c9eebSPatrick Williams def _stagename(self) -> str: 437*ee3c9eebSPatrick Williams """ Create a name for the Docker stage associated with this pkg. """ 438*ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 439*ee3c9eebSPatrick Williams 440*ee3c9eebSPatrick Williams def _url(self) -> str: 441*ee3c9eebSPatrick Williams """ Get the URL for this package. """ 442*ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 443*ee3c9eebSPatrick Williams 444*ee3c9eebSPatrick Williams # If the lambda exists, call it. 445*ee3c9eebSPatrick Williams if "url" in self.pkg_def: 446*ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 447*ee3c9eebSPatrick Williams 448*ee3c9eebSPatrick Williams # Default to the github archive URL. 449*ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 450*ee3c9eebSPatrick Williams 451*ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 452*ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 453*ee3c9eebSPatrick Williams 454*ee3c9eebSPatrick Williams url = self._url() 455*ee3c9eebSPatrick Williams if ".tar." not in url: 456*ee3c9eebSPatrick Williams raise NotImplementedError( 457*ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 458*ee3c9eebSPatrick Williams ) 459*ee3c9eebSPatrick Williams 460*ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 461*ee3c9eebSPatrick Williams 462*ee3c9eebSPatrick Williams if url.endswith(".bz2"): 463*ee3c9eebSPatrick Williams cmd += "j" 464*ee3c9eebSPatrick Williams elif url.endswith(".gz"): 465*ee3c9eebSPatrick Williams cmd += "z" 466*ee3c9eebSPatrick Williams else: 467*ee3c9eebSPatrick Williams raise NotImplementedError( 468*ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 469*ee3c9eebSPatrick Williams ) 470*ee3c9eebSPatrick Williams 471*ee3c9eebSPatrick Williams return cmd 472*ee3c9eebSPatrick Williams 473*ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 474*ee3c9eebSPatrick Williams """ Formulate the command necessary to 'cd' into the source dir. """ 475*ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 476*ee3c9eebSPatrick Williams 477*ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 478*ee3c9eebSPatrick Williams """ Formulate the dockerfile snippet necessary to COPY all depends. """ 479*ee3c9eebSPatrick Williams 480*ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 481*ee3c9eebSPatrick Williams return "" 482*ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 483*ee3c9eebSPatrick Williams 484*ee3c9eebSPatrick Williams @staticmethod 485*ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 486*ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 487*ee3c9eebSPatrick Williams packages into a Docker stage. 488*ee3c9eebSPatrick Williams """ 489*ee3c9eebSPatrick Williams 490*ee3c9eebSPatrick Williams copy_cmds = "" 491*ee3c9eebSPatrick Williams 492*ee3c9eebSPatrick Williams # Sort the packages for consistency. 493*ee3c9eebSPatrick Williams for p in sorted(pkgs): 494*ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 495*ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 496*ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 497*ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 498*ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 499*ee3c9eebSPatrick Williams 500*ee3c9eebSPatrick Williams return copy_cmds 501*ee3c9eebSPatrick Williams 502*ee3c9eebSPatrick Williams def _df_build(self) -> str: 503*ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 504*ee3c9eebSPatrick Williams install a package into a Docker stage. 505*ee3c9eebSPatrick Williams """ 506*ee3c9eebSPatrick Williams 507*ee3c9eebSPatrick Williams # Download and extract source. 508*ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 509*ee3c9eebSPatrick Williams 510*ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 511*ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 512*ee3c9eebSPatrick Williams if custom_post_dl: 513*ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 514*ee3c9eebSPatrick Williams 515*ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 516*ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 517*ee3c9eebSPatrick Williams if build_type == "autoconf": 518*ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 519*ee3c9eebSPatrick Williams elif build_type == "cmake": 520*ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 521*ee3c9eebSPatrick Williams elif build_type == "custom": 522*ee3c9eebSPatrick Williams result += self._cmd_build_custom() 523*ee3c9eebSPatrick Williams elif build_type == "make": 524*ee3c9eebSPatrick Williams result += self._cmd_build_make() 525*ee3c9eebSPatrick Williams elif build_type == "meson": 526*ee3c9eebSPatrick Williams result += self._cmd_build_meson() 527*ee3c9eebSPatrick Williams else: 528*ee3c9eebSPatrick Williams raise NotImplementedError( 529*ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 530*ee3c9eebSPatrick Williams ) 531*ee3c9eebSPatrick Williams 532*ee3c9eebSPatrick Williams return result 533*ee3c9eebSPatrick Williams 534*ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 535*ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 536*ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 537*ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 538*ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 539*ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 540*ee3c9eebSPatrick Williams return result 541*ee3c9eebSPatrick Williams 542*ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 543*ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 544*ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 545*ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 546*ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 547*ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 548*ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 549*ee3c9eebSPatrick Williams result += "cd .." 550*ee3c9eebSPatrick Williams return result 551*ee3c9eebSPatrick Williams 552*ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 553*ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 554*ee3c9eebSPatrick Williams 555*ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 556*ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 557*ee3c9eebSPatrick Williams 558*ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 559*ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 560*ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 561*ee3c9eebSPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 562*ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 563*ee3c9eebSPatrick Williams return result 564*ee3c9eebSPatrick Williams 565*ee3c9eebSPatrick Williams 566*ee3c9eebSPatrick Williamsclass Docker: 567*ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 568*ee3c9eebSPatrick Williams 569*ee3c9eebSPatrick Williams @staticmethod 570*ee3c9eebSPatrick Williams def timestamp() -> str: 571*ee3c9eebSPatrick Williams """ Generate a timestamp for today using the ISO week. """ 572*ee3c9eebSPatrick Williams today = date.today().isocalendar() 573*ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 574*ee3c9eebSPatrick Williams 575*ee3c9eebSPatrick Williams @staticmethod 576*ee3c9eebSPatrick Williams def tagname(pkgname: str, dockerfile: str) -> str: 577*ee3c9eebSPatrick Williams """ Generate a tag name for a package using a hash of the Dockerfile. """ 578*ee3c9eebSPatrick Williams result = docker_image_name 579*ee3c9eebSPatrick Williams if pkgname: 580*ee3c9eebSPatrick Williams result += "-" + pkgname 581*ee3c9eebSPatrick Williams 582*ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 583*ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 584*ee3c9eebSPatrick Williams 585*ee3c9eebSPatrick Williams return result 586*ee3c9eebSPatrick Williams 587*ee3c9eebSPatrick Williams @staticmethod 588*ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 589*ee3c9eebSPatrick Williams """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 590*ee3c9eebSPatrick Williams 591*ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 592*ee3c9eebSPatrick Williams if not force_build: 593*ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 594*ee3c9eebSPatrick Williams print(f"Image {tag} already exists. Skipping.", file=sys.stderr) 595*ee3c9eebSPatrick Williams return 596*ee3c9eebSPatrick Williams 597*ee3c9eebSPatrick Williams # Build it. 598*ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 599*ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 600*ee3c9eebSPatrick Williams # progress but not polute stdout. Later on we output the final 601*ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 602*ee3c9eebSPatrick Williams # 603*ee3c9eebSPatrick Williams # Other unusual flags: 604*ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 605*ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 606*ee3c9eebSPatrick Williams docker.build( 607*ee3c9eebSPatrick Williams proxy_args, 608*ee3c9eebSPatrick Williams "--network=host", 609*ee3c9eebSPatrick Williams "--force-rm", 610*ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 611*ee3c9eebSPatrick Williams "-t", 612*ee3c9eebSPatrick Williams tag, 613*ee3c9eebSPatrick Williams "-", 614*ee3c9eebSPatrick Williams _in=dockerfile, 615*ee3c9eebSPatrick Williams _out=( 616*ee3c9eebSPatrick Williams lambda line: print( 617*ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 618*ee3c9eebSPatrick Williams ) 619*ee3c9eebSPatrick Williams ), 620*ee3c9eebSPatrick Williams ) 621*ee3c9eebSPatrick Williams 622*ee3c9eebSPatrick Williams 623*ee3c9eebSPatrick Williams# Read a bunch of environment variables. 624*ee3c9eebSPatrick Williamsdocker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 625*ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 626*ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 627*ee3c9eebSPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:focal") 628*ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 629*ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 630*ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 631*ee3c9eebSPatrick Williams 632*ee3c9eebSPatrick Williams# Set up some common variables. 633*ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 634*ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 635*ee3c9eebSPatrick Williamsgid = os.getgid() 636*ee3c9eebSPatrick Williamsuid = os.getuid() 637*ee3c9eebSPatrick Williams 638*ee3c9eebSPatrick Williams# Determine the architecture for Docker. 639*ee3c9eebSPatrick Williamsarch = uname("-m").strip() 640*ee3c9eebSPatrick Williamsif arch == "ppc64le": 641*ee3c9eebSPatrick Williams docker_base = "ppc64le/" 642*ee3c9eebSPatrick Williamselif arch == "x86_64": 643*ee3c9eebSPatrick Williams docker_base = "" 644*ee3c9eebSPatrick Williamselse: 645*ee3c9eebSPatrick Williams print( 646*ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 647*ee3c9eebSPatrick Williams file=sys.stderr, 648*ee3c9eebSPatrick Williams ) 649*ee3c9eebSPatrick Williams sys.exit(1) 650*ee3c9eebSPatrick Williams 65102871c91SPatrick Williams# Special flags if setting up a deb mirror. 65202871c91SPatrick Williamsmirror = "" 65302871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 65402871c91SPatrick Williams mirror = f""" 65502871c91SPatrick WilliamsRUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 65602871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 65702871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 65802871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 65902871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 66002871c91SPatrick Williams""" 66102871c91SPatrick Williams 66202871c91SPatrick Williams# Special flags for proxying. 66302871c91SPatrick Williamsproxy_cmd = "" 66402871c91SPatrick Williamsproxy_args = [] 66502871c91SPatrick Williamsif http_proxy: 66602871c91SPatrick Williams proxy_cmd = f""" 66702871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 66802871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 66902871c91SPatrick Williams""" 67002871c91SPatrick Williams proxy_args.extend( 67102871c91SPatrick Williams [ 67202871c91SPatrick Williams "--build-arg", 67302871c91SPatrick Williams f"http_proxy={http_proxy}", 67402871c91SPatrick Williams "--build-arg", 67502871c91SPatrick Williams "https_proxy={https_proxy}", 67602871c91SPatrick Williams ] 67702871c91SPatrick Williams ) 67802871c91SPatrick Williams 679*ee3c9eebSPatrick Williams# Create base Dockerfile. 680a18d9c57SPatrick Williamsdockerfile_base = f""" 681a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 68202871c91SPatrick Williams 68302871c91SPatrick Williams{mirror} 68402871c91SPatrick Williams 68502871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 68602871c91SPatrick Williams 68702871c91SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/" 68802871c91SPatrick Williams 68902871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 69002871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 69102871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 69250837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 69350837436SPatrick Williams ( apt-get install ubuntu-dbgsym-keyring || \ 69450837436SPatrick Williams ( apt-get install -yy dirmngr && \ 69550837436SPatrick Williams apt-key adv --keyserver keyserver.ubuntu.com \ 69650837436SPatrick Williams --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 69702871c91SPatrick Williams 69802871c91SPatrick Williams# Parse the current repo list into a debug repo list 69902871c91SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 70002871c91SPatrick Williams 70102871c91SPatrick Williams# Remove non-existent debug repos 70202871c91SPatrick WilliamsRUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 70302871c91SPatrick Williams 70402871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 70502871c91SPatrick Williams 70602871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 70702871c91SPatrick Williams gcc-10 \ 70802871c91SPatrick Williams g++-10 \ 70902871c91SPatrick Williams libc6-dbg \ 71002871c91SPatrick Williams libc6-dev \ 71102871c91SPatrick Williams libtool \ 71202871c91SPatrick Williams bison \ 71302871c91SPatrick Williams libdbus-1-dev \ 71402871c91SPatrick Williams flex \ 71502871c91SPatrick Williams cmake \ 71602871c91SPatrick Williams python3 \ 71702871c91SPatrick Williams python3-dev\ 71802871c91SPatrick Williams python3-yaml \ 71902871c91SPatrick Williams python3-mako \ 72002871c91SPatrick Williams python3-pip \ 72102871c91SPatrick Williams python3-setuptools \ 72202871c91SPatrick Williams python3-git \ 72302871c91SPatrick Williams python3-socks \ 72402871c91SPatrick Williams pkg-config \ 72502871c91SPatrick Williams autoconf \ 72602871c91SPatrick Williams autoconf-archive \ 72702871c91SPatrick Williams libsystemd-dev \ 72802871c91SPatrick Williams systemd \ 72902871c91SPatrick Williams libssl-dev \ 73002871c91SPatrick Williams libevdev-dev \ 73102871c91SPatrick Williams libevdev2-dbgsym \ 73202871c91SPatrick Williams libjpeg-dev \ 73302871c91SPatrick Williams libpng-dev \ 73402871c91SPatrick Williams ninja-build \ 73502871c91SPatrick Williams sudo \ 73602871c91SPatrick Williams curl \ 73702871c91SPatrick Williams git \ 73802871c91SPatrick Williams dbus \ 73902871c91SPatrick Williams iputils-ping \ 74002871c91SPatrick Williams clang-10 \ 74102871c91SPatrick Williams clang-format-10 \ 74202871c91SPatrick Williams clang-tidy-10 \ 74302871c91SPatrick Williams clang-tools-10 \ 74402871c91SPatrick Williams shellcheck \ 74502871c91SPatrick Williams npm \ 74602871c91SPatrick Williams iproute2 \ 74702871c91SPatrick Williams libnl-3-dev \ 74802871c91SPatrick Williams libnl-genl-3-dev \ 74902871c91SPatrick Williams libconfig++-dev \ 75002871c91SPatrick Williams libsnmp-dev \ 75102871c91SPatrick Williams valgrind \ 75202871c91SPatrick Williams valgrind-dbg \ 75302871c91SPatrick Williams libpam0g-dev \ 75402871c91SPatrick Williams xxd \ 75502871c91SPatrick Williams libi2c-dev \ 75602871c91SPatrick Williams wget \ 75702871c91SPatrick Williams libldap2-dev \ 75802871c91SPatrick Williams libprotobuf-dev \ 75902871c91SPatrick Williams libperlio-gzip-perl \ 76002871c91SPatrick Williams libjson-perl \ 76102871c91SPatrick Williams protobuf-compiler \ 76202871c91SPatrick Williams libgpiod-dev \ 76302871c91SPatrick Williams device-tree-compiler \ 76402871c91SPatrick Williams cppcheck \ 76502871c91SPatrick Williams libpciaccess-dev \ 76602871c91SPatrick Williams libmimetic-dev \ 76702871c91SPatrick Williams libxml2-utils \ 7680eedeedaSPatrick Williams libxml-simple-perl \ 7690eedeedaSPatrick Williams rsync 77002871c91SPatrick Williams 77102871c91SPatrick WilliamsRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \ 77202871c91SPatrick Williams --slave /usr/bin/g++ g++ /usr/bin/g++-10 \ 77302871c91SPatrick Williams --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \ 77402871c91SPatrick Williams --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \ 77502871c91SPatrick Williams --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10 77602871c91SPatrick Williams 77702871c91SPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \ 77802871c91SPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \ 77902871c91SPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \ 78002871c91SPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \ 78102871c91SPatrick Williams --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py 78202871c91SPatrick Williams 78350837436SPatrick Williams""" 78450837436SPatrick Williams 78550837436SPatrick Williamsif is_automated_ci_build: 78650837436SPatrick Williams dockerfile_base += f""" 78750837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 78850837436SPatrick Williams# to re-run `apt-get update` daily. 789*ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 79050837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 79150837436SPatrick Williams 79250837436SPatrick Williams""" 79350837436SPatrick Williams 79450837436SPatrick Williamsdockerfile_base += f""" 79502871c91SPatrick WilliamsRUN pip3 install inflection 79602871c91SPatrick WilliamsRUN pip3 install pycodestyle 79702871c91SPatrick WilliamsRUN pip3 install jsonschema 79802871c91SPatrick WilliamsRUN pip3 install meson==0.54.3 79902871c91SPatrick WilliamsRUN pip3 install protobuf 800a18d9c57SPatrick Williams""" 80102871c91SPatrick Williams 802*ee3c9eebSPatrick Williams# Build the base and stage docker images. 803*ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 804*ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 805*ee3c9eebSPatrick WilliamsPackage.generate_all() 80602871c91SPatrick Williams 807*ee3c9eebSPatrick Williams# Create the final Dockerfile. 808a18d9c57SPatrick Williamsdockerfile = f""" 80902871c91SPatrick Williams# Build the final output image 810a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 811*ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 81202871c91SPatrick Williams 81302871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 81402871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 81502871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 816*ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 81702871c91SPatrick Williams 81802871c91SPatrick Williams# Final configuration for the workspace 81902871c91SPatrick WilliamsRUN grep -q {gid} /etc/group || groupadd -g {gid} {username} 82002871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 82102871c91SPatrick WilliamsRUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 82202871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 82302871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 82402871c91SPatrick Williams 82502871c91SPatrick Williams{proxy_cmd} 82602871c91SPatrick Williams 82702871c91SPatrick WilliamsRUN /bin/bash 82802871c91SPatrick Williams""" 82902871c91SPatrick Williams 830a18d9c57SPatrick Williams# Do the final docker build 831*ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 832*ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 833*ee3c9eebSPatrick Williams 83400536fbeSPatrick Williams# Print the tag of the final image. 83500536fbeSPatrick Williamsprint(docker_final_img_name) 836