102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# default is ubuntu:focal 1050837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1150837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1250837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1350837436SPatrick Williams# (ex. Jenkins)> 1402871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1502871c91SPatrick Williams# repositories> 1602871c91SPatrick Williams# default is master, which will be used if input branch not 1702871c91SPatrick Williams# provided or not found 1802871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1902871c91SPatrick Williams# default ones in /etc/apt/sources.list> 2002871c91SPatrick Williams# default is empty, and no mirror is used. 2102871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2202871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2302871c91SPatrick Williams 2402871c91SPatrick Williamsimport os 2502871c91SPatrick Williamsimport sys 26b16f3e20SPatrick Williamsimport threading 27a18d9c57SPatrick Williamsfrom datetime import date 28a18d9c57SPatrick Williamsfrom hashlib import sha256 29ee3c9eebSPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 30ee3c9eebSPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional 3102871c91SPatrick Williams 32ee3c9eebSPatrick Williamstry: 33ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 34ee3c9eebSPatrick Williams from typing import TypedDict 35ee3c9eebSPatrick Williamsexcept: 36ee3c9eebSPatrick Williams 37ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 38ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 39ee3c9eebSPatrick Williams def __init_subclass__(cls, **kwargs): 40ee3c9eebSPatrick Williams super().__init_subclass__() 41ee3c9eebSPatrick Williams 42ee3c9eebSPatrick Williams 43ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 44aae36d18SPatrick Williamsprefix = "/usr/local" 4502871c91SPatrick Williamsproc_count = nproc().strip() 4602871c91SPatrick Williams 47ee3c9eebSPatrick Williams 48ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 49ee3c9eebSPatrick Williams """ Package Definition for packages dictionary. """ 50ee3c9eebSPatrick Williams 51ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 52ee3c9eebSPatrick Williams rev: str 53ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 54ee3c9eebSPatrick Williams url: Callable[[str, str], str] 55ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 56ee3c9eebSPatrick Williams depends: Iterable[str] 57ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 58ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 59ee3c9eebSPatrick Williams build_type: str 60ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 61ee3c9eebSPatrick Williams build_steps: Iterable[str] 62ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 63ee3c9eebSPatrick Williams config_flags: Iterable[str] 64ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 65ee3c9eebSPatrick Williams config_env: Iterable[str] 66ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 67ee3c9eebSPatrick Williams # before config / build / install. 68ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 696bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 706bce2ca1SPatrick Williams custom_post_install: Iterable[str] 71ee3c9eebSPatrick Williams 72ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 73ee3c9eebSPatrick Williams __tag: str 74ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 75ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 76ee3c9eebSPatrick Williams 7702871c91SPatrick Williams 787204324cSPatrick Williams# Packages to include in image. 797204324cSPatrick Williamspackages = { 80ee3c9eebSPatrick Williams "boost": PackageDef( 81ee3c9eebSPatrick Williams rev="1.75.0", 82ee3c9eebSPatrick Williams url=( 83*5f2549eaSPatrick Williams lambda pkg, rev: f"https://downloads.yoctoproject.org/mirror/sources/{pkg}_{rev.replace('.', '_')}.tar.bz2" 842abc4a48SPatrick Williams ), 85ee3c9eebSPatrick Williams build_type="custom", 86ee3c9eebSPatrick Williams build_steps=[ 87aae36d18SPatrick Williams f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 88aae36d18SPatrick Williams "./b2", 89aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 90aae36d18SPatrick Williams ], 91ee3c9eebSPatrick Williams ), 92ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 93ee3c9eebSPatrick Williams rev="v1.3.0", 94ee3c9eebSPatrick Williams build_type="custom", 95ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 96ee3c9eebSPatrick Williams ), 97ee3c9eebSPatrick Williams "catchorg/Catch2": PackageDef( 98ee3c9eebSPatrick Williams rev="v2.12.2", 99ee3c9eebSPatrick Williams build_type="cmake", 100ee3c9eebSPatrick Williams config_flags=["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"], 101ee3c9eebSPatrick Williams ), 102ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 103ee3c9eebSPatrick Williams rev="v1.9.1", 104ee3c9eebSPatrick Williams build_type="cmake", 105ee3c9eebSPatrick Williams config_flags=[ 106aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 107aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 108aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 109aae36d18SPatrick Williams ], 110ee3c9eebSPatrick Williams ), 111ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 112ee3c9eebSPatrick Williams rev="7.1.3", 113ee3c9eebSPatrick Williams build_type="cmake", 114ee3c9eebSPatrick Williams config_flags=[ 115aae36d18SPatrick Williams "-DFMT_DOC=OFF", 116aae36d18SPatrick Williams "-DFMT_TEST=OFF", 117aae36d18SPatrick Williams ], 118ee3c9eebSPatrick Williams ), 11902871c91SPatrick Williams # Snapshot from 2020-01-03 120ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 121ee3c9eebSPatrick Williams rev="3a0746bf5f601dfed05330aefcb6854354fce07d", 122ee3c9eebSPatrick Williams build_type="custom", 123ee3c9eebSPatrick Williams build_steps=[ 124aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 125aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 126aae36d18SPatrick Williams ], 127ee3c9eebSPatrick Williams ), 128ee3c9eebSPatrick Williams "google/googletest": PackageDef( 129ee3c9eebSPatrick Williams rev="release-1.10.0", 130ee3c9eebSPatrick Williams build_type="cmake", 131ee3c9eebSPatrick Williams config_env=["CXXFLAGS=-std=c++17"], 132ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 133ee3c9eebSPatrick Williams ), 13402871c91SPatrick Williams # Release 2020-08-06 135ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 136ee3c9eebSPatrick Williams rev="v3.9.1", 1376bce2ca1SPatrick Williams build_type="cmake", 1386bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1396bce2ca1SPatrick Williams custom_post_install=[ 140aae36d18SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 141aae36d18SPatrick Williams ], 142ee3c9eebSPatrick Williams ), 14302871c91SPatrick Williams # Snapshot from 2019-05-24 144ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 145ee3c9eebSPatrick Williams rev="v1.15", 146ee3c9eebSPatrick Williams build_type="make", 147ee3c9eebSPatrick Williams ), 1480eedeedaSPatrick Williams # dev-5.8 2021-01-11 149ee3c9eebSPatrick Williams "openbmc/linux": PackageDef( 150ee3c9eebSPatrick Williams rev="3cc95ae40716e56f81b69615781f54c78079042d", 151ee3c9eebSPatrick Williams build_type="custom", 152ee3c9eebSPatrick Williams build_steps=[ 153aae36d18SPatrick Williams f"make -j{proc_count} defconfig", 154aae36d18SPatrick Williams f"make INSTALL_HDR_PATH={prefix} headers_install", 155aae36d18SPatrick Williams ], 156ee3c9eebSPatrick Williams ), 1570eedeedaSPatrick Williams # Snapshot from 2020-06-13 158ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 159ee3c9eebSPatrick Williams rev="LibVNCServer-0.9.13", 160ee3c9eebSPatrick Williams build_type="cmake", 161ee3c9eebSPatrick Williams ), 162ee3c9eebSPatrick Williams "martinmoene/span-lite": PackageDef( 163ee3c9eebSPatrick Williams rev="v0.8.1", 164ee3c9eebSPatrick Williams build_type="cmake", 165ee3c9eebSPatrick Williams config_flags=[ 166aae36d18SPatrick Williams "-DSPAN_LITE_OPT_BUILD_TESTS=OFF", 167aae36d18SPatrick Williams ], 168ee3c9eebSPatrick Williams ), 1690eedeedaSPatrick Williams # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_8.0.0.bb 170ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 171ee3c9eebSPatrick Williams rev="8.0.0", 172ee3c9eebSPatrick Williams build_type="cmake", 173ee3c9eebSPatrick Williams ), 17402871c91SPatrick Williams # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 175ee3c9eebSPatrick Williams "CPPAlliance/url": PackageDef( 176ee3c9eebSPatrick Williams rev="a56ae0df6d3078319755fbaa67822b4fa7fd352b", 177ee3c9eebSPatrick Williams build_type="cmake", 178ee3c9eebSPatrick Williams config_flags=[ 179aae36d18SPatrick Williams "-DBOOST_URL_BUILD_EXAMPLES=OFF", 180aae36d18SPatrick Williams "-DBOOST_URL_BUILD_TESTS=OFF", 181aae36d18SPatrick Williams "-DBOOST_URL_STANDALONE=ON", 182aae36d18SPatrick Williams ], 183ee3c9eebSPatrick Williams ), 1840eedeedaSPatrick Williams # version from ./meta-openembedded/meta-oe/dynamic-layers/networking-layer/recipes-devtools/valijson/valijson_0.3.bb 1850eedeedaSPatrick Williams # Snapshot from 2020-12-02 - fix for curlpp dependency 186ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 187ee3c9eebSPatrick Williams rev="8cc83c8be9c1c927f5da952b2333b30e5f0353be", 188ee3c9eebSPatrick Williams build_type="cmake", 189ee3c9eebSPatrick Williams config_flags=[ 1900eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1910eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 192aae36d18SPatrick Williams ], 193ee3c9eebSPatrick Williams ), 19402871c91SPatrick Williams # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb 195ee3c9eebSPatrick Williams "nlohmann/fifo_map": PackageDef( 196ee3c9eebSPatrick Williams rev="0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9", 197ee3c9eebSPatrick Williams build_type="custom", 198ee3c9eebSPatrick Williams build_steps=[f"cp src/fifo_map.hpp {prefix}/include/"], 199ee3c9eebSPatrick Williams ), 200ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 201ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 202ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 203ee3c9eebSPatrick Williams build_type="meson", 204ee3c9eebSPatrick Williams config_flags=[ 205aae36d18SPatrick Williams "-Dexamples=false", 206aae36d18SPatrick Williams "-Dtests=disabled", 207aae36d18SPatrick Williams ], 208ee3c9eebSPatrick Williams ), 209ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 210ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 211ee3c9eebSPatrick Williams build_type="meson", 212ee3c9eebSPatrick Williams config_flags=[ 213aae36d18SPatrick Williams "-Ddata_com_ibm=true", 214aae36d18SPatrick Williams "-Ddata_org_open_power=true", 215aae36d18SPatrick Williams ], 216ee3c9eebSPatrick Williams ), 217ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 218ee3c9eebSPatrick Williams depends=[ 21983394610SPatrick Williams "USCiLab/cereal", 22083394610SPatrick Williams "nlohmann/fifo_map", 22183394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 22283394610SPatrick Williams "openbmc/sdbusplus", 22383394610SPatrick Williams "openbmc/sdeventplus", 224aae36d18SPatrick Williams ], 225ee3c9eebSPatrick Williams build_type="autoconf", 226ee3c9eebSPatrick Williams config_flags=[ 227aae36d18SPatrick Williams "--enable-metadata-processing", 228aae36d18SPatrick Williams f"YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml", 229aae36d18SPatrick Williams ], 230ee3c9eebSPatrick Williams ), 231ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 232ee3c9eebSPatrick Williams depends=[ 23383394610SPatrick Williams "boost", 23483394610SPatrick Williams "leethomason/tinyxml2", 23583394610SPatrick Williams "openbmc/phosphor-logging", 23683394610SPatrick Williams "openbmc/sdbusplus", 237aae36d18SPatrick Williams ], 238ee3c9eebSPatrick Williams build_type="autoconf", 239ee3c9eebSPatrick Williams ), 240ee3c9eebSPatrick Williams "openbmc/pldm": PackageDef( 241ee3c9eebSPatrick Williams depends=[ 24283394610SPatrick Williams "CLIUtils/CLI11", 24383394610SPatrick Williams "boost", 24483394610SPatrick Williams "nlohmann/json", 24583394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 24683394610SPatrick Williams "openbmc/phosphor-logging", 24783394610SPatrick Williams "openbmc/sdbusplus", 24883394610SPatrick Williams "openbmc/sdeventplus", 249aae36d18SPatrick Williams ], 250ee3c9eebSPatrick Williams build_type="meson", 251ee3c9eebSPatrick Williams config_flags=[ 252aae36d18SPatrick Williams "-Dlibpldm-only=enabled", 253aae36d18SPatrick Williams "-Doem-ibm=enabled", 254aae36d18SPatrick Williams "-Dtests=disabled", 255aae36d18SPatrick Williams ], 256ee3c9eebSPatrick Williams ), 257ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 258ee3c9eebSPatrick Williams build_type="meson", 259ee3c9eebSPatrick Williams custom_post_dl=[ 260aae36d18SPatrick Williams "cd tools", 261aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 262aae36d18SPatrick Williams "cd ..", 263aae36d18SPatrick Williams ], 264ee3c9eebSPatrick Williams config_flags=[ 265aae36d18SPatrick Williams "-Dexamples=disabled", 266aae36d18SPatrick Williams "-Dtests=disabled", 267aae36d18SPatrick Williams ], 268b16f3e20SPatrick Williams ), 269ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 270ee3c9eebSPatrick Williams depends=["Naios/function2", "openbmc/stdplus"], 271ee3c9eebSPatrick Williams build_type="meson", 272ee3c9eebSPatrick Williams config_flags=[ 273ee3c9eebSPatrick Williams "-Dexamples=false", 274ee3c9eebSPatrick Williams "-Dtests=disabled", 275ee3c9eebSPatrick Williams ], 276ee3c9eebSPatrick Williams ), 277ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 278ee3c9eebSPatrick Williams depends=["fmtlib/fmt", "martinmoene/span-lite"], 279ee3c9eebSPatrick Williams build_type="meson", 280ee3c9eebSPatrick Williams config_flags=[ 281ee3c9eebSPatrick Williams "-Dexamples=false", 282ee3c9eebSPatrick Williams "-Dtests=disabled", 283ee3c9eebSPatrick Williams ], 284ee3c9eebSPatrick Williams ), 285ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 28602871c91SPatrick Williams 28702871c91SPatrick Williams# Define common flags used for builds 28802871c91SPatrick Williamsconfigure_flags = " ".join( 28902871c91SPatrick Williams [ 29002871c91SPatrick Williams f"--prefix={prefix}", 29102871c91SPatrick Williams ] 29202871c91SPatrick Williams) 29302871c91SPatrick Williamscmake_flags = " ".join( 29402871c91SPatrick Williams [ 29502871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 2960f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 29702871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 2980f2086b3SPatrick Williams "-GNinja", 2990f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 30002871c91SPatrick Williams ] 30102871c91SPatrick Williams) 30202871c91SPatrick Williamsmeson_flags = " ".join( 30302871c91SPatrick Williams [ 30402871c91SPatrick Williams "--wrap-mode=nodownload", 30502871c91SPatrick Williams f"-Dprefix={prefix}", 30602871c91SPatrick Williams ] 30702871c91SPatrick Williams) 30802871c91SPatrick Williams 309ee3c9eebSPatrick Williams 310ee3c9eebSPatrick Williamsclass Package(threading.Thread): 311ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 312ee3c9eebSPatrick Williams 313ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 314ee3c9eebSPatrick Williams Package.generate_all(). 315ee3c9eebSPatrick Williams """ 316ee3c9eebSPatrick Williams 317ee3c9eebSPatrick Williams # Copy the packages dictionary. 318ee3c9eebSPatrick Williams packages = packages.copy() 319ee3c9eebSPatrick Williams 320ee3c9eebSPatrick Williams # Lock used for thread-safety. 321ee3c9eebSPatrick Williams lock = threading.Lock() 322ee3c9eebSPatrick Williams 323ee3c9eebSPatrick Williams def __init__(self, pkg: str): 324ee3c9eebSPatrick Williams """ pkg - The name of this package (ex. foo/bar ) """ 325ee3c9eebSPatrick Williams super(Package, self).__init__() 326ee3c9eebSPatrick Williams 327ee3c9eebSPatrick Williams self.package = pkg 328ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 329ee3c9eebSPatrick Williams 330ee3c9eebSPatrick Williams # Reference to this package's 331ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 332ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 333ee3c9eebSPatrick Williams 334ee3c9eebSPatrick Williams def run(self) -> None: 335ee3c9eebSPatrick Williams """ Thread 'run' function. Builds the Docker stage. """ 336ee3c9eebSPatrick Williams 337ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 338ee3c9eebSPatrick Williams self._update_rev() 339ee3c9eebSPatrick Williams 340ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 341ee3c9eebSPatrick Williams # This section is locked because we are looking into another 342ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 343ee3c9eebSPatrick Williams Package.lock.acquire() 344ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 345ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 346ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 347ee3c9eebSPatrick Williams ] 348ee3c9eebSPatrick Williams Package.lock.release() 349ee3c9eebSPatrick Williams 350ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 351ee3c9eebSPatrick Williams # for the "COPY" commands. 352ee3c9eebSPatrick Williams for deppkg in deps: 353ee3c9eebSPatrick Williams deppkg.join() 354ee3c9eebSPatrick Williams 355ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 356ee3c9eebSPatrick Williams dockerfile = f""" 357ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 358ee3c9eebSPatrick Williams{self._df_copycmds()} 359ee3c9eebSPatrick Williams{self._df_build()} 360ee3c9eebSPatrick Williams""" 361ee3c9eebSPatrick Williams 362ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 363ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 364ee3c9eebSPatrick Williams # which can be accessed by other threads. 365ee3c9eebSPatrick Williams Package.lock.acquire() 366ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 367ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 368ee3c9eebSPatrick Williams Package.lock.release() 369ee3c9eebSPatrick Williams 370ee3c9eebSPatrick Williams # Do the build / save any exceptions. 371ee3c9eebSPatrick Williams try: 372ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 373ee3c9eebSPatrick Williams except Exception as e: 374ee3c9eebSPatrick Williams self.exception = e 375ee3c9eebSPatrick Williams 376ee3c9eebSPatrick Williams @classmethod 377ee3c9eebSPatrick Williams def generate_all(cls) -> None: 378ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 379ee3c9eebSPatrick Williams 380ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 381ee3c9eebSPatrick Williams package 'depends' specifications. 382ee3c9eebSPatrick Williams """ 383ee3c9eebSPatrick Williams 384ee3c9eebSPatrick Williams # Create a Package for each defined package. 385ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 386ee3c9eebSPatrick Williams 387ee3c9eebSPatrick Williams # Start building them all. 3886dbd7807SPatrick Williams # This section is locked because threads depend on each other, 3896dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 3906dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 3916dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 3926dbd7807SPatrick Williams Package.lock.acquire() 393ee3c9eebSPatrick Williams for t in pkg_threads: 394ee3c9eebSPatrick Williams t.start() 3956dbd7807SPatrick Williams Package.lock.release() 396ee3c9eebSPatrick Williams 397ee3c9eebSPatrick Williams # Wait for completion. 398ee3c9eebSPatrick Williams for t in pkg_threads: 399ee3c9eebSPatrick Williams t.join() 400ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 401ee3c9eebSPatrick Williams if t.exception: 402ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 403ee3c9eebSPatrick Williams raise t.exception 404ee3c9eebSPatrick Williams 405ee3c9eebSPatrick Williams @staticmethod 406ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 407ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 408ee3c9eebSPatrick Williams into the final image. 409ee3c9eebSPatrick Williams """ 410ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 411ee3c9eebSPatrick Williams 412ee3c9eebSPatrick Williams @classmethod 413ee3c9eebSPatrick Williams def depcache(cls) -> str: 414ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 415ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 416ee3c9eebSPatrick Williams """ 417ee3c9eebSPatrick Williams 418ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 419ee3c9eebSPatrick Williams depcache = "" 420ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 421ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 422ee3c9eebSPatrick Williams return depcache 423ee3c9eebSPatrick Williams 424ee3c9eebSPatrick Williams def _update_rev(self) -> None: 425ee3c9eebSPatrick Williams """ Look up the HEAD for missing a static rev. """ 426ee3c9eebSPatrick Williams 427ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 428ee3c9eebSPatrick Williams return 429ee3c9eebSPatrick Williams 43065b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 43165b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 43265b21fb9SPatrick Williams print( 43365b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 43465b21fb9SPatrick Williams file=sys.stderr, 43565b21fb9SPatrick Williams ) 43665b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 43765b21fb9SPatrick Williams return 43865b21fb9SPatrick Williams 439ee3c9eebSPatrick Williams # Ask Github for all the branches. 440ee3c9eebSPatrick Williams lookup = git("ls-remote", "--heads", f"https://github.com/{self.package}") 441ee3c9eebSPatrick Williams 442ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 443ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 444ee3c9eebSPatrick Williams Package.lock.acquire() 445ee3c9eebSPatrick Williams for line in lookup.split("\n"): 446ee3c9eebSPatrick Williams if f"refs/heads/{branch}" in line: 447ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 448ee3c9eebSPatrick Williams elif f"refs/heads/master" in line and "rev" not in self.pkg_def: 449ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 450ee3c9eebSPatrick Williams Package.lock.release() 451ee3c9eebSPatrick Williams 452ee3c9eebSPatrick Williams def _stagename(self) -> str: 453ee3c9eebSPatrick Williams """ Create a name for the Docker stage associated with this pkg. """ 454ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 455ee3c9eebSPatrick Williams 456ee3c9eebSPatrick Williams def _url(self) -> str: 457ee3c9eebSPatrick Williams """ Get the URL for this package. """ 458ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 459ee3c9eebSPatrick Williams 460ee3c9eebSPatrick Williams # If the lambda exists, call it. 461ee3c9eebSPatrick Williams if "url" in self.pkg_def: 462ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 463ee3c9eebSPatrick Williams 464ee3c9eebSPatrick Williams # Default to the github archive URL. 465ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 466ee3c9eebSPatrick Williams 467ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 468ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 469ee3c9eebSPatrick Williams 470ee3c9eebSPatrick Williams url = self._url() 471ee3c9eebSPatrick Williams if ".tar." not in url: 472ee3c9eebSPatrick Williams raise NotImplementedError( 473ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 474ee3c9eebSPatrick Williams ) 475ee3c9eebSPatrick Williams 476ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 477ee3c9eebSPatrick Williams 478ee3c9eebSPatrick Williams if url.endswith(".bz2"): 479ee3c9eebSPatrick Williams cmd += "j" 480ee3c9eebSPatrick Williams elif url.endswith(".gz"): 481ee3c9eebSPatrick Williams cmd += "z" 482ee3c9eebSPatrick Williams else: 483ee3c9eebSPatrick Williams raise NotImplementedError( 484ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 485ee3c9eebSPatrick Williams ) 486ee3c9eebSPatrick Williams 487ee3c9eebSPatrick Williams return cmd 488ee3c9eebSPatrick Williams 489ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 490ee3c9eebSPatrick Williams """ Formulate the command necessary to 'cd' into the source dir. """ 491ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 492ee3c9eebSPatrick Williams 493ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 494ee3c9eebSPatrick Williams """ Formulate the dockerfile snippet necessary to COPY all depends. """ 495ee3c9eebSPatrick Williams 496ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 497ee3c9eebSPatrick Williams return "" 498ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 499ee3c9eebSPatrick Williams 500ee3c9eebSPatrick Williams @staticmethod 501ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 502ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 503ee3c9eebSPatrick Williams packages into a Docker stage. 504ee3c9eebSPatrick Williams """ 505ee3c9eebSPatrick Williams 506ee3c9eebSPatrick Williams copy_cmds = "" 507ee3c9eebSPatrick Williams 508ee3c9eebSPatrick Williams # Sort the packages for consistency. 509ee3c9eebSPatrick Williams for p in sorted(pkgs): 510ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 511ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 512ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 513ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 514ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 515ee3c9eebSPatrick Williams 516ee3c9eebSPatrick Williams return copy_cmds 517ee3c9eebSPatrick Williams 518ee3c9eebSPatrick Williams def _df_build(self) -> str: 519ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 520ee3c9eebSPatrick Williams install a package into a Docker stage. 521ee3c9eebSPatrick Williams """ 522ee3c9eebSPatrick Williams 523ee3c9eebSPatrick Williams # Download and extract source. 524ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 525ee3c9eebSPatrick Williams 526ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 527ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 528ee3c9eebSPatrick Williams if custom_post_dl: 529ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 530ee3c9eebSPatrick Williams 531ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 532ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 533ee3c9eebSPatrick Williams if build_type == "autoconf": 534ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 535ee3c9eebSPatrick Williams elif build_type == "cmake": 536ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 537ee3c9eebSPatrick Williams elif build_type == "custom": 538ee3c9eebSPatrick Williams result += self._cmd_build_custom() 539ee3c9eebSPatrick Williams elif build_type == "make": 540ee3c9eebSPatrick Williams result += self._cmd_build_make() 541ee3c9eebSPatrick Williams elif build_type == "meson": 542ee3c9eebSPatrick Williams result += self._cmd_build_meson() 543ee3c9eebSPatrick Williams else: 544ee3c9eebSPatrick Williams raise NotImplementedError( 545ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 546ee3c9eebSPatrick Williams ) 547ee3c9eebSPatrick Williams 5486bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5496bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5506bce2ca1SPatrick Williams if custom_post_install: 5516bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5526bce2ca1SPatrick Williams 553ee3c9eebSPatrick Williams return result 554ee3c9eebSPatrick Williams 555ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 556ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 557ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 558ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 559ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 560ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 561ee3c9eebSPatrick Williams return result 562ee3c9eebSPatrick Williams 563ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 564ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 565ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 566ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 567ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 568ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 569ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 570ee3c9eebSPatrick Williams result += "cd .." 571ee3c9eebSPatrick Williams return result 572ee3c9eebSPatrick Williams 573ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 574ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 575ee3c9eebSPatrick Williams 576ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 577ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 578ee3c9eebSPatrick Williams 579ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 580ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 581ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 582ee3c9eebSPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 583ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 584ee3c9eebSPatrick Williams return result 585ee3c9eebSPatrick Williams 586ee3c9eebSPatrick Williams 587ee3c9eebSPatrick Williamsclass Docker: 588ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 589ee3c9eebSPatrick Williams 590ee3c9eebSPatrick Williams @staticmethod 591ee3c9eebSPatrick Williams def timestamp() -> str: 592ee3c9eebSPatrick Williams """ Generate a timestamp for today using the ISO week. """ 593ee3c9eebSPatrick Williams today = date.today().isocalendar() 594ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 595ee3c9eebSPatrick Williams 596ee3c9eebSPatrick Williams @staticmethod 597ee3c9eebSPatrick Williams def tagname(pkgname: str, dockerfile: str) -> str: 598ee3c9eebSPatrick Williams """ Generate a tag name for a package using a hash of the Dockerfile. """ 599ee3c9eebSPatrick Williams result = docker_image_name 600ee3c9eebSPatrick Williams if pkgname: 601ee3c9eebSPatrick Williams result += "-" + pkgname 602ee3c9eebSPatrick Williams 603ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 604ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 605ee3c9eebSPatrick Williams 606ee3c9eebSPatrick Williams return result 607ee3c9eebSPatrick Williams 608ee3c9eebSPatrick Williams @staticmethod 609ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 610ee3c9eebSPatrick Williams """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 611ee3c9eebSPatrick Williams 612ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 613ee3c9eebSPatrick Williams if not force_build: 614ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 615ee3c9eebSPatrick Williams print(f"Image {tag} already exists. Skipping.", file=sys.stderr) 616ee3c9eebSPatrick Williams return 617ee3c9eebSPatrick Williams 618ee3c9eebSPatrick Williams # Build it. 619ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 620ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 621ee3c9eebSPatrick Williams # progress but not polute stdout. Later on we output the final 622ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 623ee3c9eebSPatrick Williams # 624ee3c9eebSPatrick Williams # Other unusual flags: 625ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 626ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 627ee3c9eebSPatrick Williams docker.build( 628ee3c9eebSPatrick Williams proxy_args, 629ee3c9eebSPatrick Williams "--network=host", 630ee3c9eebSPatrick Williams "--force-rm", 631ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 632ee3c9eebSPatrick Williams "-t", 633ee3c9eebSPatrick Williams tag, 634ee3c9eebSPatrick Williams "-", 635ee3c9eebSPatrick Williams _in=dockerfile, 636ee3c9eebSPatrick Williams _out=( 637ee3c9eebSPatrick Williams lambda line: print( 638ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 639ee3c9eebSPatrick Williams ) 640ee3c9eebSPatrick Williams ), 641ee3c9eebSPatrick Williams ) 642ee3c9eebSPatrick Williams 643ee3c9eebSPatrick Williams 644ee3c9eebSPatrick Williams# Read a bunch of environment variables. 645ee3c9eebSPatrick Williamsdocker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 646ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 647ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 6489263f4d5SEd Tanousdistro = os.environ.get("DISTRO", "ubuntu:groovy") 649ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 650ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 651ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 652ee3c9eebSPatrick Williams 65365b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 65465b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 65565b21fb9SPatrick Williams 656ee3c9eebSPatrick Williams# Set up some common variables. 657ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 658ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 659ee3c9eebSPatrick Williamsgid = os.getgid() 660ee3c9eebSPatrick Williamsuid = os.getuid() 661ee3c9eebSPatrick Williams 662ee3c9eebSPatrick Williams# Determine the architecture for Docker. 663ee3c9eebSPatrick Williamsarch = uname("-m").strip() 664ee3c9eebSPatrick Williamsif arch == "ppc64le": 665ee3c9eebSPatrick Williams docker_base = "ppc64le/" 666ee3c9eebSPatrick Williamselif arch == "x86_64": 667ee3c9eebSPatrick Williams docker_base = "" 668ee3c9eebSPatrick Williamselse: 669ee3c9eebSPatrick Williams print( 670ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 671ee3c9eebSPatrick Williams file=sys.stderr, 672ee3c9eebSPatrick Williams ) 673ee3c9eebSPatrick Williams sys.exit(1) 674ee3c9eebSPatrick Williams 67502871c91SPatrick Williams# Special flags if setting up a deb mirror. 67602871c91SPatrick Williamsmirror = "" 67702871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 67802871c91SPatrick Williams mirror = f""" 67902871c91SPatrick WilliamsRUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 68002871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 68102871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 68202871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 68302871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 68402871c91SPatrick Williams""" 68502871c91SPatrick Williams 68602871c91SPatrick Williams# Special flags for proxying. 68702871c91SPatrick Williamsproxy_cmd = "" 68802871c91SPatrick Williamsproxy_args = [] 68902871c91SPatrick Williamsif http_proxy: 69002871c91SPatrick Williams proxy_cmd = f""" 69102871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 69202871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 69302871c91SPatrick Williams""" 69402871c91SPatrick Williams proxy_args.extend( 69502871c91SPatrick Williams [ 69602871c91SPatrick Williams "--build-arg", 69702871c91SPatrick Williams f"http_proxy={http_proxy}", 69802871c91SPatrick Williams "--build-arg", 699d461cd6aSLei YU f"https_proxy={http_proxy}", 70002871c91SPatrick Williams ] 70102871c91SPatrick Williams ) 70202871c91SPatrick Williams 703ee3c9eebSPatrick Williams# Create base Dockerfile. 704a18d9c57SPatrick Williamsdockerfile_base = f""" 705a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 70602871c91SPatrick Williams 70702871c91SPatrick Williams{mirror} 70802871c91SPatrick Williams 70902871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 71002871c91SPatrick Williams 71102871c91SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/" 71202871c91SPatrick Williams 713bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 714bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 715bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 716bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 717bb16ac14SPatrick Williams# the update. 718bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 719bb16ac14SPatrick Williams 72002871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 72102871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 72202871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 72350837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 72450837436SPatrick Williams ( apt-get install ubuntu-dbgsym-keyring || \ 72550837436SPatrick Williams ( apt-get install -yy dirmngr && \ 72650837436SPatrick Williams apt-key adv --keyserver keyserver.ubuntu.com \ 72750837436SPatrick Williams --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 72802871c91SPatrick Williams 72902871c91SPatrick Williams# Parse the current repo list into a debug repo list 73002871c91SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 73102871c91SPatrick Williams 73202871c91SPatrick Williams# Remove non-existent debug repos 73302871c91SPatrick WilliamsRUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 73402871c91SPatrick Williams 73502871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 73602871c91SPatrick Williams 73702871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 73802871c91SPatrick Williams gcc-10 \ 73902871c91SPatrick Williams g++-10 \ 74002871c91SPatrick Williams libc6-dbg \ 74102871c91SPatrick Williams libc6-dev \ 74202871c91SPatrick Williams libtool \ 74302871c91SPatrick Williams bison \ 74402871c91SPatrick Williams libdbus-1-dev \ 74502871c91SPatrick Williams flex \ 74602871c91SPatrick Williams cmake \ 74702871c91SPatrick Williams python3 \ 74802871c91SPatrick Williams python3-dev\ 74902871c91SPatrick Williams python3-yaml \ 75002871c91SPatrick Williams python3-mako \ 75102871c91SPatrick Williams python3-pip \ 75202871c91SPatrick Williams python3-setuptools \ 75302871c91SPatrick Williams python3-git \ 75402871c91SPatrick Williams python3-socks \ 75502871c91SPatrick Williams pkg-config \ 75602871c91SPatrick Williams autoconf \ 75702871c91SPatrick Williams autoconf-archive \ 75802871c91SPatrick Williams libsystemd-dev \ 75902871c91SPatrick Williams systemd \ 76002871c91SPatrick Williams libssl-dev \ 76102871c91SPatrick Williams libevdev-dev \ 76202871c91SPatrick Williams libevdev2-dbgsym \ 76302871c91SPatrick Williams libjpeg-dev \ 76402871c91SPatrick Williams libpng-dev \ 76502871c91SPatrick Williams ninja-build \ 76602871c91SPatrick Williams sudo \ 76702871c91SPatrick Williams curl \ 76802871c91SPatrick Williams git \ 76902871c91SPatrick Williams dbus \ 77002871c91SPatrick Williams iputils-ping \ 771d83ca9a3SPatrick Williams clang-11 \ 772d83ca9a3SPatrick Williams clang-format-11 \ 773d83ca9a3SPatrick Williams clang-tidy-11 \ 774d83ca9a3SPatrick Williams clang-tools-11 \ 77502871c91SPatrick Williams shellcheck \ 77602871c91SPatrick Williams npm \ 77702871c91SPatrick Williams iproute2 \ 77802871c91SPatrick Williams libnl-3-dev \ 77902871c91SPatrick Williams libnl-genl-3-dev \ 78002871c91SPatrick Williams libconfig++-dev \ 78102871c91SPatrick Williams libsnmp-dev \ 78202871c91SPatrick Williams valgrind \ 78302871c91SPatrick Williams valgrind-dbg \ 78402871c91SPatrick Williams libpam0g-dev \ 78502871c91SPatrick Williams xxd \ 78602871c91SPatrick Williams libi2c-dev \ 78702871c91SPatrick Williams wget \ 78802871c91SPatrick Williams libldap2-dev \ 78902871c91SPatrick Williams libprotobuf-dev \ 79002871c91SPatrick Williams libperlio-gzip-perl \ 79102871c91SPatrick Williams libjson-perl \ 79202871c91SPatrick Williams protobuf-compiler \ 79302871c91SPatrick Williams libgpiod-dev \ 79402871c91SPatrick Williams device-tree-compiler \ 79502871c91SPatrick Williams cppcheck \ 79602871c91SPatrick Williams libpciaccess-dev \ 79702871c91SPatrick Williams libmimetic-dev \ 79802871c91SPatrick Williams libxml2-utils \ 7990eedeedaSPatrick Williams libxml-simple-perl \ 8000eedeedaSPatrick Williams rsync 80102871c91SPatrick Williams 80202871c91SPatrick WilliamsRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \ 80302871c91SPatrick Williams --slave /usr/bin/g++ g++ /usr/bin/g++-10 \ 80402871c91SPatrick Williams --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \ 80502871c91SPatrick Williams --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \ 80602871c91SPatrick Williams --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10 80702871c91SPatrick Williams 808d83ca9a3SPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-11 1000 \ 809d83ca9a3SPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-11 \ 810d83ca9a3SPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-11 \ 811d83ca9a3SPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-11 \ 812d83ca9a3SPatrick Williams --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-11.py \ 813d83ca9a3SPatrick Williams --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-11 81402871c91SPatrick Williams 81550837436SPatrick Williams""" 81650837436SPatrick Williams 81750837436SPatrick Williamsif is_automated_ci_build: 81850837436SPatrick Williams dockerfile_base += f""" 81950837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 82050837436SPatrick Williams# to re-run `apt-get update` daily. 821ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 82250837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 82350837436SPatrick Williams 82450837436SPatrick Williams""" 82550837436SPatrick Williams 82650837436SPatrick Williamsdockerfile_base += f""" 82702871c91SPatrick WilliamsRUN pip3 install inflection 82802871c91SPatrick WilliamsRUN pip3 install pycodestyle 82902871c91SPatrick WilliamsRUN pip3 install jsonschema 83086198702SManojkiran EdaRUN pip3 install meson==0.57.1 83102871c91SPatrick WilliamsRUN pip3 install protobuf 832e6f120aaSManojkiran EdaRUN pip3 install codespell 833a18d9c57SPatrick Williams""" 83402871c91SPatrick Williams 835ee3c9eebSPatrick Williams# Build the base and stage docker images. 836ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 837ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 838ee3c9eebSPatrick WilliamsPackage.generate_all() 83902871c91SPatrick Williams 840ee3c9eebSPatrick Williams# Create the final Dockerfile. 841a18d9c57SPatrick Williamsdockerfile = f""" 84202871c91SPatrick Williams# Build the final output image 843a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 844ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 84502871c91SPatrick Williams 84602871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 84702871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 84802871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 849ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 85002871c91SPatrick Williams 85102871c91SPatrick Williams# Final configuration for the workspace 85202871c91SPatrick WilliamsRUN grep -q {gid} /etc/group || groupadd -g {gid} {username} 85302871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 85402871c91SPatrick WilliamsRUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 85502871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 85602871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 85702871c91SPatrick Williams 858305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 859305a9a5dSAndrew Geissler# and data installs 860305a9a5dSAndrew GeisslerRUN chown -R {username}:{username} /usr/local 861305a9a5dSAndrew Geissler 86202871c91SPatrick Williams{proxy_cmd} 86302871c91SPatrick Williams 86402871c91SPatrick WilliamsRUN /bin/bash 86502871c91SPatrick Williams""" 86602871c91SPatrick Williams 867a18d9c57SPatrick Williams# Do the final docker build 868ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 869ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 870ee3c9eebSPatrick Williams 87100536fbeSPatrick Williams# Print the tag of the final image. 87200536fbeSPatrick Williamsprint(docker_final_img_name) 873