102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1050837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1150837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1250837436SPatrick Williams# (ex. Jenkins)> 1302871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1402871c91SPatrick Williams# repositories> 1502871c91SPatrick Williams# default is master, which will be used if input branch not 1602871c91SPatrick Williams# provided or not found 1702871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1802871c91SPatrick Williams# default ones in /etc/apt/sources.list> 1902871c91SPatrick Williams# default is empty, and no mirror is used. 2002871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2102871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2202871c91SPatrick Williams 2302871c91SPatrick Williamsimport os 2402871c91SPatrick Williamsimport sys 25b16f3e20SPatrick Williamsimport threading 26a18d9c57SPatrick Williamsfrom datetime import date 27a18d9c57SPatrick Williamsfrom hashlib import sha256 28e08ffba8SPatrick Williams 29e08ffba8SPatrick Williams# typing.Dict is used for type-hints. 30e08ffba8SPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional # noqa: F401 3102871c91SPatrick Williams 3241d86218SPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 3341d86218SPatrick Williams 34ee3c9eebSPatrick Williamstry: 35ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 36ee3c9eebSPatrick Williams from typing import TypedDict 3741d86218SPatrick Williamsexcept Exception: 38ee3c9eebSPatrick Williams 39ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 40ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 4141d86218SPatrick Williams def __init_subclass__(cls, **kwargs: Any) -> None: 42ee3c9eebSPatrick Williams super().__init_subclass__() 43ee3c9eebSPatrick Williams 44ee3c9eebSPatrick Williams 45ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 46aae36d18SPatrick Williamsprefix = "/usr/local" 4702871c91SPatrick Williamsproc_count = nproc().strip() 4802871c91SPatrick Williams 49ee3c9eebSPatrick Williams 50ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 51ee3c9eebSPatrick Williams """Package Definition for packages dictionary.""" 52ee3c9eebSPatrick Williams 53ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 54ee3c9eebSPatrick Williams rev: str 55ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 56ee3c9eebSPatrick Williams url: Callable[[str, str], str] 57ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 58ee3c9eebSPatrick Williams depends: Iterable[str] 59ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 60ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 61ee3c9eebSPatrick Williams build_type: str 62ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 63ee3c9eebSPatrick Williams build_steps: Iterable[str] 64ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 65ee3c9eebSPatrick Williams config_flags: Iterable[str] 66ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 67ee3c9eebSPatrick Williams config_env: Iterable[str] 68ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 69ee3c9eebSPatrick Williams # before config / build / install. 70ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 716bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 726bce2ca1SPatrick Williams custom_post_install: Iterable[str] 73ee3c9eebSPatrick Williams 74ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 75ee3c9eebSPatrick Williams __tag: str 76ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 77ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 78ee3c9eebSPatrick Williams 7902871c91SPatrick Williams 807204324cSPatrick Williams# Packages to include in image. 817204324cSPatrick Williamspackages = { 82ee3c9eebSPatrick Williams "boost": PackageDef( 8342ff4328SEd Tanous rev="1.83.0", 84ee3c9eebSPatrick Williams url=( 8545bfd1ffSEd Tanous lambda pkg, rev: f"https://boostorg.jfrog.io/artifactory/main/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.gz" # noqa: E501 862abc4a48SPatrick Williams ), 87ee3c9eebSPatrick Williams build_type="custom", 88ee3c9eebSPatrick Williams build_steps=[ 89e08ffba8SPatrick Williams ( 90782f41f4SBrad Bishop "curl --remote-name" 91876ea1efSPatrick Williams " https://github.com/williamspatrick/beast/commit/98f8b1fbd059a35754c2c7b2841769cf8d021272.patch" 92876ea1efSPatrick Williams " && patch -p2 <" 93876ea1efSPatrick Williams " 98f8b1fbd059a35754c2c7b2841769cf8d021272.patch &&" 94e08ffba8SPatrick Williams " ./bootstrap.sh" 9542ff4328SEd Tanous f" --prefix={prefix} --with-libraries=context,coroutine,url" 96e08ffba8SPatrick Williams ), 97aae36d18SPatrick Williams "./b2", 98aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 99aae36d18SPatrick Williams ], 100ee3c9eebSPatrick Williams ), 101ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 102c1977839SPatrick Williams rev="v1.3.2", 103ee3c9eebSPatrick Williams build_type="custom", 104ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 105ee3c9eebSPatrick Williams ), 106c7198558SEd Tanous "danmar/cppcheck": PackageDef( 10751021786SPatrick Williams rev="2.12.1", 108c7198558SEd Tanous build_type="cmake", 109c7198558SEd Tanous ), 110ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 111fc39733aSPatrick Williams rev="v2.3.2", 112ee3c9eebSPatrick Williams build_type="cmake", 113ee3c9eebSPatrick Williams config_flags=[ 114aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 115aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 116aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 117aae36d18SPatrick Williams ], 118ee3c9eebSPatrick Williams ), 119ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 120c061e07bSPatrick Williams rev="10.1.1", 121ee3c9eebSPatrick Williams build_type="cmake", 122ee3c9eebSPatrick Williams config_flags=[ 123aae36d18SPatrick Williams "-DFMT_DOC=OFF", 124aae36d18SPatrick Williams "-DFMT_TEST=OFF", 125aae36d18SPatrick Williams ], 126ee3c9eebSPatrick Williams ), 127ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 128cb09974cSPatrick Williams rev="4.2.4", 129ee3c9eebSPatrick Williams build_type="custom", 130ee3c9eebSPatrick Williams build_steps=[ 131aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 132aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 133aae36d18SPatrick Williams ], 134ee3c9eebSPatrick Williams ), 135ee3c9eebSPatrick Williams "google/googletest": PackageDef( 136fdf243bbSPatrick Williams rev="v1.14.0", 137ee3c9eebSPatrick Williams build_type="cmake", 1384dd32c02SWilliam A. Kennington III config_env=["CXXFLAGS=-std=c++20"], 139ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 140ee3c9eebSPatrick Williams ), 141178b4b29SEd Tanous "nghttp2/nghttp2": PackageDef( 142*2b5df8bcSPatrick Williams rev="v1.57.0", 143178b4b29SEd Tanous build_type="cmake", 144178b4b29SEd Tanous config_env=["CXXFLAGS=-std=c++20"], 145178b4b29SEd Tanous config_flags=[ 146178b4b29SEd Tanous "-DENABLE_LIB_ONLY=ON", 147178b4b29SEd Tanous "-DENABLE_STATIC_LIB=ON", 148178b4b29SEd Tanous ], 149178b4b29SEd Tanous ), 150ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 151c1977839SPatrick Williams rev="v3.11.2", 1526bce2ca1SPatrick Williams build_type="cmake", 1536bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1546bce2ca1SPatrick Williams custom_post_install=[ 155e08ffba8SPatrick Williams ( 156e08ffba8SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp" 157e08ffba8SPatrick Williams f" {prefix}/include/json.hpp" 158e08ffba8SPatrick Williams ), 159aae36d18SPatrick Williams ], 160ee3c9eebSPatrick Williams ), 161058e3a34SPrzemyslaw Czarnowski "json-c/json-c": PackageDef( 162058e3a34SPrzemyslaw Czarnowski rev="json-c-0.16-20220414", 163058e3a34SPrzemyslaw Czarnowski build_type="cmake", 164058e3a34SPrzemyslaw Czarnowski ), 16502871c91SPatrick Williams # Snapshot from 2019-05-24 166ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 167ee3c9eebSPatrick Williams rev="v1.15", 168ee3c9eebSPatrick Williams build_type="make", 169ee3c9eebSPatrick Williams ), 170ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 171ee3c9eebSPatrick Williams rev="LibVNCServer-0.9.13", 172ee3c9eebSPatrick Williams build_type="cmake", 173ee3c9eebSPatrick Williams ), 174ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 175c1977839SPatrick Williams rev="9.0.0", 176ee3c9eebSPatrick Williams build_type="cmake", 177ee3c9eebSPatrick Williams ), 178ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 179c1977839SPatrick Williams rev="v0.7", 180ee3c9eebSPatrick Williams build_type="cmake", 181ee3c9eebSPatrick Williams config_flags=[ 1820eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1830eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 184aae36d18SPatrick Williams ], 185ee3c9eebSPatrick Williams ), 186ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 187ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 188ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 189ee3c9eebSPatrick Williams build_type="meson", 190ee3c9eebSPatrick Williams config_flags=[ 191aae36d18SPatrick Williams "-Dexamples=false", 192aae36d18SPatrick Williams "-Dtests=disabled", 193aae36d18SPatrick Williams ], 194ee3c9eebSPatrick Williams ), 195ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 196ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 197ee3c9eebSPatrick Williams build_type="meson", 1984fe87776SWilliam A. Kennington III config_flags=["-Dgenerate_md=false"], 199ee3c9eebSPatrick Williams ), 200ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 201ee3c9eebSPatrick Williams depends=[ 20283394610SPatrick Williams "USCiLab/cereal", 20383394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 20483394610SPatrick Williams "openbmc/sdbusplus", 20583394610SPatrick Williams "openbmc/sdeventplus", 206aae36d18SPatrick Williams ], 207f79ce4c4SPatrick Williams build_type="meson", 208ee3c9eebSPatrick Williams config_flags=[ 2096c98f280SWilliam A. Kennington III "-Dlibonly=true", 2106c98f280SWilliam A. Kennington III "-Dtests=disabled", 2115eabdae9SPatrick Williams f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml", 212aae36d18SPatrick Williams ], 213ee3c9eebSPatrick Williams ), 214ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 215ee3c9eebSPatrick Williams depends=[ 21611e5762cSBrad Bishop "CLIUtils/CLI11", 21770af95caSPatrick Williams "boost", 21883394610SPatrick Williams "leethomason/tinyxml2", 21970af95caSPatrick Williams "openbmc/phosphor-dbus-interfaces", 22083394610SPatrick Williams "openbmc/phosphor-logging", 22183394610SPatrick Williams "openbmc/sdbusplus", 222aae36d18SPatrick Williams ], 2231197e359SBrad Bishop build_type="meson", 2241197e359SBrad Bishop config_flags=[ 2251197e359SBrad Bishop "-Dtests=disabled", 2261197e359SBrad Bishop ], 227ee3c9eebSPatrick Williams ), 228c02ff271SJason M. Bills "openbmc/libpeci": PackageDef( 229c02ff271SJason M. Bills build_type="meson", 230c02ff271SJason M. Bills config_flags=[ 231c02ff271SJason M. Bills "-Draw-peci=disabled", 232c02ff271SJason M. Bills ], 233c02ff271SJason M. Bills ), 2341c19e453SManojkiran Eda "openbmc/libpldm": PackageDef( 235ee3c9eebSPatrick Williams build_type="meson", 236ee3c9eebSPatrick Williams config_flags=[ 23729d69bb0SAndrew Jeffery "-Dabi=deprecated,stable", 238aae36d18SPatrick Williams "-Doem-ibm=enabled", 239aae36d18SPatrick Williams "-Dtests=disabled", 240aae36d18SPatrick Williams ], 241ee3c9eebSPatrick Williams ), 242ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 243ee3c9eebSPatrick Williams build_type="meson", 244ee3c9eebSPatrick Williams custom_post_dl=[ 245aae36d18SPatrick Williams "cd tools", 246aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 247aae36d18SPatrick Williams "cd ..", 248aae36d18SPatrick Williams ], 249ee3c9eebSPatrick Williams config_flags=[ 250aae36d18SPatrick Williams "-Dexamples=disabled", 251aae36d18SPatrick Williams "-Dtests=disabled", 252aae36d18SPatrick Williams ], 253b16f3e20SPatrick Williams ), 254ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 25570af95caSPatrick Williams depends=[ 25670af95caSPatrick Williams "Naios/function2", 25770af95caSPatrick Williams "openbmc/stdplus", 25870af95caSPatrick Williams ], 259ee3c9eebSPatrick Williams build_type="meson", 260ee3c9eebSPatrick Williams config_flags=[ 261ee3c9eebSPatrick Williams "-Dexamples=false", 262ee3c9eebSPatrick Williams "-Dtests=disabled", 263ee3c9eebSPatrick Williams ], 264ee3c9eebSPatrick Williams ), 265ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 26670af95caSPatrick Williams depends=[ 26770af95caSPatrick Williams "fmtlib/fmt", 268ca1bf0c0SWilliam A. Kennington III "google/googletest", 269ca1bf0c0SWilliam A. Kennington III "Naios/function2", 27070af95caSPatrick Williams ], 271ee3c9eebSPatrick Williams build_type="meson", 272ee3c9eebSPatrick Williams config_flags=[ 273ee3c9eebSPatrick Williams "-Dexamples=false", 274ee3c9eebSPatrick Williams "-Dtests=disabled", 275ca1bf0c0SWilliam A. Kennington III "-Dgtest=enabled", 276ee3c9eebSPatrick Williams ], 277ee3c9eebSPatrick Williams ), 278ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 27902871c91SPatrick Williams 28002871c91SPatrick Williams# Define common flags used for builds 28102871c91SPatrick Williamsconfigure_flags = " ".join( 28202871c91SPatrick Williams [ 28302871c91SPatrick Williams f"--prefix={prefix}", 28402871c91SPatrick Williams ] 28502871c91SPatrick Williams) 28602871c91SPatrick Williamscmake_flags = " ".join( 28702871c91SPatrick Williams [ 28802871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 2890f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 29002871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 2910f2086b3SPatrick Williams "-GNinja", 2920f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 29302871c91SPatrick Williams ] 29402871c91SPatrick Williams) 29502871c91SPatrick Williamsmeson_flags = " ".join( 29602871c91SPatrick Williams [ 29702871c91SPatrick Williams "--wrap-mode=nodownload", 29802871c91SPatrick Williams f"-Dprefix={prefix}", 29902871c91SPatrick Williams ] 30002871c91SPatrick Williams) 30102871c91SPatrick Williams 302ee3c9eebSPatrick Williams 303ee3c9eebSPatrick Williamsclass Package(threading.Thread): 304ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 305ee3c9eebSPatrick Williams 306ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 307ee3c9eebSPatrick Williams Package.generate_all(). 308ee3c9eebSPatrick Williams """ 309ee3c9eebSPatrick Williams 310ee3c9eebSPatrick Williams # Copy the packages dictionary. 311ee3c9eebSPatrick Williams packages = packages.copy() 312ee3c9eebSPatrick Williams 313ee3c9eebSPatrick Williams # Lock used for thread-safety. 314ee3c9eebSPatrick Williams lock = threading.Lock() 315ee3c9eebSPatrick Williams 316ee3c9eebSPatrick Williams def __init__(self, pkg: str): 317ee3c9eebSPatrick Williams """pkg - The name of this package (ex. foo/bar )""" 318ee3c9eebSPatrick Williams super(Package, self).__init__() 319ee3c9eebSPatrick Williams 320ee3c9eebSPatrick Williams self.package = pkg 321ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 322ee3c9eebSPatrick Williams 323ee3c9eebSPatrick Williams # Reference to this package's 324ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 325ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 326ee3c9eebSPatrick Williams 327ee3c9eebSPatrick Williams def run(self) -> None: 328ee3c9eebSPatrick Williams """Thread 'run' function. Builds the Docker stage.""" 329ee3c9eebSPatrick Williams 330ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 331ee3c9eebSPatrick Williams self._update_rev() 332ee3c9eebSPatrick Williams 333ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 334ee3c9eebSPatrick Williams # This section is locked because we are looking into another 335ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 336ee3c9eebSPatrick Williams Package.lock.acquire() 337ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 338ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 339ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 340ee3c9eebSPatrick Williams ] 341ee3c9eebSPatrick Williams Package.lock.release() 342ee3c9eebSPatrick Williams 343ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 344ee3c9eebSPatrick Williams # for the "COPY" commands. 345ee3c9eebSPatrick Williams for deppkg in deps: 346ee3c9eebSPatrick Williams deppkg.join() 347ee3c9eebSPatrick Williams 348ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 349ee3c9eebSPatrick Williams dockerfile = f""" 350ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 351ee3c9eebSPatrick Williams{self._df_copycmds()} 352ee3c9eebSPatrick Williams{self._df_build()} 353ee3c9eebSPatrick Williams""" 354ee3c9eebSPatrick Williams 355ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 356ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 357ee3c9eebSPatrick Williams # which can be accessed by other threads. 358ee3c9eebSPatrick Williams Package.lock.acquire() 359ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 360ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 361ee3c9eebSPatrick Williams Package.lock.release() 362ee3c9eebSPatrick Williams 363ee3c9eebSPatrick Williams # Do the build / save any exceptions. 364ee3c9eebSPatrick Williams try: 365ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 366ee3c9eebSPatrick Williams except Exception as e: 367ee3c9eebSPatrick Williams self.exception = e 368ee3c9eebSPatrick Williams 369ee3c9eebSPatrick Williams @classmethod 370ee3c9eebSPatrick Williams def generate_all(cls) -> None: 371ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 372ee3c9eebSPatrick Williams 373ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 374ee3c9eebSPatrick Williams package 'depends' specifications. 375ee3c9eebSPatrick Williams """ 376ee3c9eebSPatrick Williams 377ee3c9eebSPatrick Williams # Create a Package for each defined package. 378ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 379ee3c9eebSPatrick Williams 380ee3c9eebSPatrick Williams # Start building them all. 3816dbd7807SPatrick Williams # This section is locked because threads depend on each other, 3826dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 3836dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 3846dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 3856dbd7807SPatrick Williams Package.lock.acquire() 386ee3c9eebSPatrick Williams for t in pkg_threads: 387ee3c9eebSPatrick Williams t.start() 3886dbd7807SPatrick Williams Package.lock.release() 389ee3c9eebSPatrick Williams 390ee3c9eebSPatrick Williams # Wait for completion. 391ee3c9eebSPatrick Williams for t in pkg_threads: 392ee3c9eebSPatrick Williams t.join() 393ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 394ee3c9eebSPatrick Williams if t.exception: 395ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 396ee3c9eebSPatrick Williams raise t.exception 397ee3c9eebSPatrick Williams 398ee3c9eebSPatrick Williams @staticmethod 399ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 400ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 401ee3c9eebSPatrick Williams into the final image. 402ee3c9eebSPatrick Williams """ 403ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 404ee3c9eebSPatrick Williams 405ee3c9eebSPatrick Williams @classmethod 406ee3c9eebSPatrick Williams def depcache(cls) -> str: 407ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 408ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 409ee3c9eebSPatrick Williams """ 410ee3c9eebSPatrick Williams 411ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 412ee3c9eebSPatrick Williams depcache = "" 413ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 414ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 415ee3c9eebSPatrick Williams return depcache 416ee3c9eebSPatrick Williams 417ee3c9eebSPatrick Williams def _update_rev(self) -> None: 418ee3c9eebSPatrick Williams """Look up the HEAD for missing a static rev.""" 419ee3c9eebSPatrick Williams 420ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 421ee3c9eebSPatrick Williams return 422ee3c9eebSPatrick Williams 42365b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 42465b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 42565b21fb9SPatrick Williams print( 42665b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 42765b21fb9SPatrick Williams file=sys.stderr, 42865b21fb9SPatrick Williams ) 42965b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 43065b21fb9SPatrick Williams return 43165b21fb9SPatrick Williams 432ee3c9eebSPatrick Williams # Ask Github for all the branches. 43305fb2a0aSPatrick Williams lookup = git( 43405fb2a0aSPatrick Williams "ls-remote", "--heads", f"https://github.com/{self.package}" 43505fb2a0aSPatrick Williams ) 436ee3c9eebSPatrick Williams 437ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 438ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 439ee3c9eebSPatrick Williams Package.lock.acquire() 440ee3c9eebSPatrick Williams for line in lookup.split("\n"): 441ee3c9eebSPatrick Williams if f"refs/heads/{branch}" in line: 442ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 443c7d73646SPatrick Williams elif ( 444c7d73646SPatrick Williams "refs/heads/master" in line or "refs/heads/main" in line 445c7d73646SPatrick Williams ) and "rev" not in self.pkg_def: 446ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 447ee3c9eebSPatrick Williams Package.lock.release() 448ee3c9eebSPatrick Williams 449ee3c9eebSPatrick Williams def _stagename(self) -> str: 450ee3c9eebSPatrick Williams """Create a name for the Docker stage associated with this pkg.""" 451ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 452ee3c9eebSPatrick Williams 453ee3c9eebSPatrick Williams def _url(self) -> str: 454ee3c9eebSPatrick Williams """Get the URL for this package.""" 455ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 456ee3c9eebSPatrick Williams 457ee3c9eebSPatrick Williams # If the lambda exists, call it. 458ee3c9eebSPatrick Williams if "url" in self.pkg_def: 459ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 460ee3c9eebSPatrick Williams 461ee3c9eebSPatrick Williams # Default to the github archive URL. 462ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 463ee3c9eebSPatrick Williams 464ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 465ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 466ee3c9eebSPatrick Williams 467ee3c9eebSPatrick Williams url = self._url() 468ee3c9eebSPatrick Williams if ".tar." not in url: 469ee3c9eebSPatrick Williams raise NotImplementedError( 470ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 471ee3c9eebSPatrick Williams ) 472ee3c9eebSPatrick Williams 473ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 474ee3c9eebSPatrick Williams 475ee3c9eebSPatrick Williams if url.endswith(".bz2"): 476ee3c9eebSPatrick Williams cmd += "j" 477ee3c9eebSPatrick Williams elif url.endswith(".gz"): 478ee3c9eebSPatrick Williams cmd += "z" 479ee3c9eebSPatrick Williams else: 480ee3c9eebSPatrick Williams raise NotImplementedError( 481ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 482ee3c9eebSPatrick Williams ) 483ee3c9eebSPatrick Williams 484ee3c9eebSPatrick Williams return cmd 485ee3c9eebSPatrick Williams 486ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 487ee3c9eebSPatrick Williams """Formulate the command necessary to 'cd' into the source dir.""" 488ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 489ee3c9eebSPatrick Williams 490ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 491ee3c9eebSPatrick Williams """Formulate the dockerfile snippet necessary to COPY all depends.""" 492ee3c9eebSPatrick Williams 493ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 494ee3c9eebSPatrick Williams return "" 495ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 496ee3c9eebSPatrick Williams 497ee3c9eebSPatrick Williams @staticmethod 498ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 499ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 500ee3c9eebSPatrick Williams packages into a Docker stage. 501ee3c9eebSPatrick Williams """ 502ee3c9eebSPatrick Williams 503ee3c9eebSPatrick Williams copy_cmds = "" 504ee3c9eebSPatrick Williams 505ee3c9eebSPatrick Williams # Sort the packages for consistency. 506ee3c9eebSPatrick Williams for p in sorted(pkgs): 507ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 508ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 509ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 510ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 511ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 512ee3c9eebSPatrick Williams 513ee3c9eebSPatrick Williams return copy_cmds 514ee3c9eebSPatrick Williams 515ee3c9eebSPatrick Williams def _df_build(self) -> str: 516ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 517ee3c9eebSPatrick Williams install a package into a Docker stage. 518ee3c9eebSPatrick Williams """ 519ee3c9eebSPatrick Williams 520ee3c9eebSPatrick Williams # Download and extract source. 521ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 522ee3c9eebSPatrick Williams 523ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 524ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 525ee3c9eebSPatrick Williams if custom_post_dl: 526ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 527ee3c9eebSPatrick Williams 528ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 529ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 530ee3c9eebSPatrick Williams if build_type == "autoconf": 531ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 532ee3c9eebSPatrick Williams elif build_type == "cmake": 533ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 534ee3c9eebSPatrick Williams elif build_type == "custom": 535ee3c9eebSPatrick Williams result += self._cmd_build_custom() 536ee3c9eebSPatrick Williams elif build_type == "make": 537ee3c9eebSPatrick Williams result += self._cmd_build_make() 538ee3c9eebSPatrick Williams elif build_type == "meson": 539ee3c9eebSPatrick Williams result += self._cmd_build_meson() 540ee3c9eebSPatrick Williams else: 541ee3c9eebSPatrick Williams raise NotImplementedError( 542ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 543ee3c9eebSPatrick Williams ) 544ee3c9eebSPatrick Williams 5456bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5466bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5476bce2ca1SPatrick Williams if custom_post_install: 5486bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5496bce2ca1SPatrick Williams 550ee3c9eebSPatrick Williams return result 551ee3c9eebSPatrick Williams 552ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 553ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 554ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 555ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 556ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 557ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 558ee3c9eebSPatrick Williams return result 559ee3c9eebSPatrick Williams 560ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 561ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 562ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 563ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 564ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 565ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 566ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 567ee3c9eebSPatrick Williams result += "cd .." 568ee3c9eebSPatrick Williams return result 569ee3c9eebSPatrick Williams 570ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 571ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 572ee3c9eebSPatrick Williams 573ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 574ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 575ee3c9eebSPatrick Williams 576ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 577ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 578ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 579e2da11adSAndrew Jeffery result = f"{env} meson setup builddir {meson_flags} {options} && " 580ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 581ee3c9eebSPatrick Williams return result 582ee3c9eebSPatrick Williams 583ee3c9eebSPatrick Williams 584ee3c9eebSPatrick Williamsclass Docker: 585ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 586ee3c9eebSPatrick Williams 587ee3c9eebSPatrick Williams @staticmethod 588ee3c9eebSPatrick Williams def timestamp() -> str: 589ee3c9eebSPatrick Williams """Generate a timestamp for today using the ISO week.""" 590ee3c9eebSPatrick Williams today = date.today().isocalendar() 591ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 592ee3c9eebSPatrick Williams 593ee3c9eebSPatrick Williams @staticmethod 59441d86218SPatrick Williams def tagname(pkgname: Optional[str], dockerfile: str) -> str: 595ee3c9eebSPatrick Williams """Generate a tag name for a package using a hash of the Dockerfile.""" 596ee3c9eebSPatrick Williams result = docker_image_name 597ee3c9eebSPatrick Williams if pkgname: 598ee3c9eebSPatrick Williams result += "-" + pkgname 599ee3c9eebSPatrick Williams 600ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 601ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 602ee3c9eebSPatrick Williams 603ee3c9eebSPatrick Williams return result 604ee3c9eebSPatrick Williams 605ee3c9eebSPatrick Williams @staticmethod 606ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 60722e6110bSAndrew Geissler """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 608ee3c9eebSPatrick Williams 609ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 610ee3c9eebSPatrick Williams if not force_build: 611ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 61205fb2a0aSPatrick Williams print( 61305fb2a0aSPatrick Williams f"Image {tag} already exists. Skipping.", file=sys.stderr 61405fb2a0aSPatrick Williams ) 615ee3c9eebSPatrick Williams return 616ee3c9eebSPatrick Williams 617ee3c9eebSPatrick Williams # Build it. 618ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 619ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 620ee3c9eebSPatrick Williams # progress but not polute stdout. Later on we output the final 621ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 622ee3c9eebSPatrick Williams # 623ee3c9eebSPatrick Williams # Other unusual flags: 624ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 625ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 626ee3c9eebSPatrick Williams docker.build( 627ee3c9eebSPatrick Williams proxy_args, 628ee3c9eebSPatrick Williams "--network=host", 629ee3c9eebSPatrick Williams "--force-rm", 630ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 631ee3c9eebSPatrick Williams "-t", 632ee3c9eebSPatrick Williams tag, 633ee3c9eebSPatrick Williams "-", 634ee3c9eebSPatrick Williams _in=dockerfile, 635ee3c9eebSPatrick Williams _out=( 636ee3c9eebSPatrick Williams lambda line: print( 637ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 638ee3c9eebSPatrick Williams ) 639ee3c9eebSPatrick Williams ), 640ee3c9eebSPatrick Williams ) 641ee3c9eebSPatrick Williams 642ee3c9eebSPatrick Williams 643ee3c9eebSPatrick Williams# Read a bunch of environment variables. 64405fb2a0aSPatrick Williamsdocker_image_name = os.environ.get( 64505fb2a0aSPatrick Williams "DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test" 64605fb2a0aSPatrick Williams) 647ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 648ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 649c6caa184SWilliam A. Kennington IIIdistro = os.environ.get("DISTRO", "ubuntu:mantic") 650ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 651ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 652ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 653ee3c9eebSPatrick Williams 65465b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 65565b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 65665b21fb9SPatrick Williams 657d0dabc3eSAndrew Geissler# Ensure appropriate docker build output to see progress and identify 658d0dabc3eSAndrew Geissler# any issues 659d0dabc3eSAndrew Geissleros.environ["BUILDKIT_PROGRESS"] = "plain" 660d0dabc3eSAndrew Geissler 661ee3c9eebSPatrick Williams# Set up some common variables. 662ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 663ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 664ee3c9eebSPatrick Williamsgid = os.getgid() 665ee3c9eebSPatrick Williamsuid = os.getuid() 666ee3c9eebSPatrick Williams 6676825a018SJosh Lehan# Use well-known constants if user is root 6686825a018SJosh Lehanif username == "root": 6696825a018SJosh Lehan homedir = "/root" 6706825a018SJosh Lehan gid = 0 6716825a018SJosh Lehan uid = 0 6726825a018SJosh Lehan 673ee3c9eebSPatrick Williams# Determine the architecture for Docker. 674ee3c9eebSPatrick Williamsarch = uname("-m").strip() 675ee3c9eebSPatrick Williamsif arch == "ppc64le": 676ee3c9eebSPatrick Williams docker_base = "ppc64le/" 677ee3c9eebSPatrick Williamselif arch == "x86_64": 678ee3c9eebSPatrick Williams docker_base = "" 679051b05b7SThang Q. Nguyenelif arch == "aarch64": 680f98f1a8dSThang Q. Nguyen docker_base = "arm64v8/" 681ee3c9eebSPatrick Williamselse: 682ee3c9eebSPatrick Williams print( 683ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 684ee3c9eebSPatrick Williams file=sys.stderr, 685ee3c9eebSPatrick Williams ) 686ee3c9eebSPatrick Williams sys.exit(1) 687ee3c9eebSPatrick Williams 68802871c91SPatrick Williams# Special flags if setting up a deb mirror. 68902871c91SPatrick Williamsmirror = "" 69002871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 69102871c91SPatrick Williams mirror = f""" 692e08ffba8SPatrick WilliamsRUN echo "deb {ubuntu_mirror} \ 693e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME) \ 694e08ffba8SPatrick Williams main restricted universe multiverse" > /etc/apt/sources.list && \\ 695e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 696e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-updates \ 697e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 698e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 699e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-security \ 700e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 701e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 702e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-proposed \ 703e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 704e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 705e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-backports \ 706e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list 70702871c91SPatrick Williams""" 70802871c91SPatrick Williams 70902871c91SPatrick Williams# Special flags for proxying. 71002871c91SPatrick Williamsproxy_cmd = "" 71134ec77e8SAdrian Ambrożewiczproxy_keyserver = "" 71202871c91SPatrick Williamsproxy_args = [] 71302871c91SPatrick Williamsif http_proxy: 71402871c91SPatrick Williams proxy_cmd = f""" 71502871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 71602871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 71702871c91SPatrick Williams""" 71834ec77e8SAdrian Ambrożewicz proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 71934ec77e8SAdrian Ambrożewicz 72002871c91SPatrick Williams proxy_args.extend( 72102871c91SPatrick Williams [ 72202871c91SPatrick Williams "--build-arg", 72302871c91SPatrick Williams f"http_proxy={http_proxy}", 72402871c91SPatrick Williams "--build-arg", 725d461cd6aSLei YU f"https_proxy={http_proxy}", 72602871c91SPatrick Williams ] 72702871c91SPatrick Williams ) 72802871c91SPatrick Williams 729ee3c9eebSPatrick Williams# Create base Dockerfile. 730a18d9c57SPatrick Williamsdockerfile_base = f""" 731a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 73202871c91SPatrick Williams 73302871c91SPatrick Williams{mirror} 73402871c91SPatrick Williams 73502871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 73602871c91SPatrick Williams 7378949d3c3SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/" 73802871c91SPatrick Williams 739bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 740bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 741bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 742bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 743bb16ac14SPatrick Williams# the update. 744bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 745bb16ac14SPatrick Williams 74602871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 74702871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 74802871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 749575b5e4cSJagpal Singh Gill# Known issue with gpg to get keys via proxy - 750575b5e4cSJagpal Singh Gill# https://bugs.launchpad.net/ubuntu/+source/gnupg2/+bug/1788190, hence using 751575b5e4cSJagpal Singh Gill# curl to get keys. 75250837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 753938d303fSJian Zhang ( apt-get install -yy gpgv ubuntu-dbgsym-keyring || \ 754575b5e4cSJagpal Singh Gill ( apt-get install -yy dirmngr curl && \ 755575b5e4cSJagpal Singh Gill curl -sSL \ 756575b5e4cSJagpal Singh Gill 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622' \ 757575b5e4cSJagpal Singh Gill | apt-key add - )) 75802871c91SPatrick Williams 75902871c91SPatrick Williams# Parse the current repo list into a debug repo list 760e08ffba8SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' \ 761e08ffba8SPatrick Williams /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 76202871c91SPatrick Williams 76302871c91SPatrick Williams# Remove non-existent debug repos 76441d86218SPatrick WilliamsRUN sed -i '/-\\(backports\\|security\\) /d' /etc/apt/sources.list.d/debug.list 76502871c91SPatrick Williams 76602871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 76702871c91SPatrick Williams 76802871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 76958f1915eSAndrew Jeffery abi-compliance-checker \ 7708b112068SAndrew Jeffery abi-dumper \ 77102871c91SPatrick Williams autoconf \ 77202871c91SPatrick Williams autoconf-archive \ 773af49ed51SAndrew Geissler bison \ 774e2e62e5cSPatrick Williams clang-17 \ 775e2e62e5cSPatrick Williams clang-format-17 \ 776e2e62e5cSPatrick Williams clang-tidy-17 \ 777e2e62e5cSPatrick Williams clang-tools-17 \ 778af49ed51SAndrew Geissler cmake \ 779af49ed51SAndrew Geissler curl \ 780af49ed51SAndrew Geissler dbus \ 781af49ed51SAndrew Geissler device-tree-compiler \ 782af49ed51SAndrew Geissler flex \ 783961f148bSPatrick Williams g++-13 \ 784961f148bSPatrick Williams gcc-13 \ 785af49ed51SAndrew Geissler git \ 78602871c91SPatrick Williams iproute2 \ 787af49ed51SAndrew Geissler iputils-ping \ 788524a331cSManojkiran Eda libaudit-dev \ 789af49ed51SAndrew Geissler libc6-dbg \ 790af49ed51SAndrew Geissler libc6-dev \ 791af49ed51SAndrew Geissler libconfig++-dev \ 792af49ed51SAndrew Geissler libcryptsetup-dev \ 793af49ed51SAndrew Geissler libdbus-1-dev \ 794af49ed51SAndrew Geissler libevdev-dev \ 795af49ed51SAndrew Geissler libgpiod-dev \ 796af49ed51SAndrew Geissler libi2c-dev \ 797af49ed51SAndrew Geissler libjpeg-dev \ 798af49ed51SAndrew Geissler libjson-perl \ 799af49ed51SAndrew Geissler libldap2-dev \ 800af49ed51SAndrew Geissler libmimetic-dev \ 80102871c91SPatrick Williams libnl-3-dev \ 80202871c91SPatrick Williams libnl-genl-3-dev \ 80302871c91SPatrick Williams libpam0g-dev \ 80402871c91SPatrick Williams libpciaccess-dev \ 805af49ed51SAndrew Geissler libperlio-gzip-perl \ 806af49ed51SAndrew Geissler libpng-dev \ 807af49ed51SAndrew Geissler libprotobuf-dev \ 808af49ed51SAndrew Geissler libsnmp-dev \ 809af49ed51SAndrew Geissler libssl-dev \ 810af49ed51SAndrew Geissler libsystemd-dev \ 811af49ed51SAndrew Geissler libtool \ 812af49ed51SAndrew Geissler liburing-dev \ 81302871c91SPatrick Williams libxml2-utils \ 8140eedeedaSPatrick Williams libxml-simple-perl \ 815af49ed51SAndrew Geissler ninja-build \ 816af49ed51SAndrew Geissler npm \ 817af49ed51SAndrew Geissler pkg-config \ 818af49ed51SAndrew Geissler protobuf-compiler \ 819af49ed51SAndrew Geissler python3 \ 820af49ed51SAndrew Geissler python3-dev\ 821af49ed51SAndrew Geissler python3-git \ 822af49ed51SAndrew Geissler python3-mako \ 823af49ed51SAndrew Geissler python3-pip \ 824af49ed51SAndrew Geissler python3-setuptools \ 825af49ed51SAndrew Geissler python3-socks \ 826af49ed51SAndrew Geissler python3-yaml \ 8279adf68d6SJohn Wedig rsync \ 828af49ed51SAndrew Geissler shellcheck \ 829af49ed51SAndrew Geissler sudo \ 830af49ed51SAndrew Geissler systemd \ 831af49ed51SAndrew Geissler valgrind \ 832eef3c37dSAndrew Geissler valgrind-dbgsym \ 833b565f825SAndrew Geissler vim \ 834af49ed51SAndrew Geissler wget \ 835af49ed51SAndrew Geissler xxd 83602871c91SPatrick Williams 837961f148bSPatrick WilliamsRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 13 \ 838961f148bSPatrick Williams --slave /usr/bin/g++ g++ /usr/bin/g++-13 \ 839961f148bSPatrick Williams --slave /usr/bin/gcov gcov /usr/bin/gcov-13 \ 840961f148bSPatrick Williams --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-13 \ 841961f148bSPatrick Williams --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-13 842961f148bSPatrick WilliamsRUN update-alternatives --remove cpp /usr/bin/cpp && \ 843961f148bSPatrick Williams update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-13 13 84402871c91SPatrick Williams 845e2e62e5cSPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 1000 \ 846e2e62e5cSPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-17 \ 847e2e62e5cSPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-17 \ 848e2e62e5cSPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-17 \ 849e08ffba8SPatrick Williams --slave /usr/bin/run-clang-tidy run-clang-tidy.py \ 850e2e62e5cSPatrick Williams /usr/bin/run-clang-tidy-17 \ 851e2e62e5cSPatrick Williams --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-17 85202871c91SPatrick Williams 85350837436SPatrick Williams""" 85450837436SPatrick Williams 85550837436SPatrick Williamsif is_automated_ci_build: 85650837436SPatrick Williams dockerfile_base += f""" 85750837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 85850837436SPatrick Williams# to re-run `apt-get update` daily. 859ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 86050837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 86150837436SPatrick Williams 86250837436SPatrick Williams""" 86350837436SPatrick Williams 86441d86218SPatrick Williamsdockerfile_base += """ 8655e4d8402SPatrick WilliamsRUN pip3 install --break-system-packages \ 866818023dfSPatrick Williams beautysh \ 867818023dfSPatrick Williams black \ 868818023dfSPatrick Williams codespell \ 869818023dfSPatrick Williams flake8 \ 870818023dfSPatrick Williams gitlint \ 871818023dfSPatrick Williams inflection \ 872818023dfSPatrick Williams isort \ 873818023dfSPatrick Williams jsonschema \ 874b20d9819SPatrick Williams meson==1.2.3 \ 875818023dfSPatrick Williams protobuf \ 876818023dfSPatrick Williams requests 877b08ddf77SPatrick Williams 878b08ddf77SPatrick WilliamsRUN npm install -g \ 879b08ddf77SPatrick Williams eslint@latest eslint-plugin-json@latest \ 8807d41f6d2SPatrick Williams markdownlint-cli@latest \ 881b08ddf77SPatrick Williams prettier@latest 882fb9948a3SEd Tanous""" 883fb9948a3SEd Tanous 884ee3c9eebSPatrick Williams# Build the base and stage docker images. 885ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 886ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 887ee3c9eebSPatrick WilliamsPackage.generate_all() 88802871c91SPatrick Williams 889ee3c9eebSPatrick Williams# Create the final Dockerfile. 890a18d9c57SPatrick Williamsdockerfile = f""" 89102871c91SPatrick Williams# Build the final output image 892a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 893ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 89402871c91SPatrick Williams 89502871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 89602871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 89702871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 898ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 89902871c91SPatrick Williams 90067cc0616SPatrick Williams# Ensure the group, user, and home directory are created (or rename them if 90167cc0616SPatrick Williams# they already exist). 90267cc0616SPatrick WilliamsRUN if grep -q ":{gid}:" /etc/group ; then \ 90367cc0616SPatrick Williams groupmod -n {username} $(awk -F : '{{ if ($3 == {gid}) {{ print $1 }} }}' /etc/group) ; \ 90467cc0616SPatrick Williams else \ 90567cc0616SPatrick Williams groupadd -f -g {gid} {username} ; \ 90667cc0616SPatrick Williams fi 90702871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 90867cc0616SPatrick WilliamsRUN if grep -q ":{uid}:" /etc/passwd ; then \ 90973b3ee91SPatrick Williams usermod -l {username} -d {homedir} -m $(awk -F : '{{ if ($3 == {uid}) {{ print $1 }} }}' /etc/passwd) ; \ 91067cc0616SPatrick Williams else \ 91167cc0616SPatrick Williams useradd -d {homedir} -m -u {uid} -g {gid} {username} ; \ 91267cc0616SPatrick Williams fi 91302871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 91402871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 91502871c91SPatrick Williams 916305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 917305a9a5dSAndrew Geissler# and data installs 9187bb00b13SAndrew GeisslerRUN chown -R {username}:{username} /usr/local/share 919305a9a5dSAndrew Geissler 92002871c91SPatrick Williams{proxy_cmd} 92102871c91SPatrick Williams 92202871c91SPatrick WilliamsRUN /bin/bash 92302871c91SPatrick Williams""" 92402871c91SPatrick Williams 925a18d9c57SPatrick Williams# Do the final docker build 926ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 927ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 928ee3c9eebSPatrick Williams 92900536fbeSPatrick Williams# Print the tag of the final image. 93000536fbeSPatrick Williamsprint(docker_final_img_name) 931