102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1050837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1150837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1250837436SPatrick Williams# (ex. Jenkins)> 1302871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1402871c91SPatrick Williams# repositories> 1502871c91SPatrick Williams# default is master, which will be used if input branch not 1602871c91SPatrick Williams# provided or not found 1702871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1802871c91SPatrick Williams# default ones in /etc/apt/sources.list> 1902871c91SPatrick Williams# default is empty, and no mirror is used. 2002871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2102871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2202871c91SPatrick Williams 2302871c91SPatrick Williamsimport os 24f3d27e64SAndrew Geisslerimport re 2502871c91SPatrick Williamsimport sys 26b16f3e20SPatrick Williamsimport threading 27a18d9c57SPatrick Williamsfrom datetime import date 28a18d9c57SPatrick Williamsfrom hashlib import sha256 29e08ffba8SPatrick Williams 30e08ffba8SPatrick Williams# typing.Dict is used for type-hints. 31e08ffba8SPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional # noqa: F401 3202871c91SPatrick Williams 3341d86218SPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 3441d86218SPatrick Williams 35ee3c9eebSPatrick Williamstry: 36ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 37ee3c9eebSPatrick Williams from typing import TypedDict 3841d86218SPatrick Williamsexcept Exception: 39ee3c9eebSPatrick Williams 40ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 41ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 4241d86218SPatrick Williams def __init_subclass__(cls, **kwargs: Any) -> None: 43ee3c9eebSPatrick Williams super().__init_subclass__() 44ee3c9eebSPatrick Williams 45ee3c9eebSPatrick Williams 46ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 47aae36d18SPatrick Williamsprefix = "/usr/local" 4802871c91SPatrick Williamsproc_count = nproc().strip() 4902871c91SPatrick Williams 50ee3c9eebSPatrick Williams 51ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 52ee3c9eebSPatrick Williams """Package Definition for packages dictionary.""" 53ee3c9eebSPatrick Williams 54ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 55ee3c9eebSPatrick Williams rev: str 56ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 57ee3c9eebSPatrick Williams url: Callable[[str, str], str] 58ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 59ee3c9eebSPatrick Williams depends: Iterable[str] 60ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 61ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 62ee3c9eebSPatrick Williams build_type: str 63ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 64ee3c9eebSPatrick Williams build_steps: Iterable[str] 65ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 66ee3c9eebSPatrick Williams config_flags: Iterable[str] 67ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 68ee3c9eebSPatrick Williams config_env: Iterable[str] 69ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 70ee3c9eebSPatrick Williams # before config / build / install. 71ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 726bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 736bce2ca1SPatrick Williams custom_post_install: Iterable[str] 74ee3c9eebSPatrick Williams 75ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 76ee3c9eebSPatrick Williams __tag: str 77ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 78ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 79ee3c9eebSPatrick Williams 8002871c91SPatrick Williams 817204324cSPatrick Williams# Packages to include in image. 827204324cSPatrick Williamspackages = { 83ee3c9eebSPatrick Williams "boost": PackageDef( 8405806f5eSAndrew Geissler rev="1.84.0", 85ee3c9eebSPatrick Williams url=( 8638b46870SAndrew Geissler lambda pkg, rev: f"https://github.com/boostorg/{pkg}/releases/download/{pkg}-{rev}/{pkg}-{rev}.tar.gz" 872abc4a48SPatrick Williams ), 88ee3c9eebSPatrick Williams build_type="custom", 89ee3c9eebSPatrick Williams build_steps=[ 90e08ffba8SPatrick Williams ( 91e08ffba8SPatrick Williams "./bootstrap.sh" 9242ff4328SEd Tanous f" --prefix={prefix} --with-libraries=context,coroutine,url" 93e08ffba8SPatrick Williams ), 94aae36d18SPatrick Williams "./b2", 95aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 96aae36d18SPatrick Williams ], 97ee3c9eebSPatrick Williams ), 98ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 99c1977839SPatrick Williams rev="v1.3.2", 100ee3c9eebSPatrick Williams build_type="custom", 101ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 102ee3c9eebSPatrick Williams ), 103c7198558SEd Tanous "danmar/cppcheck": PackageDef( 10451021786SPatrick Williams rev="2.12.1", 105c7198558SEd Tanous build_type="cmake", 106c7198558SEd Tanous ), 107ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 108fc39733aSPatrick Williams rev="v2.3.2", 109ee3c9eebSPatrick Williams build_type="cmake", 110ee3c9eebSPatrick Williams config_flags=[ 111aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 112aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 113aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 114aae36d18SPatrick Williams ], 115ee3c9eebSPatrick Williams ), 116ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 117c061e07bSPatrick Williams rev="10.1.1", 118ee3c9eebSPatrick Williams build_type="cmake", 119ee3c9eebSPatrick Williams config_flags=[ 120aae36d18SPatrick Williams "-DFMT_DOC=OFF", 121aae36d18SPatrick Williams "-DFMT_TEST=OFF", 122aae36d18SPatrick Williams ], 123ee3c9eebSPatrick Williams ), 124ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 125cb09974cSPatrick Williams rev="4.2.4", 126ee3c9eebSPatrick Williams build_type="custom", 127ee3c9eebSPatrick Williams build_steps=[ 128aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 129aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 130aae36d18SPatrick Williams ], 131ee3c9eebSPatrick Williams ), 132ee3c9eebSPatrick Williams "google/googletest": PackageDef( 133fdf243bbSPatrick Williams rev="v1.14.0", 134ee3c9eebSPatrick Williams build_type="cmake", 1354dd32c02SWilliam A. Kennington III config_env=["CXXFLAGS=-std=c++20"], 136ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 137ee3c9eebSPatrick Williams ), 138178b4b29SEd Tanous "nghttp2/nghttp2": PackageDef( 139abb106a9SEd Tanous rev="v1.61.0", 140178b4b29SEd Tanous build_type="cmake", 141178b4b29SEd Tanous config_env=["CXXFLAGS=-std=c++20"], 142178b4b29SEd Tanous config_flags=[ 143178b4b29SEd Tanous "-DENABLE_LIB_ONLY=ON", 144178b4b29SEd Tanous "-DENABLE_STATIC_LIB=ON", 145178b4b29SEd Tanous ], 146178b4b29SEd Tanous ), 147ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 148c1977839SPatrick Williams rev="v3.11.2", 1496bce2ca1SPatrick Williams build_type="cmake", 1506bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1516bce2ca1SPatrick Williams custom_post_install=[ 152e08ffba8SPatrick Williams ( 153e08ffba8SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp" 154e08ffba8SPatrick Williams f" {prefix}/include/json.hpp" 155e08ffba8SPatrick Williams ), 156aae36d18SPatrick Williams ], 157ee3c9eebSPatrick Williams ), 158058e3a34SPrzemyslaw Czarnowski "json-c/json-c": PackageDef( 159eee65beeSPatrick Williams rev="json-c-0.17-20230812", 160058e3a34SPrzemyslaw Czarnowski build_type="cmake", 161058e3a34SPrzemyslaw Czarnowski ), 162ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 163c042132cSPatrick Williams rev="LibVNCServer-0.9.14", 164ee3c9eebSPatrick Williams build_type="cmake", 165ee3c9eebSPatrick Williams ), 166ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 167c1977839SPatrick Williams rev="9.0.0", 168ee3c9eebSPatrick Williams build_type="cmake", 169ee3c9eebSPatrick Williams ), 170ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 1715a2c113cSPatrick Williams rev="v1.0.1", 172ee3c9eebSPatrick Williams build_type="cmake", 173ee3c9eebSPatrick Williams config_flags=[ 1740eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1750eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 176aae36d18SPatrick Williams ], 177ee3c9eebSPatrick Williams ), 178ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 179ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 180ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 181ee3c9eebSPatrick Williams build_type="meson", 182ee3c9eebSPatrick Williams config_flags=[ 183aae36d18SPatrick Williams "-Dexamples=false", 184aae36d18SPatrick Williams "-Dtests=disabled", 185aae36d18SPatrick Williams ], 186ee3c9eebSPatrick Williams ), 187ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 188ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 189ee3c9eebSPatrick Williams build_type="meson", 1904fe87776SWilliam A. Kennington III config_flags=["-Dgenerate_md=false"], 191ee3c9eebSPatrick Williams ), 192ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 193ee3c9eebSPatrick Williams depends=[ 19483394610SPatrick Williams "USCiLab/cereal", 19583394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 19683394610SPatrick Williams "openbmc/sdbusplus", 19783394610SPatrick Williams "openbmc/sdeventplus", 198aae36d18SPatrick Williams ], 199f79ce4c4SPatrick Williams build_type="meson", 200ee3c9eebSPatrick Williams config_flags=[ 2016c98f280SWilliam A. Kennington III "-Dlibonly=true", 2026c98f280SWilliam A. Kennington III "-Dtests=disabled", 2035eabdae9SPatrick Williams f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml", 204aae36d18SPatrick Williams ], 205ee3c9eebSPatrick Williams ), 206ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 207ee3c9eebSPatrick Williams depends=[ 20811e5762cSBrad Bishop "CLIUtils/CLI11", 20970af95caSPatrick Williams "boost", 21083394610SPatrick Williams "leethomason/tinyxml2", 21170af95caSPatrick Williams "openbmc/phosphor-dbus-interfaces", 21283394610SPatrick Williams "openbmc/phosphor-logging", 21383394610SPatrick Williams "openbmc/sdbusplus", 214aae36d18SPatrick Williams ], 2151197e359SBrad Bishop build_type="meson", 2161197e359SBrad Bishop config_flags=[ 2171197e359SBrad Bishop "-Dtests=disabled", 2181197e359SBrad Bishop ], 219ee3c9eebSPatrick Williams ), 220c02ff271SJason M. Bills "openbmc/libpeci": PackageDef( 221c02ff271SJason M. Bills build_type="meson", 222c02ff271SJason M. Bills config_flags=[ 223c02ff271SJason M. Bills "-Draw-peci=disabled", 224c02ff271SJason M. Bills ], 225c02ff271SJason M. Bills ), 2261c19e453SManojkiran Eda "openbmc/libpldm": PackageDef( 227ee3c9eebSPatrick Williams build_type="meson", 228ee3c9eebSPatrick Williams config_flags=[ 22929d69bb0SAndrew Jeffery "-Dabi=deprecated,stable", 230aae36d18SPatrick Williams "-Doem-ibm=enabled", 231aae36d18SPatrick Williams "-Dtests=disabled", 232aae36d18SPatrick Williams ], 233ee3c9eebSPatrick Williams ), 234ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 235ee3c9eebSPatrick Williams build_type="meson", 236ee3c9eebSPatrick Williams custom_post_dl=[ 237aae36d18SPatrick Williams "cd tools", 238aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 239aae36d18SPatrick Williams "cd ..", 240aae36d18SPatrick Williams ], 241ee3c9eebSPatrick Williams config_flags=[ 242aae36d18SPatrick Williams "-Dexamples=disabled", 243aae36d18SPatrick Williams "-Dtests=disabled", 244aae36d18SPatrick Williams ], 245b16f3e20SPatrick Williams ), 246ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 24770af95caSPatrick Williams depends=[ 24870af95caSPatrick Williams "Naios/function2", 24970af95caSPatrick Williams "openbmc/stdplus", 25070af95caSPatrick Williams ], 251ee3c9eebSPatrick Williams build_type="meson", 252ee3c9eebSPatrick Williams config_flags=[ 253ee3c9eebSPatrick Williams "-Dexamples=false", 254ee3c9eebSPatrick Williams "-Dtests=disabled", 255ee3c9eebSPatrick Williams ], 256ee3c9eebSPatrick Williams ), 257ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 25870af95caSPatrick Williams depends=[ 25970af95caSPatrick Williams "fmtlib/fmt", 260ca1bf0c0SWilliam A. Kennington III "google/googletest", 261ca1bf0c0SWilliam A. Kennington III "Naios/function2", 26270af95caSPatrick Williams ], 263ee3c9eebSPatrick Williams build_type="meson", 264ee3c9eebSPatrick Williams config_flags=[ 265ee3c9eebSPatrick Williams "-Dexamples=false", 266ee3c9eebSPatrick Williams "-Dtests=disabled", 267ca1bf0c0SWilliam A. Kennington III "-Dgtest=enabled", 268ee3c9eebSPatrick Williams ], 269ee3c9eebSPatrick Williams ), 270ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 27102871c91SPatrick Williams 27202871c91SPatrick Williams# Define common flags used for builds 27302871c91SPatrick Williamsconfigure_flags = " ".join( 27402871c91SPatrick Williams [ 27502871c91SPatrick Williams f"--prefix={prefix}", 27602871c91SPatrick Williams ] 27702871c91SPatrick Williams) 27802871c91SPatrick Williamscmake_flags = " ".join( 27902871c91SPatrick Williams [ 28002871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 2810f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 28202871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 2830f2086b3SPatrick Williams "-GNinja", 2840f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 28502871c91SPatrick Williams ] 28602871c91SPatrick Williams) 28702871c91SPatrick Williamsmeson_flags = " ".join( 28802871c91SPatrick Williams [ 28902871c91SPatrick Williams "--wrap-mode=nodownload", 29002871c91SPatrick Williams f"-Dprefix={prefix}", 29102871c91SPatrick Williams ] 29202871c91SPatrick Williams) 29302871c91SPatrick Williams 294ee3c9eebSPatrick Williams 295ee3c9eebSPatrick Williamsclass Package(threading.Thread): 296ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 297ee3c9eebSPatrick Williams 298ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 299ee3c9eebSPatrick Williams Package.generate_all(). 300ee3c9eebSPatrick Williams """ 301ee3c9eebSPatrick Williams 302ee3c9eebSPatrick Williams # Copy the packages dictionary. 303ee3c9eebSPatrick Williams packages = packages.copy() 304ee3c9eebSPatrick Williams 305ee3c9eebSPatrick Williams # Lock used for thread-safety. 306ee3c9eebSPatrick Williams lock = threading.Lock() 307ee3c9eebSPatrick Williams 308ee3c9eebSPatrick Williams def __init__(self, pkg: str): 309ee3c9eebSPatrick Williams """pkg - The name of this package (ex. foo/bar )""" 310ee3c9eebSPatrick Williams super(Package, self).__init__() 311ee3c9eebSPatrick Williams 312ee3c9eebSPatrick Williams self.package = pkg 313ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 314ee3c9eebSPatrick Williams 315ee3c9eebSPatrick Williams # Reference to this package's 316ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 317ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 318ee3c9eebSPatrick Williams 319ee3c9eebSPatrick Williams def run(self) -> None: 320ee3c9eebSPatrick Williams """Thread 'run' function. Builds the Docker stage.""" 321ee3c9eebSPatrick Williams 322ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 323ee3c9eebSPatrick Williams self._update_rev() 324ee3c9eebSPatrick Williams 325ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 326ee3c9eebSPatrick Williams # This section is locked because we are looking into another 327ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 328ee3c9eebSPatrick Williams Package.lock.acquire() 329ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 330ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 331ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 332ee3c9eebSPatrick Williams ] 333ee3c9eebSPatrick Williams Package.lock.release() 334ee3c9eebSPatrick Williams 335ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 336ee3c9eebSPatrick Williams # for the "COPY" commands. 337ee3c9eebSPatrick Williams for deppkg in deps: 338ee3c9eebSPatrick Williams deppkg.join() 339ee3c9eebSPatrick Williams 340ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 341ee3c9eebSPatrick Williams dockerfile = f""" 342ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 343ee3c9eebSPatrick Williams{self._df_copycmds()} 344ee3c9eebSPatrick Williams{self._df_build()} 345ee3c9eebSPatrick Williams""" 346ee3c9eebSPatrick Williams 347ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 348ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 349ee3c9eebSPatrick Williams # which can be accessed by other threads. 350ee3c9eebSPatrick Williams Package.lock.acquire() 351ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 352ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 353ee3c9eebSPatrick Williams Package.lock.release() 354ee3c9eebSPatrick Williams 355ee3c9eebSPatrick Williams # Do the build / save any exceptions. 356ee3c9eebSPatrick Williams try: 357ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 358ee3c9eebSPatrick Williams except Exception as e: 359ee3c9eebSPatrick Williams self.exception = e 360ee3c9eebSPatrick Williams 361ee3c9eebSPatrick Williams @classmethod 362ee3c9eebSPatrick Williams def generate_all(cls) -> None: 363ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 364ee3c9eebSPatrick Williams 365ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 366ee3c9eebSPatrick Williams package 'depends' specifications. 367ee3c9eebSPatrick Williams """ 368ee3c9eebSPatrick Williams 369ee3c9eebSPatrick Williams # Create a Package for each defined package. 370ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 371ee3c9eebSPatrick Williams 372ee3c9eebSPatrick Williams # Start building them all. 3736dbd7807SPatrick Williams # This section is locked because threads depend on each other, 3746dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 3756dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 3766dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 3776dbd7807SPatrick Williams Package.lock.acquire() 378ee3c9eebSPatrick Williams for t in pkg_threads: 379ee3c9eebSPatrick Williams t.start() 3806dbd7807SPatrick Williams Package.lock.release() 381ee3c9eebSPatrick Williams 382ee3c9eebSPatrick Williams # Wait for completion. 383ee3c9eebSPatrick Williams for t in pkg_threads: 384ee3c9eebSPatrick Williams t.join() 385ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 386ee3c9eebSPatrick Williams if t.exception: 387ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 388ee3c9eebSPatrick Williams raise t.exception 389ee3c9eebSPatrick Williams 390ee3c9eebSPatrick Williams @staticmethod 391ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 392ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 393ee3c9eebSPatrick Williams into the final image. 394ee3c9eebSPatrick Williams """ 395ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 396ee3c9eebSPatrick Williams 397ee3c9eebSPatrick Williams @classmethod 398ee3c9eebSPatrick Williams def depcache(cls) -> str: 399ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 400ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 401ee3c9eebSPatrick Williams """ 402ee3c9eebSPatrick Williams 403ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 404ee3c9eebSPatrick Williams depcache = "" 405ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 406ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 407ee3c9eebSPatrick Williams return depcache 408ee3c9eebSPatrick Williams 409ee3c9eebSPatrick Williams def _update_rev(self) -> None: 410ee3c9eebSPatrick Williams """Look up the HEAD for missing a static rev.""" 411ee3c9eebSPatrick Williams 412ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 413ee3c9eebSPatrick Williams return 414ee3c9eebSPatrick Williams 41565b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 41665b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 41765b21fb9SPatrick Williams print( 41865b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 41965b21fb9SPatrick Williams file=sys.stderr, 42065b21fb9SPatrick Williams ) 42165b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 42265b21fb9SPatrick Williams return 42365b21fb9SPatrick Williams 424ee3c9eebSPatrick Williams # Ask Github for all the branches. 42505fb2a0aSPatrick Williams lookup = git( 42605fb2a0aSPatrick Williams "ls-remote", "--heads", f"https://github.com/{self.package}" 42705fb2a0aSPatrick Williams ) 428ee3c9eebSPatrick Williams 429ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 430ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 431ee3c9eebSPatrick Williams Package.lock.acquire() 432ee3c9eebSPatrick Williams for line in lookup.split("\n"): 433f3d27e64SAndrew Geissler if re.fullmatch(f".*{branch}$", line.strip()): 434ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 435f3d27e64SAndrew Geissler break 436c7d73646SPatrick Williams elif ( 437c7d73646SPatrick Williams "refs/heads/master" in line or "refs/heads/main" in line 438c7d73646SPatrick Williams ) and "rev" not in self.pkg_def: 439ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 440ee3c9eebSPatrick Williams Package.lock.release() 441ee3c9eebSPatrick Williams 442ee3c9eebSPatrick Williams def _stagename(self) -> str: 443ee3c9eebSPatrick Williams """Create a name for the Docker stage associated with this pkg.""" 444ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 445ee3c9eebSPatrick Williams 446ee3c9eebSPatrick Williams def _url(self) -> str: 447ee3c9eebSPatrick Williams """Get the URL for this package.""" 448ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 449ee3c9eebSPatrick Williams 450ee3c9eebSPatrick Williams # If the lambda exists, call it. 451ee3c9eebSPatrick Williams if "url" in self.pkg_def: 452ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 453ee3c9eebSPatrick Williams 454ee3c9eebSPatrick Williams # Default to the github archive URL. 455ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 456ee3c9eebSPatrick Williams 457ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 458ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 459ee3c9eebSPatrick Williams 460ee3c9eebSPatrick Williams url = self._url() 461ee3c9eebSPatrick Williams if ".tar." not in url: 462ee3c9eebSPatrick Williams raise NotImplementedError( 463ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 464ee3c9eebSPatrick Williams ) 465ee3c9eebSPatrick Williams 466ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 467ee3c9eebSPatrick Williams 468ee3c9eebSPatrick Williams if url.endswith(".bz2"): 469ee3c9eebSPatrick Williams cmd += "j" 470ee3c9eebSPatrick Williams elif url.endswith(".gz"): 471ee3c9eebSPatrick Williams cmd += "z" 472ee3c9eebSPatrick Williams else: 473ee3c9eebSPatrick Williams raise NotImplementedError( 474ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 475ee3c9eebSPatrick Williams ) 476ee3c9eebSPatrick Williams 477ee3c9eebSPatrick Williams return cmd 478ee3c9eebSPatrick Williams 479ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 480ee3c9eebSPatrick Williams """Formulate the command necessary to 'cd' into the source dir.""" 481ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 482ee3c9eebSPatrick Williams 483ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 484ee3c9eebSPatrick Williams """Formulate the dockerfile snippet necessary to COPY all depends.""" 485ee3c9eebSPatrick Williams 486ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 487ee3c9eebSPatrick Williams return "" 488ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 489ee3c9eebSPatrick Williams 490ee3c9eebSPatrick Williams @staticmethod 491ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 492ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 493ee3c9eebSPatrick Williams packages into a Docker stage. 494ee3c9eebSPatrick Williams """ 495ee3c9eebSPatrick Williams 496ee3c9eebSPatrick Williams copy_cmds = "" 497ee3c9eebSPatrick Williams 498ee3c9eebSPatrick Williams # Sort the packages for consistency. 499ee3c9eebSPatrick Williams for p in sorted(pkgs): 500ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 501ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 502ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 503ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 504ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 505ee3c9eebSPatrick Williams 506ee3c9eebSPatrick Williams return copy_cmds 507ee3c9eebSPatrick Williams 508ee3c9eebSPatrick Williams def _df_build(self) -> str: 509ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 510ee3c9eebSPatrick Williams install a package into a Docker stage. 511ee3c9eebSPatrick Williams """ 512ee3c9eebSPatrick Williams 513ee3c9eebSPatrick Williams # Download and extract source. 514ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 515ee3c9eebSPatrick Williams 516ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 517ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 518ee3c9eebSPatrick Williams if custom_post_dl: 519ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 520ee3c9eebSPatrick Williams 521ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 522ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 523ee3c9eebSPatrick Williams if build_type == "autoconf": 524ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 525ee3c9eebSPatrick Williams elif build_type == "cmake": 526ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 527ee3c9eebSPatrick Williams elif build_type == "custom": 528ee3c9eebSPatrick Williams result += self._cmd_build_custom() 529ee3c9eebSPatrick Williams elif build_type == "make": 530ee3c9eebSPatrick Williams result += self._cmd_build_make() 531ee3c9eebSPatrick Williams elif build_type == "meson": 532ee3c9eebSPatrick Williams result += self._cmd_build_meson() 533ee3c9eebSPatrick Williams else: 534ee3c9eebSPatrick Williams raise NotImplementedError( 535ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 536ee3c9eebSPatrick Williams ) 537ee3c9eebSPatrick Williams 5386bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5396bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5406bce2ca1SPatrick Williams if custom_post_install: 5416bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5426bce2ca1SPatrick Williams 543ee3c9eebSPatrick Williams return result 544ee3c9eebSPatrick Williams 545ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 546ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 547ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 548ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 549ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 550ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 551ee3c9eebSPatrick Williams return result 552ee3c9eebSPatrick Williams 553ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 554ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 555ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 556ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 557ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 558ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 559ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 560ee3c9eebSPatrick Williams result += "cd .." 561ee3c9eebSPatrick Williams return result 562ee3c9eebSPatrick Williams 563ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 564ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 565ee3c9eebSPatrick Williams 566ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 567ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 568ee3c9eebSPatrick Williams 569ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 570ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 571ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 572e2da11adSAndrew Jeffery result = f"{env} meson setup builddir {meson_flags} {options} && " 573ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 574ee3c9eebSPatrick Williams return result 575ee3c9eebSPatrick Williams 576ee3c9eebSPatrick Williams 577ee3c9eebSPatrick Williamsclass Docker: 578ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 579ee3c9eebSPatrick Williams 580ee3c9eebSPatrick Williams @staticmethod 581ee3c9eebSPatrick Williams def timestamp() -> str: 582ee3c9eebSPatrick Williams """Generate a timestamp for today using the ISO week.""" 583ee3c9eebSPatrick Williams today = date.today().isocalendar() 584ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 585ee3c9eebSPatrick Williams 586ee3c9eebSPatrick Williams @staticmethod 58741d86218SPatrick Williams def tagname(pkgname: Optional[str], dockerfile: str) -> str: 588ee3c9eebSPatrick Williams """Generate a tag name for a package using a hash of the Dockerfile.""" 589ee3c9eebSPatrick Williams result = docker_image_name 590ee3c9eebSPatrick Williams if pkgname: 591ee3c9eebSPatrick Williams result += "-" + pkgname 592ee3c9eebSPatrick Williams 593ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 594ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 595ee3c9eebSPatrick Williams 596ee3c9eebSPatrick Williams return result 597ee3c9eebSPatrick Williams 598ee3c9eebSPatrick Williams @staticmethod 599ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 60022e6110bSAndrew Geissler """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 601ee3c9eebSPatrick Williams 602ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 603ee3c9eebSPatrick Williams if not force_build: 604ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 60505fb2a0aSPatrick Williams print( 60605fb2a0aSPatrick Williams f"Image {tag} already exists. Skipping.", file=sys.stderr 60705fb2a0aSPatrick Williams ) 608ee3c9eebSPatrick Williams return 609ee3c9eebSPatrick Williams 610ee3c9eebSPatrick Williams # Build it. 611ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 612ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 613a6ebc6e2SManojkiran Eda # progress but not pollute stdout. Later on we output the final 614ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 615ee3c9eebSPatrick Williams # 616ee3c9eebSPatrick Williams # Other unusual flags: 617ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 618ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 619ee3c9eebSPatrick Williams docker.build( 620ee3c9eebSPatrick Williams proxy_args, 621ee3c9eebSPatrick Williams "--network=host", 622ee3c9eebSPatrick Williams "--force-rm", 623ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 624ee3c9eebSPatrick Williams "-t", 625ee3c9eebSPatrick Williams tag, 626ee3c9eebSPatrick Williams "-", 627ee3c9eebSPatrick Williams _in=dockerfile, 628ee3c9eebSPatrick Williams _out=( 629ee3c9eebSPatrick Williams lambda line: print( 630ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 631ee3c9eebSPatrick Williams ) 632ee3c9eebSPatrick Williams ), 63388dd7929SJonathan Doman _err_to_out=True, 634ee3c9eebSPatrick Williams ) 635ee3c9eebSPatrick Williams 636ee3c9eebSPatrick Williams 637ee3c9eebSPatrick Williams# Read a bunch of environment variables. 63805fb2a0aSPatrick Williamsdocker_image_name = os.environ.get( 63905fb2a0aSPatrick Williams "DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test" 64005fb2a0aSPatrick Williams) 641ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 642ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 6437c95a37cSPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:noble") 644ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 645ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 646ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 647ee3c9eebSPatrick Williams 64865b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 64965b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 65065b21fb9SPatrick Williams 651d0dabc3eSAndrew Geissler# Ensure appropriate docker build output to see progress and identify 652d0dabc3eSAndrew Geissler# any issues 653d0dabc3eSAndrew Geissleros.environ["BUILDKIT_PROGRESS"] = "plain" 654d0dabc3eSAndrew Geissler 655ee3c9eebSPatrick Williams# Set up some common variables. 656ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 657ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 658ee3c9eebSPatrick Williamsgid = os.getgid() 659ee3c9eebSPatrick Williamsuid = os.getuid() 660ee3c9eebSPatrick Williams 6616825a018SJosh Lehan# Use well-known constants if user is root 6626825a018SJosh Lehanif username == "root": 6636825a018SJosh Lehan homedir = "/root" 6646825a018SJosh Lehan gid = 0 6656825a018SJosh Lehan uid = 0 6666825a018SJosh Lehan 667ee3c9eebSPatrick Williams# Determine the architecture for Docker. 668ee3c9eebSPatrick Williamsarch = uname("-m").strip() 669ee3c9eebSPatrick Williamsif arch == "ppc64le": 670ee3c9eebSPatrick Williams docker_base = "ppc64le/" 671ee3c9eebSPatrick Williamselif arch == "x86_64": 672ee3c9eebSPatrick Williams docker_base = "" 673051b05b7SThang Q. Nguyenelif arch == "aarch64": 674f98f1a8dSThang Q. Nguyen docker_base = "arm64v8/" 675ee3c9eebSPatrick Williamselse: 676ee3c9eebSPatrick Williams print( 677ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 678ee3c9eebSPatrick Williams file=sys.stderr, 679ee3c9eebSPatrick Williams ) 680ee3c9eebSPatrick Williams sys.exit(1) 681ee3c9eebSPatrick Williams 68202871c91SPatrick Williams# Special flags if setting up a deb mirror. 68302871c91SPatrick Williamsmirror = "" 68402871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 68502871c91SPatrick Williams mirror = f""" 686e08ffba8SPatrick WilliamsRUN echo "deb {ubuntu_mirror} \ 687e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME) \ 688e08ffba8SPatrick Williams main restricted universe multiverse" > /etc/apt/sources.list && \\ 689e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 690e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-updates \ 691e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 692e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 693e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-security \ 694e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 695e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 696e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-proposed \ 697e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 698e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 699e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-backports \ 700e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list 70102871c91SPatrick Williams""" 70202871c91SPatrick Williams 70302871c91SPatrick Williams# Special flags for proxying. 70402871c91SPatrick Williamsproxy_cmd = "" 70534ec77e8SAdrian Ambrożewiczproxy_keyserver = "" 70602871c91SPatrick Williamsproxy_args = [] 70702871c91SPatrick Williamsif http_proxy: 70802871c91SPatrick Williams proxy_cmd = f""" 70902871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 71002871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 71102871c91SPatrick Williams""" 71234ec77e8SAdrian Ambrożewicz proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 71334ec77e8SAdrian Ambrożewicz 71402871c91SPatrick Williams proxy_args.extend( 71502871c91SPatrick Williams [ 71602871c91SPatrick Williams "--build-arg", 71702871c91SPatrick Williams f"http_proxy={http_proxy}", 71802871c91SPatrick Williams "--build-arg", 719d461cd6aSLei YU f"https_proxy={http_proxy}", 72002871c91SPatrick Williams ] 72102871c91SPatrick Williams ) 72202871c91SPatrick Williams 723ee3c9eebSPatrick Williams# Create base Dockerfile. 724a18d9c57SPatrick Williamsdockerfile_base = f""" 725a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 72602871c91SPatrick Williams 72702871c91SPatrick Williams{mirror} 72802871c91SPatrick Williams 72902871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 73002871c91SPatrick Williams 7318949d3c3SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/" 73202871c91SPatrick Williams 733bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 734bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 735bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 736bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 737bb16ac14SPatrick Williams# the update. 738bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 739bb16ac14SPatrick Williams 74002871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 74102871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 74202871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 743575b5e4cSJagpal Singh Gill# Known issue with gpg to get keys via proxy - 744575b5e4cSJagpal Singh Gill# https://bugs.launchpad.net/ubuntu/+source/gnupg2/+bug/1788190, hence using 745575b5e4cSJagpal Singh Gill# curl to get keys. 74650837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 747938d303fSJian Zhang ( apt-get install -yy gpgv ubuntu-dbgsym-keyring || \ 748575b5e4cSJagpal Singh Gill ( apt-get install -yy dirmngr curl && \ 749575b5e4cSJagpal Singh Gill curl -sSL \ 750575b5e4cSJagpal Singh Gill 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622' \ 751575b5e4cSJagpal Singh Gill | apt-key add - )) 75202871c91SPatrick Williams 75302871c91SPatrick Williams# Parse the current repo list into a debug repo list 754e08ffba8SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' \ 755e08ffba8SPatrick Williams /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 75602871c91SPatrick Williams 75702871c91SPatrick Williams# Remove non-existent debug repos 75841d86218SPatrick WilliamsRUN sed -i '/-\\(backports\\|security\\) /d' /etc/apt/sources.list.d/debug.list 75902871c91SPatrick Williams 76002871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 76102871c91SPatrick Williams 76202871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 76358f1915eSAndrew Jeffery abi-compliance-checker \ 7648b112068SAndrew Jeffery abi-dumper \ 76502871c91SPatrick Williams autoconf \ 76602871c91SPatrick Williams autoconf-archive \ 767af49ed51SAndrew Geissler bison \ 768e2e62e5cSPatrick Williams clang-17 \ 769e2e62e5cSPatrick Williams clang-format-17 \ 770e2e62e5cSPatrick Williams clang-tidy-17 \ 771e2e62e5cSPatrick Williams clang-tools-17 \ 772af49ed51SAndrew Geissler cmake \ 773af49ed51SAndrew Geissler curl \ 774af49ed51SAndrew Geissler dbus \ 775af49ed51SAndrew Geissler device-tree-compiler \ 776af49ed51SAndrew Geissler flex \ 777961f148bSPatrick Williams g++-13 \ 778961f148bSPatrick Williams gcc-13 \ 779af49ed51SAndrew Geissler git \ 78002871c91SPatrick Williams iproute2 \ 781af49ed51SAndrew Geissler iputils-ping \ 782524a331cSManojkiran Eda libaudit-dev \ 783af49ed51SAndrew Geissler libc6-dbg \ 784af49ed51SAndrew Geissler libc6-dev \ 785af49ed51SAndrew Geissler libconfig++-dev \ 786af49ed51SAndrew Geissler libcryptsetup-dev \ 787af49ed51SAndrew Geissler libdbus-1-dev \ 788af49ed51SAndrew Geissler libevdev-dev \ 789af49ed51SAndrew Geissler libgpiod-dev \ 790af49ed51SAndrew Geissler libi2c-dev \ 791af49ed51SAndrew Geissler libjpeg-dev \ 792af49ed51SAndrew Geissler libjson-perl \ 793af49ed51SAndrew Geissler libldap2-dev \ 794af49ed51SAndrew Geissler libmimetic-dev \ 79502871c91SPatrick Williams libnl-3-dev \ 79602871c91SPatrick Williams libnl-genl-3-dev \ 79702871c91SPatrick Williams libpam0g-dev \ 79802871c91SPatrick Williams libpciaccess-dev \ 799af49ed51SAndrew Geissler libperlio-gzip-perl \ 800af49ed51SAndrew Geissler libpng-dev \ 801af49ed51SAndrew Geissler libprotobuf-dev \ 802af49ed51SAndrew Geissler libsnmp-dev \ 803af49ed51SAndrew Geissler libssl-dev \ 804af49ed51SAndrew Geissler libsystemd-dev \ 805af49ed51SAndrew Geissler libtool \ 806af49ed51SAndrew Geissler liburing-dev \ 80702871c91SPatrick Williams libxml2-utils \ 8080eedeedaSPatrick Williams libxml-simple-perl \ 809af49ed51SAndrew Geissler ninja-build \ 810af49ed51SAndrew Geissler npm \ 811af49ed51SAndrew Geissler pkg-config \ 812af49ed51SAndrew Geissler protobuf-compiler \ 813af49ed51SAndrew Geissler python3 \ 814af49ed51SAndrew Geissler python3-dev\ 815af49ed51SAndrew Geissler python3-git \ 816af49ed51SAndrew Geissler python3-mako \ 817af49ed51SAndrew Geissler python3-pip \ 81825ba1e2fSWilliam A. Kennington III python3-protobuf \ 819af49ed51SAndrew Geissler python3-setuptools \ 820af49ed51SAndrew Geissler python3-socks \ 821af49ed51SAndrew Geissler python3-yaml \ 8229adf68d6SJohn Wedig rsync \ 823af49ed51SAndrew Geissler shellcheck \ 8248dd1bfe6SEwelina Walkusz socat \ 825af49ed51SAndrew Geissler sudo \ 826af49ed51SAndrew Geissler systemd \ 827af49ed51SAndrew Geissler valgrind \ 828b565f825SAndrew Geissler vim \ 829af49ed51SAndrew Geissler wget \ 830af49ed51SAndrew Geissler xxd 83102871c91SPatrick Williams 832961f148bSPatrick WilliamsRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 13 \ 833961f148bSPatrick Williams --slave /usr/bin/g++ g++ /usr/bin/g++-13 \ 834961f148bSPatrick Williams --slave /usr/bin/gcov gcov /usr/bin/gcov-13 \ 835961f148bSPatrick Williams --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-13 \ 836961f148bSPatrick Williams --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-13 837961f148bSPatrick WilliamsRUN update-alternatives --remove cpp /usr/bin/cpp && \ 838961f148bSPatrick Williams update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-13 13 83902871c91SPatrick Williams 840e2e62e5cSPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 1000 \ 841e2e62e5cSPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-17 \ 842e2e62e5cSPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-17 \ 843e2e62e5cSPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-17 \ 844e08ffba8SPatrick Williams --slave /usr/bin/run-clang-tidy run-clang-tidy.py \ 845e2e62e5cSPatrick Williams /usr/bin/run-clang-tidy-17 \ 846e2e62e5cSPatrick Williams --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-17 84702871c91SPatrick Williams 84850837436SPatrick Williams""" 84950837436SPatrick Williams 85050837436SPatrick Williamsif is_automated_ci_build: 85150837436SPatrick Williams dockerfile_base += f""" 852a6ebc6e2SManojkiran Eda# Run an arbitrary command to pollute the docker cache regularly force us 85350837436SPatrick Williams# to re-run `apt-get update` daily. 854ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 85550837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 85650837436SPatrick Williams 85750837436SPatrick Williams""" 85850837436SPatrick Williams 85941d86218SPatrick Williamsdockerfile_base += """ 8605e4d8402SPatrick WilliamsRUN pip3 install --break-system-packages \ 861818023dfSPatrick Williams beautysh \ 862818023dfSPatrick Williams black \ 863818023dfSPatrick Williams codespell \ 864818023dfSPatrick Williams flake8 \ 865*2d8c551fSEwelina Walkusz gcovr \ 866818023dfSPatrick Williams gitlint \ 867818023dfSPatrick Williams inflection \ 868818023dfSPatrick Williams isort \ 869818023dfSPatrick Williams jsonschema \ 87016baaf73SPatrick Williams meson==1.3.0 \ 871818023dfSPatrick Williams requests 872b08ddf77SPatrick Williams 873b08ddf77SPatrick WilliamsRUN npm install -g \ 874d0757deaSXinnan Xie eslint@v8.56.0 eslint-plugin-json@v3.1.0 \ 8757d41f6d2SPatrick Williams markdownlint-cli@latest \ 876b08ddf77SPatrick Williams prettier@latest 877fb9948a3SEd Tanous""" 878fb9948a3SEd Tanous 879ee3c9eebSPatrick Williams# Build the base and stage docker images. 880ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 881ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 882ee3c9eebSPatrick WilliamsPackage.generate_all() 88302871c91SPatrick Williams 884ee3c9eebSPatrick Williams# Create the final Dockerfile. 885a18d9c57SPatrick Williamsdockerfile = f""" 88602871c91SPatrick Williams# Build the final output image 887a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 888ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 88902871c91SPatrick Williams 89002871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 89102871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 89202871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 893ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 89402871c91SPatrick Williams 89567cc0616SPatrick Williams# Ensure the group, user, and home directory are created (or rename them if 89667cc0616SPatrick Williams# they already exist). 89767cc0616SPatrick WilliamsRUN if grep -q ":{gid}:" /etc/group ; then \ 89867cc0616SPatrick Williams groupmod -n {username} $(awk -F : '{{ if ($3 == {gid}) {{ print $1 }} }}' /etc/group) ; \ 89967cc0616SPatrick Williams else \ 90067cc0616SPatrick Williams groupadd -f -g {gid} {username} ; \ 90167cc0616SPatrick Williams fi 90202871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 90367cc0616SPatrick WilliamsRUN if grep -q ":{uid}:" /etc/passwd ; then \ 90473b3ee91SPatrick Williams usermod -l {username} -d {homedir} -m $(awk -F : '{{ if ($3 == {uid}) {{ print $1 }} }}' /etc/passwd) ; \ 90567cc0616SPatrick Williams else \ 90667cc0616SPatrick Williams useradd -d {homedir} -m -u {uid} -g {gid} {username} ; \ 90767cc0616SPatrick Williams fi 90802871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 90902871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 91002871c91SPatrick Williams 911305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 912305a9a5dSAndrew Geissler# and data installs 9137bb00b13SAndrew GeisslerRUN chown -R {username}:{username} /usr/local/share 914305a9a5dSAndrew Geissler 915ab4fee83SJonathan Doman# Update library cache 916ab4fee83SJonathan DomanRUN ldconfig 917ab4fee83SJonathan Doman 91802871c91SPatrick Williams{proxy_cmd} 91902871c91SPatrick Williams 92002871c91SPatrick WilliamsRUN /bin/bash 92102871c91SPatrick Williams""" 92202871c91SPatrick Williams 923a18d9c57SPatrick Williams# Do the final docker build 924ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 925ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 926ee3c9eebSPatrick Williams 92700536fbeSPatrick Williams# Print the tag of the final image. 92800536fbeSPatrick Williamsprint(docker_final_img_name) 929