102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1050837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1150837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1250837436SPatrick Williams# (ex. Jenkins)> 1302871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1402871c91SPatrick Williams# repositories> 1502871c91SPatrick Williams# default is master, which will be used if input branch not 1602871c91SPatrick Williams# provided or not found 1702871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1802871c91SPatrick Williams# default ones in /etc/apt/sources.list> 1902871c91SPatrick Williams# default is empty, and no mirror is used. 2002871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2102871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2202871c91SPatrick Williams 2302871c91SPatrick Williamsimport os 24f3d27e64SAndrew Geisslerimport re 2502871c91SPatrick Williamsimport sys 26b16f3e20SPatrick Williamsimport threading 27a18d9c57SPatrick Williamsfrom datetime import date 28a18d9c57SPatrick Williamsfrom hashlib import sha256 29e08ffba8SPatrick Williams 30e08ffba8SPatrick Williams# typing.Dict is used for type-hints. 31e08ffba8SPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional # noqa: F401 3202871c91SPatrick Williams 3341d86218SPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 3441d86218SPatrick Williams 35ee3c9eebSPatrick Williamstry: 36ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 37ee3c9eebSPatrick Williams from typing import TypedDict 3841d86218SPatrick Williamsexcept Exception: 39ee3c9eebSPatrick Williams 40ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 41ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 4241d86218SPatrick Williams def __init_subclass__(cls, **kwargs: Any) -> None: 43ee3c9eebSPatrick Williams super().__init_subclass__() 44ee3c9eebSPatrick Williams 45ee3c9eebSPatrick Williams 46ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 47aae36d18SPatrick Williamsprefix = "/usr/local" 4802871c91SPatrick Williamsproc_count = nproc().strip() 4902871c91SPatrick Williams 50ee3c9eebSPatrick Williams 51ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 52ee3c9eebSPatrick Williams """Package Definition for packages dictionary.""" 53ee3c9eebSPatrick Williams 54ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 55ee3c9eebSPatrick Williams rev: str 56ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 57ee3c9eebSPatrick Williams url: Callable[[str, str], str] 58ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 59ee3c9eebSPatrick Williams depends: Iterable[str] 60ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 61ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 62ee3c9eebSPatrick Williams build_type: str 63ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 64ee3c9eebSPatrick Williams build_steps: Iterable[str] 65ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 66ee3c9eebSPatrick Williams config_flags: Iterable[str] 67ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 68ee3c9eebSPatrick Williams config_env: Iterable[str] 69ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 70ee3c9eebSPatrick Williams # before config / build / install. 71ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 726bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 736bce2ca1SPatrick Williams custom_post_install: Iterable[str] 74ee3c9eebSPatrick Williams 75ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 76ee3c9eebSPatrick Williams __tag: str 77ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 78ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 79ee3c9eebSPatrick Williams 8002871c91SPatrick Williams 817204324cSPatrick Williams# Packages to include in image. 827204324cSPatrick Williamspackages = { 83ee3c9eebSPatrick Williams "boost": PackageDef( 8405806f5eSAndrew Geissler rev="1.84.0", 85ee3c9eebSPatrick Williams url=( 8638b46870SAndrew Geissler lambda pkg, rev: f"https://github.com/boostorg/{pkg}/releases/download/{pkg}-{rev}/{pkg}-{rev}.tar.gz" 872abc4a48SPatrick Williams ), 88ee3c9eebSPatrick Williams build_type="custom", 89ee3c9eebSPatrick Williams build_steps=[ 90e08ffba8SPatrick Williams ( 91e08ffba8SPatrick Williams "./bootstrap.sh" 9242ff4328SEd Tanous f" --prefix={prefix} --with-libraries=context,coroutine,url" 93e08ffba8SPatrick Williams ), 94aae36d18SPatrick Williams "./b2", 95aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 96aae36d18SPatrick Williams ], 97ee3c9eebSPatrick Williams ), 98ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 99c1977839SPatrick Williams rev="v1.3.2", 100ee3c9eebSPatrick Williams build_type="custom", 101ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 102ee3c9eebSPatrick Williams ), 103c7198558SEd Tanous "danmar/cppcheck": PackageDef( 10451021786SPatrick Williams rev="2.12.1", 105c7198558SEd Tanous build_type="cmake", 106c7198558SEd Tanous ), 107ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 108fc39733aSPatrick Williams rev="v2.3.2", 109ee3c9eebSPatrick Williams build_type="cmake", 110ee3c9eebSPatrick Williams config_flags=[ 111aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 112aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 113aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 114aae36d18SPatrick Williams ], 115ee3c9eebSPatrick Williams ), 116ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 117c061e07bSPatrick Williams rev="10.1.1", 118ee3c9eebSPatrick Williams build_type="cmake", 119ee3c9eebSPatrick Williams config_flags=[ 120aae36d18SPatrick Williams "-DFMT_DOC=OFF", 121aae36d18SPatrick Williams "-DFMT_TEST=OFF", 122aae36d18SPatrick Williams ], 123ee3c9eebSPatrick Williams ), 124ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 125cb09974cSPatrick Williams rev="4.2.4", 126ee3c9eebSPatrick Williams build_type="custom", 127ee3c9eebSPatrick Williams build_steps=[ 128aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 129aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 130aae36d18SPatrick Williams ], 131ee3c9eebSPatrick Williams ), 132ee3c9eebSPatrick Williams "google/googletest": PackageDef( 133fdf243bbSPatrick Williams rev="v1.14.0", 134ee3c9eebSPatrick Williams build_type="cmake", 1354dd32c02SWilliam A. Kennington III config_env=["CXXFLAGS=-std=c++20"], 136ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 137ee3c9eebSPatrick Williams ), 138178b4b29SEd Tanous "nghttp2/nghttp2": PackageDef( 139abb106a9SEd Tanous rev="v1.61.0", 140178b4b29SEd Tanous build_type="cmake", 141178b4b29SEd Tanous config_env=["CXXFLAGS=-std=c++20"], 142178b4b29SEd Tanous config_flags=[ 143178b4b29SEd Tanous "-DENABLE_LIB_ONLY=ON", 144178b4b29SEd Tanous "-DENABLE_STATIC_LIB=ON", 145178b4b29SEd Tanous ], 146178b4b29SEd Tanous ), 147ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 148c1977839SPatrick Williams rev="v3.11.2", 1496bce2ca1SPatrick Williams build_type="cmake", 1506bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1516bce2ca1SPatrick Williams custom_post_install=[ 152e08ffba8SPatrick Williams ( 153e08ffba8SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp" 154e08ffba8SPatrick Williams f" {prefix}/include/json.hpp" 155e08ffba8SPatrick Williams ), 156aae36d18SPatrick Williams ], 157ee3c9eebSPatrick Williams ), 158058e3a34SPrzemyslaw Czarnowski "json-c/json-c": PackageDef( 159eee65beeSPatrick Williams rev="json-c-0.17-20230812", 160058e3a34SPrzemyslaw Czarnowski build_type="cmake", 161058e3a34SPrzemyslaw Czarnowski ), 162ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 163f01a7242SPatrick Williams rev="v1.16", 164ee3c9eebSPatrick Williams build_type="make", 165ee3c9eebSPatrick Williams ), 166ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 167c042132cSPatrick Williams rev="LibVNCServer-0.9.14", 168ee3c9eebSPatrick Williams build_type="cmake", 169ee3c9eebSPatrick Williams ), 170ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 171c1977839SPatrick Williams rev="9.0.0", 172ee3c9eebSPatrick Williams build_type="cmake", 173ee3c9eebSPatrick Williams ), 174ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 1755a2c113cSPatrick Williams rev="v1.0.1", 176ee3c9eebSPatrick Williams build_type="cmake", 177ee3c9eebSPatrick Williams config_flags=[ 1780eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1790eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 180aae36d18SPatrick Williams ], 181ee3c9eebSPatrick Williams ), 182ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 183ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 184ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 185ee3c9eebSPatrick Williams build_type="meson", 186ee3c9eebSPatrick Williams config_flags=[ 187aae36d18SPatrick Williams "-Dexamples=false", 188aae36d18SPatrick Williams "-Dtests=disabled", 189aae36d18SPatrick Williams ], 190ee3c9eebSPatrick Williams ), 191ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 192ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 193ee3c9eebSPatrick Williams build_type="meson", 1944fe87776SWilliam A. Kennington III config_flags=["-Dgenerate_md=false"], 195ee3c9eebSPatrick Williams ), 196ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 197ee3c9eebSPatrick Williams depends=[ 19883394610SPatrick Williams "USCiLab/cereal", 19983394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 20083394610SPatrick Williams "openbmc/sdbusplus", 20183394610SPatrick Williams "openbmc/sdeventplus", 202aae36d18SPatrick Williams ], 203f79ce4c4SPatrick Williams build_type="meson", 204ee3c9eebSPatrick Williams config_flags=[ 2056c98f280SWilliam A. Kennington III "-Dlibonly=true", 2066c98f280SWilliam A. Kennington III "-Dtests=disabled", 2075eabdae9SPatrick Williams f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml", 208aae36d18SPatrick Williams ], 209ee3c9eebSPatrick Williams ), 210ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 211ee3c9eebSPatrick Williams depends=[ 21211e5762cSBrad Bishop "CLIUtils/CLI11", 21370af95caSPatrick Williams "boost", 21483394610SPatrick Williams "leethomason/tinyxml2", 21570af95caSPatrick Williams "openbmc/phosphor-dbus-interfaces", 21683394610SPatrick Williams "openbmc/phosphor-logging", 21783394610SPatrick Williams "openbmc/sdbusplus", 218aae36d18SPatrick Williams ], 2191197e359SBrad Bishop build_type="meson", 2201197e359SBrad Bishop config_flags=[ 2211197e359SBrad Bishop "-Dtests=disabled", 2221197e359SBrad Bishop ], 223ee3c9eebSPatrick Williams ), 224c02ff271SJason M. Bills "openbmc/libpeci": PackageDef( 225c02ff271SJason M. Bills build_type="meson", 226c02ff271SJason M. Bills config_flags=[ 227c02ff271SJason M. Bills "-Draw-peci=disabled", 228c02ff271SJason M. Bills ], 229c02ff271SJason M. Bills ), 2301c19e453SManojkiran Eda "openbmc/libpldm": PackageDef( 231ee3c9eebSPatrick Williams build_type="meson", 232ee3c9eebSPatrick Williams config_flags=[ 23329d69bb0SAndrew Jeffery "-Dabi=deprecated,stable", 234aae36d18SPatrick Williams "-Doem-ibm=enabled", 235aae36d18SPatrick Williams "-Dtests=disabled", 236aae36d18SPatrick Williams ], 237ee3c9eebSPatrick Williams ), 238ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 239ee3c9eebSPatrick Williams build_type="meson", 240ee3c9eebSPatrick Williams custom_post_dl=[ 241aae36d18SPatrick Williams "cd tools", 242aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 243aae36d18SPatrick Williams "cd ..", 244aae36d18SPatrick Williams ], 245ee3c9eebSPatrick Williams config_flags=[ 246aae36d18SPatrick Williams "-Dexamples=disabled", 247aae36d18SPatrick Williams "-Dtests=disabled", 248aae36d18SPatrick Williams ], 249b16f3e20SPatrick Williams ), 250ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 25170af95caSPatrick Williams depends=[ 25270af95caSPatrick Williams "Naios/function2", 25370af95caSPatrick Williams "openbmc/stdplus", 25470af95caSPatrick Williams ], 255ee3c9eebSPatrick Williams build_type="meson", 256ee3c9eebSPatrick Williams config_flags=[ 257ee3c9eebSPatrick Williams "-Dexamples=false", 258ee3c9eebSPatrick Williams "-Dtests=disabled", 259ee3c9eebSPatrick Williams ], 260ee3c9eebSPatrick Williams ), 261ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 26270af95caSPatrick Williams depends=[ 26370af95caSPatrick Williams "fmtlib/fmt", 264ca1bf0c0SWilliam A. Kennington III "google/googletest", 265ca1bf0c0SWilliam A. Kennington III "Naios/function2", 26670af95caSPatrick Williams ], 267ee3c9eebSPatrick Williams build_type="meson", 268ee3c9eebSPatrick Williams config_flags=[ 269ee3c9eebSPatrick Williams "-Dexamples=false", 270ee3c9eebSPatrick Williams "-Dtests=disabled", 271ca1bf0c0SWilliam A. Kennington III "-Dgtest=enabled", 272ee3c9eebSPatrick Williams ], 273ee3c9eebSPatrick Williams ), 274ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 27502871c91SPatrick Williams 27602871c91SPatrick Williams# Define common flags used for builds 27702871c91SPatrick Williamsconfigure_flags = " ".join( 27802871c91SPatrick Williams [ 27902871c91SPatrick Williams f"--prefix={prefix}", 28002871c91SPatrick Williams ] 28102871c91SPatrick Williams) 28202871c91SPatrick Williamscmake_flags = " ".join( 28302871c91SPatrick Williams [ 28402871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 2850f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 28602871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 2870f2086b3SPatrick Williams "-GNinja", 2880f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 28902871c91SPatrick Williams ] 29002871c91SPatrick Williams) 29102871c91SPatrick Williamsmeson_flags = " ".join( 29202871c91SPatrick Williams [ 29302871c91SPatrick Williams "--wrap-mode=nodownload", 29402871c91SPatrick Williams f"-Dprefix={prefix}", 29502871c91SPatrick Williams ] 29602871c91SPatrick Williams) 29702871c91SPatrick Williams 298ee3c9eebSPatrick Williams 299ee3c9eebSPatrick Williamsclass Package(threading.Thread): 300ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 301ee3c9eebSPatrick Williams 302ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 303ee3c9eebSPatrick Williams Package.generate_all(). 304ee3c9eebSPatrick Williams """ 305ee3c9eebSPatrick Williams 306ee3c9eebSPatrick Williams # Copy the packages dictionary. 307ee3c9eebSPatrick Williams packages = packages.copy() 308ee3c9eebSPatrick Williams 309ee3c9eebSPatrick Williams # Lock used for thread-safety. 310ee3c9eebSPatrick Williams lock = threading.Lock() 311ee3c9eebSPatrick Williams 312ee3c9eebSPatrick Williams def __init__(self, pkg: str): 313ee3c9eebSPatrick Williams """pkg - The name of this package (ex. foo/bar )""" 314ee3c9eebSPatrick Williams super(Package, self).__init__() 315ee3c9eebSPatrick Williams 316ee3c9eebSPatrick Williams self.package = pkg 317ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 318ee3c9eebSPatrick Williams 319ee3c9eebSPatrick Williams # Reference to this package's 320ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 321ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 322ee3c9eebSPatrick Williams 323ee3c9eebSPatrick Williams def run(self) -> None: 324ee3c9eebSPatrick Williams """Thread 'run' function. Builds the Docker stage.""" 325ee3c9eebSPatrick Williams 326ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 327ee3c9eebSPatrick Williams self._update_rev() 328ee3c9eebSPatrick Williams 329ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 330ee3c9eebSPatrick Williams # This section is locked because we are looking into another 331ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 332ee3c9eebSPatrick Williams Package.lock.acquire() 333ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 334ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 335ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 336ee3c9eebSPatrick Williams ] 337ee3c9eebSPatrick Williams Package.lock.release() 338ee3c9eebSPatrick Williams 339ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 340ee3c9eebSPatrick Williams # for the "COPY" commands. 341ee3c9eebSPatrick Williams for deppkg in deps: 342ee3c9eebSPatrick Williams deppkg.join() 343ee3c9eebSPatrick Williams 344ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 345ee3c9eebSPatrick Williams dockerfile = f""" 346ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 347ee3c9eebSPatrick Williams{self._df_copycmds()} 348ee3c9eebSPatrick Williams{self._df_build()} 349ee3c9eebSPatrick Williams""" 350ee3c9eebSPatrick Williams 351ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 352ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 353ee3c9eebSPatrick Williams # which can be accessed by other threads. 354ee3c9eebSPatrick Williams Package.lock.acquire() 355ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 356ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 357ee3c9eebSPatrick Williams Package.lock.release() 358ee3c9eebSPatrick Williams 359ee3c9eebSPatrick Williams # Do the build / save any exceptions. 360ee3c9eebSPatrick Williams try: 361ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 362ee3c9eebSPatrick Williams except Exception as e: 363ee3c9eebSPatrick Williams self.exception = e 364ee3c9eebSPatrick Williams 365ee3c9eebSPatrick Williams @classmethod 366ee3c9eebSPatrick Williams def generate_all(cls) -> None: 367ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 368ee3c9eebSPatrick Williams 369ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 370ee3c9eebSPatrick Williams package 'depends' specifications. 371ee3c9eebSPatrick Williams """ 372ee3c9eebSPatrick Williams 373ee3c9eebSPatrick Williams # Create a Package for each defined package. 374ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 375ee3c9eebSPatrick Williams 376ee3c9eebSPatrick Williams # Start building them all. 3776dbd7807SPatrick Williams # This section is locked because threads depend on each other, 3786dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 3796dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 3806dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 3816dbd7807SPatrick Williams Package.lock.acquire() 382ee3c9eebSPatrick Williams for t in pkg_threads: 383ee3c9eebSPatrick Williams t.start() 3846dbd7807SPatrick Williams Package.lock.release() 385ee3c9eebSPatrick Williams 386ee3c9eebSPatrick Williams # Wait for completion. 387ee3c9eebSPatrick Williams for t in pkg_threads: 388ee3c9eebSPatrick Williams t.join() 389ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 390ee3c9eebSPatrick Williams if t.exception: 391ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 392ee3c9eebSPatrick Williams raise t.exception 393ee3c9eebSPatrick Williams 394ee3c9eebSPatrick Williams @staticmethod 395ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 396ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 397ee3c9eebSPatrick Williams into the final image. 398ee3c9eebSPatrick Williams """ 399ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 400ee3c9eebSPatrick Williams 401ee3c9eebSPatrick Williams @classmethod 402ee3c9eebSPatrick Williams def depcache(cls) -> str: 403ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 404ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 405ee3c9eebSPatrick Williams """ 406ee3c9eebSPatrick Williams 407ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 408ee3c9eebSPatrick Williams depcache = "" 409ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 410ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 411ee3c9eebSPatrick Williams return depcache 412ee3c9eebSPatrick Williams 413ee3c9eebSPatrick Williams def _update_rev(self) -> None: 414ee3c9eebSPatrick Williams """Look up the HEAD for missing a static rev.""" 415ee3c9eebSPatrick Williams 416ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 417ee3c9eebSPatrick Williams return 418ee3c9eebSPatrick Williams 41965b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 42065b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 42165b21fb9SPatrick Williams print( 42265b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 42365b21fb9SPatrick Williams file=sys.stderr, 42465b21fb9SPatrick Williams ) 42565b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 42665b21fb9SPatrick Williams return 42765b21fb9SPatrick Williams 428ee3c9eebSPatrick Williams # Ask Github for all the branches. 42905fb2a0aSPatrick Williams lookup = git( 43005fb2a0aSPatrick Williams "ls-remote", "--heads", f"https://github.com/{self.package}" 43105fb2a0aSPatrick Williams ) 432ee3c9eebSPatrick Williams 433ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 434ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 435ee3c9eebSPatrick Williams Package.lock.acquire() 436ee3c9eebSPatrick Williams for line in lookup.split("\n"): 437f3d27e64SAndrew Geissler if re.fullmatch(f".*{branch}$", line.strip()): 438ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 439f3d27e64SAndrew Geissler break 440c7d73646SPatrick Williams elif ( 441c7d73646SPatrick Williams "refs/heads/master" in line or "refs/heads/main" in line 442c7d73646SPatrick Williams ) and "rev" not in self.pkg_def: 443ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 444ee3c9eebSPatrick Williams Package.lock.release() 445ee3c9eebSPatrick Williams 446ee3c9eebSPatrick Williams def _stagename(self) -> str: 447ee3c9eebSPatrick Williams """Create a name for the Docker stage associated with this pkg.""" 448ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 449ee3c9eebSPatrick Williams 450ee3c9eebSPatrick Williams def _url(self) -> str: 451ee3c9eebSPatrick Williams """Get the URL for this package.""" 452ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 453ee3c9eebSPatrick Williams 454ee3c9eebSPatrick Williams # If the lambda exists, call it. 455ee3c9eebSPatrick Williams if "url" in self.pkg_def: 456ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 457ee3c9eebSPatrick Williams 458ee3c9eebSPatrick Williams # Default to the github archive URL. 459ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 460ee3c9eebSPatrick Williams 461ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 462ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 463ee3c9eebSPatrick Williams 464ee3c9eebSPatrick Williams url = self._url() 465ee3c9eebSPatrick Williams if ".tar." not in url: 466ee3c9eebSPatrick Williams raise NotImplementedError( 467ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 468ee3c9eebSPatrick Williams ) 469ee3c9eebSPatrick Williams 470ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 471ee3c9eebSPatrick Williams 472ee3c9eebSPatrick Williams if url.endswith(".bz2"): 473ee3c9eebSPatrick Williams cmd += "j" 474ee3c9eebSPatrick Williams elif url.endswith(".gz"): 475ee3c9eebSPatrick Williams cmd += "z" 476ee3c9eebSPatrick Williams else: 477ee3c9eebSPatrick Williams raise NotImplementedError( 478ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 479ee3c9eebSPatrick Williams ) 480ee3c9eebSPatrick Williams 481ee3c9eebSPatrick Williams return cmd 482ee3c9eebSPatrick Williams 483ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 484ee3c9eebSPatrick Williams """Formulate the command necessary to 'cd' into the source dir.""" 485ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 486ee3c9eebSPatrick Williams 487ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 488ee3c9eebSPatrick Williams """Formulate the dockerfile snippet necessary to COPY all depends.""" 489ee3c9eebSPatrick Williams 490ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 491ee3c9eebSPatrick Williams return "" 492ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 493ee3c9eebSPatrick Williams 494ee3c9eebSPatrick Williams @staticmethod 495ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 496ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 497ee3c9eebSPatrick Williams packages into a Docker stage. 498ee3c9eebSPatrick Williams """ 499ee3c9eebSPatrick Williams 500ee3c9eebSPatrick Williams copy_cmds = "" 501ee3c9eebSPatrick Williams 502ee3c9eebSPatrick Williams # Sort the packages for consistency. 503ee3c9eebSPatrick Williams for p in sorted(pkgs): 504ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 505ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 506ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 507ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 508ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 509ee3c9eebSPatrick Williams 510ee3c9eebSPatrick Williams return copy_cmds 511ee3c9eebSPatrick Williams 512ee3c9eebSPatrick Williams def _df_build(self) -> str: 513ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 514ee3c9eebSPatrick Williams install a package into a Docker stage. 515ee3c9eebSPatrick Williams """ 516ee3c9eebSPatrick Williams 517ee3c9eebSPatrick Williams # Download and extract source. 518ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 519ee3c9eebSPatrick Williams 520ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 521ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 522ee3c9eebSPatrick Williams if custom_post_dl: 523ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 524ee3c9eebSPatrick Williams 525ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 526ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 527ee3c9eebSPatrick Williams if build_type == "autoconf": 528ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 529ee3c9eebSPatrick Williams elif build_type == "cmake": 530ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 531ee3c9eebSPatrick Williams elif build_type == "custom": 532ee3c9eebSPatrick Williams result += self._cmd_build_custom() 533ee3c9eebSPatrick Williams elif build_type == "make": 534ee3c9eebSPatrick Williams result += self._cmd_build_make() 535ee3c9eebSPatrick Williams elif build_type == "meson": 536ee3c9eebSPatrick Williams result += self._cmd_build_meson() 537ee3c9eebSPatrick Williams else: 538ee3c9eebSPatrick Williams raise NotImplementedError( 539ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 540ee3c9eebSPatrick Williams ) 541ee3c9eebSPatrick Williams 5426bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5436bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5446bce2ca1SPatrick Williams if custom_post_install: 5456bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5466bce2ca1SPatrick Williams 547ee3c9eebSPatrick Williams return result 548ee3c9eebSPatrick Williams 549ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 550ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 551ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 552ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 553ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 554ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 555ee3c9eebSPatrick Williams return result 556ee3c9eebSPatrick Williams 557ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 558ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 559ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 560ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 561ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 562ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 563ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 564ee3c9eebSPatrick Williams result += "cd .." 565ee3c9eebSPatrick Williams return result 566ee3c9eebSPatrick Williams 567ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 568ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 569ee3c9eebSPatrick Williams 570ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 571ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 572ee3c9eebSPatrick Williams 573ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 574ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 575ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 576e2da11adSAndrew Jeffery result = f"{env} meson setup builddir {meson_flags} {options} && " 577ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 578ee3c9eebSPatrick Williams return result 579ee3c9eebSPatrick Williams 580ee3c9eebSPatrick Williams 581ee3c9eebSPatrick Williamsclass Docker: 582ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 583ee3c9eebSPatrick Williams 584ee3c9eebSPatrick Williams @staticmethod 585ee3c9eebSPatrick Williams def timestamp() -> str: 586ee3c9eebSPatrick Williams """Generate a timestamp for today using the ISO week.""" 587ee3c9eebSPatrick Williams today = date.today().isocalendar() 588ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 589ee3c9eebSPatrick Williams 590ee3c9eebSPatrick Williams @staticmethod 59141d86218SPatrick Williams def tagname(pkgname: Optional[str], dockerfile: str) -> str: 592ee3c9eebSPatrick Williams """Generate a tag name for a package using a hash of the Dockerfile.""" 593ee3c9eebSPatrick Williams result = docker_image_name 594ee3c9eebSPatrick Williams if pkgname: 595ee3c9eebSPatrick Williams result += "-" + pkgname 596ee3c9eebSPatrick Williams 597ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 598ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 599ee3c9eebSPatrick Williams 600ee3c9eebSPatrick Williams return result 601ee3c9eebSPatrick Williams 602ee3c9eebSPatrick Williams @staticmethod 603ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 60422e6110bSAndrew Geissler """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 605ee3c9eebSPatrick Williams 606ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 607ee3c9eebSPatrick Williams if not force_build: 608ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 60905fb2a0aSPatrick Williams print( 61005fb2a0aSPatrick Williams f"Image {tag} already exists. Skipping.", file=sys.stderr 61105fb2a0aSPatrick Williams ) 612ee3c9eebSPatrick Williams return 613ee3c9eebSPatrick Williams 614ee3c9eebSPatrick Williams # Build it. 615ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 616ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 617*a6ebc6e2SManojkiran Eda # progress but not pollute stdout. Later on we output the final 618ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 619ee3c9eebSPatrick Williams # 620ee3c9eebSPatrick Williams # Other unusual flags: 621ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 622ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 623ee3c9eebSPatrick Williams docker.build( 624ee3c9eebSPatrick Williams proxy_args, 625ee3c9eebSPatrick Williams "--network=host", 626ee3c9eebSPatrick Williams "--force-rm", 627ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 628ee3c9eebSPatrick Williams "-t", 629ee3c9eebSPatrick Williams tag, 630ee3c9eebSPatrick Williams "-", 631ee3c9eebSPatrick Williams _in=dockerfile, 632ee3c9eebSPatrick Williams _out=( 633ee3c9eebSPatrick Williams lambda line: print( 634ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 635ee3c9eebSPatrick Williams ) 636ee3c9eebSPatrick Williams ), 63788dd7929SJonathan Doman _err_to_out=True, 638ee3c9eebSPatrick Williams ) 639ee3c9eebSPatrick Williams 640ee3c9eebSPatrick Williams 641ee3c9eebSPatrick Williams# Read a bunch of environment variables. 64205fb2a0aSPatrick Williamsdocker_image_name = os.environ.get( 64305fb2a0aSPatrick Williams "DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test" 64405fb2a0aSPatrick Williams) 645ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 646ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 6477c95a37cSPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:noble") 648ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 649ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 650ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 651ee3c9eebSPatrick Williams 65265b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 65365b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 65465b21fb9SPatrick Williams 655d0dabc3eSAndrew Geissler# Ensure appropriate docker build output to see progress and identify 656d0dabc3eSAndrew Geissler# any issues 657d0dabc3eSAndrew Geissleros.environ["BUILDKIT_PROGRESS"] = "plain" 658d0dabc3eSAndrew Geissler 659ee3c9eebSPatrick Williams# Set up some common variables. 660ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 661ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 662ee3c9eebSPatrick Williamsgid = os.getgid() 663ee3c9eebSPatrick Williamsuid = os.getuid() 664ee3c9eebSPatrick Williams 6656825a018SJosh Lehan# Use well-known constants if user is root 6666825a018SJosh Lehanif username == "root": 6676825a018SJosh Lehan homedir = "/root" 6686825a018SJosh Lehan gid = 0 6696825a018SJosh Lehan uid = 0 6706825a018SJosh Lehan 671ee3c9eebSPatrick Williams# Determine the architecture for Docker. 672ee3c9eebSPatrick Williamsarch = uname("-m").strip() 673ee3c9eebSPatrick Williamsif arch == "ppc64le": 674ee3c9eebSPatrick Williams docker_base = "ppc64le/" 675ee3c9eebSPatrick Williamselif arch == "x86_64": 676ee3c9eebSPatrick Williams docker_base = "" 677051b05b7SThang Q. Nguyenelif arch == "aarch64": 678f98f1a8dSThang Q. Nguyen docker_base = "arm64v8/" 679ee3c9eebSPatrick Williamselse: 680ee3c9eebSPatrick Williams print( 681ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 682ee3c9eebSPatrick Williams file=sys.stderr, 683ee3c9eebSPatrick Williams ) 684ee3c9eebSPatrick Williams sys.exit(1) 685ee3c9eebSPatrick Williams 68602871c91SPatrick Williams# Special flags if setting up a deb mirror. 68702871c91SPatrick Williamsmirror = "" 68802871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 68902871c91SPatrick Williams mirror = f""" 690e08ffba8SPatrick WilliamsRUN echo "deb {ubuntu_mirror} \ 691e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME) \ 692e08ffba8SPatrick Williams main restricted universe multiverse" > /etc/apt/sources.list && \\ 693e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 694e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-updates \ 695e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 696e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 697e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-security \ 698e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 699e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 700e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-proposed \ 701e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 702e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 703e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-backports \ 704e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list 70502871c91SPatrick Williams""" 70602871c91SPatrick Williams 70702871c91SPatrick Williams# Special flags for proxying. 70802871c91SPatrick Williamsproxy_cmd = "" 70934ec77e8SAdrian Ambrożewiczproxy_keyserver = "" 71002871c91SPatrick Williamsproxy_args = [] 71102871c91SPatrick Williamsif http_proxy: 71202871c91SPatrick Williams proxy_cmd = f""" 71302871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 71402871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 71502871c91SPatrick Williams""" 71634ec77e8SAdrian Ambrożewicz proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 71734ec77e8SAdrian Ambrożewicz 71802871c91SPatrick Williams proxy_args.extend( 71902871c91SPatrick Williams [ 72002871c91SPatrick Williams "--build-arg", 72102871c91SPatrick Williams f"http_proxy={http_proxy}", 72202871c91SPatrick Williams "--build-arg", 723d461cd6aSLei YU f"https_proxy={http_proxy}", 72402871c91SPatrick Williams ] 72502871c91SPatrick Williams ) 72602871c91SPatrick Williams 727ee3c9eebSPatrick Williams# Create base Dockerfile. 728a18d9c57SPatrick Williamsdockerfile_base = f""" 729a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 73002871c91SPatrick Williams 73102871c91SPatrick Williams{mirror} 73202871c91SPatrick Williams 73302871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 73402871c91SPatrick Williams 7358949d3c3SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/" 73602871c91SPatrick Williams 737bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 738bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 739bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 740bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 741bb16ac14SPatrick Williams# the update. 742bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 743bb16ac14SPatrick Williams 74402871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 74502871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 74602871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 747575b5e4cSJagpal Singh Gill# Known issue with gpg to get keys via proxy - 748575b5e4cSJagpal Singh Gill# https://bugs.launchpad.net/ubuntu/+source/gnupg2/+bug/1788190, hence using 749575b5e4cSJagpal Singh Gill# curl to get keys. 75050837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 751938d303fSJian Zhang ( apt-get install -yy gpgv ubuntu-dbgsym-keyring || \ 752575b5e4cSJagpal Singh Gill ( apt-get install -yy dirmngr curl && \ 753575b5e4cSJagpal Singh Gill curl -sSL \ 754575b5e4cSJagpal Singh Gill 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622' \ 755575b5e4cSJagpal Singh Gill | apt-key add - )) 75602871c91SPatrick Williams 75702871c91SPatrick Williams# Parse the current repo list into a debug repo list 758e08ffba8SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' \ 759e08ffba8SPatrick Williams /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 76002871c91SPatrick Williams 76102871c91SPatrick Williams# Remove non-existent debug repos 76241d86218SPatrick WilliamsRUN sed -i '/-\\(backports\\|security\\) /d' /etc/apt/sources.list.d/debug.list 76302871c91SPatrick Williams 76402871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 76502871c91SPatrick Williams 76602871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 76758f1915eSAndrew Jeffery abi-compliance-checker \ 7688b112068SAndrew Jeffery abi-dumper \ 76902871c91SPatrick Williams autoconf \ 77002871c91SPatrick Williams autoconf-archive \ 771af49ed51SAndrew Geissler bison \ 772e2e62e5cSPatrick Williams clang-17 \ 773e2e62e5cSPatrick Williams clang-format-17 \ 774e2e62e5cSPatrick Williams clang-tidy-17 \ 775e2e62e5cSPatrick Williams clang-tools-17 \ 776af49ed51SAndrew Geissler cmake \ 777af49ed51SAndrew Geissler curl \ 778af49ed51SAndrew Geissler dbus \ 779af49ed51SAndrew Geissler device-tree-compiler \ 780af49ed51SAndrew Geissler flex \ 781961f148bSPatrick Williams g++-13 \ 782961f148bSPatrick Williams gcc-13 \ 783af49ed51SAndrew Geissler git \ 78402871c91SPatrick Williams iproute2 \ 785af49ed51SAndrew Geissler iputils-ping \ 786524a331cSManojkiran Eda libaudit-dev \ 787af49ed51SAndrew Geissler libc6-dbg \ 788af49ed51SAndrew Geissler libc6-dev \ 789af49ed51SAndrew Geissler libconfig++-dev \ 790af49ed51SAndrew Geissler libcryptsetup-dev \ 791af49ed51SAndrew Geissler libdbus-1-dev \ 792af49ed51SAndrew Geissler libevdev-dev \ 793af49ed51SAndrew Geissler libgpiod-dev \ 794af49ed51SAndrew Geissler libi2c-dev \ 795af49ed51SAndrew Geissler libjpeg-dev \ 796af49ed51SAndrew Geissler libjson-perl \ 797af49ed51SAndrew Geissler libldap2-dev \ 798af49ed51SAndrew Geissler libmimetic-dev \ 79902871c91SPatrick Williams libnl-3-dev \ 80002871c91SPatrick Williams libnl-genl-3-dev \ 80102871c91SPatrick Williams libpam0g-dev \ 80202871c91SPatrick Williams libpciaccess-dev \ 803af49ed51SAndrew Geissler libperlio-gzip-perl \ 804af49ed51SAndrew Geissler libpng-dev \ 805af49ed51SAndrew Geissler libprotobuf-dev \ 806af49ed51SAndrew Geissler libsnmp-dev \ 807af49ed51SAndrew Geissler libssl-dev \ 808af49ed51SAndrew Geissler libsystemd-dev \ 809af49ed51SAndrew Geissler libtool \ 810af49ed51SAndrew Geissler liburing-dev \ 81102871c91SPatrick Williams libxml2-utils \ 8120eedeedaSPatrick Williams libxml-simple-perl \ 813af49ed51SAndrew Geissler ninja-build \ 814af49ed51SAndrew Geissler npm \ 815af49ed51SAndrew Geissler pkg-config \ 816af49ed51SAndrew Geissler protobuf-compiler \ 817af49ed51SAndrew Geissler python3 \ 818af49ed51SAndrew Geissler python3-dev\ 819af49ed51SAndrew Geissler python3-git \ 820af49ed51SAndrew Geissler python3-mako \ 821af49ed51SAndrew Geissler python3-pip \ 82225ba1e2fSWilliam A. Kennington III python3-protobuf \ 823af49ed51SAndrew Geissler python3-setuptools \ 824af49ed51SAndrew Geissler python3-socks \ 825af49ed51SAndrew Geissler python3-yaml \ 8269adf68d6SJohn Wedig rsync \ 827af49ed51SAndrew Geissler shellcheck \ 8288dd1bfe6SEwelina Walkusz socat \ 829af49ed51SAndrew Geissler sudo \ 830af49ed51SAndrew Geissler systemd \ 831af49ed51SAndrew Geissler valgrind \ 832b565f825SAndrew Geissler vim \ 833af49ed51SAndrew Geissler wget \ 834af49ed51SAndrew Geissler xxd 83502871c91SPatrick Williams 836961f148bSPatrick WilliamsRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 13 \ 837961f148bSPatrick Williams --slave /usr/bin/g++ g++ /usr/bin/g++-13 \ 838961f148bSPatrick Williams --slave /usr/bin/gcov gcov /usr/bin/gcov-13 \ 839961f148bSPatrick Williams --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-13 \ 840961f148bSPatrick Williams --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-13 841961f148bSPatrick WilliamsRUN update-alternatives --remove cpp /usr/bin/cpp && \ 842961f148bSPatrick Williams update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-13 13 84302871c91SPatrick Williams 844e2e62e5cSPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 1000 \ 845e2e62e5cSPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-17 \ 846e2e62e5cSPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-17 \ 847e2e62e5cSPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-17 \ 848e08ffba8SPatrick Williams --slave /usr/bin/run-clang-tidy run-clang-tidy.py \ 849e2e62e5cSPatrick Williams /usr/bin/run-clang-tidy-17 \ 850e2e62e5cSPatrick Williams --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-17 85102871c91SPatrick Williams 85250837436SPatrick Williams""" 85350837436SPatrick Williams 85450837436SPatrick Williamsif is_automated_ci_build: 85550837436SPatrick Williams dockerfile_base += f""" 856*a6ebc6e2SManojkiran Eda# Run an arbitrary command to pollute the docker cache regularly force us 85750837436SPatrick Williams# to re-run `apt-get update` daily. 858ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 85950837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 86050837436SPatrick Williams 86150837436SPatrick Williams""" 86250837436SPatrick Williams 86341d86218SPatrick Williamsdockerfile_base += """ 8645e4d8402SPatrick WilliamsRUN pip3 install --break-system-packages \ 865818023dfSPatrick Williams beautysh \ 866818023dfSPatrick Williams black \ 867818023dfSPatrick Williams codespell \ 868818023dfSPatrick Williams flake8 \ 869818023dfSPatrick Williams gitlint \ 870818023dfSPatrick Williams inflection \ 871818023dfSPatrick Williams isort \ 872818023dfSPatrick Williams jsonschema \ 87316baaf73SPatrick Williams meson==1.3.0 \ 874818023dfSPatrick Williams requests 875b08ddf77SPatrick Williams 876b08ddf77SPatrick WilliamsRUN npm install -g \ 877d0757deaSXinnan Xie eslint@v8.56.0 eslint-plugin-json@v3.1.0 \ 8787d41f6d2SPatrick Williams markdownlint-cli@latest \ 879b08ddf77SPatrick Williams prettier@latest 880fb9948a3SEd Tanous""" 881fb9948a3SEd Tanous 882ee3c9eebSPatrick Williams# Build the base and stage docker images. 883ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 884ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 885ee3c9eebSPatrick WilliamsPackage.generate_all() 88602871c91SPatrick Williams 887ee3c9eebSPatrick Williams# Create the final Dockerfile. 888a18d9c57SPatrick Williamsdockerfile = f""" 88902871c91SPatrick Williams# Build the final output image 890a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 891ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 89202871c91SPatrick Williams 89302871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 89402871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 89502871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 896ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 89702871c91SPatrick Williams 89867cc0616SPatrick Williams# Ensure the group, user, and home directory are created (or rename them if 89967cc0616SPatrick Williams# they already exist). 90067cc0616SPatrick WilliamsRUN if grep -q ":{gid}:" /etc/group ; then \ 90167cc0616SPatrick Williams groupmod -n {username} $(awk -F : '{{ if ($3 == {gid}) {{ print $1 }} }}' /etc/group) ; \ 90267cc0616SPatrick Williams else \ 90367cc0616SPatrick Williams groupadd -f -g {gid} {username} ; \ 90467cc0616SPatrick Williams fi 90502871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 90667cc0616SPatrick WilliamsRUN if grep -q ":{uid}:" /etc/passwd ; then \ 90773b3ee91SPatrick Williams usermod -l {username} -d {homedir} -m $(awk -F : '{{ if ($3 == {uid}) {{ print $1 }} }}' /etc/passwd) ; \ 90867cc0616SPatrick Williams else \ 90967cc0616SPatrick Williams useradd -d {homedir} -m -u {uid} -g {gid} {username} ; \ 91067cc0616SPatrick Williams fi 91102871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 91202871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 91302871c91SPatrick Williams 914305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 915305a9a5dSAndrew Geissler# and data installs 9167bb00b13SAndrew GeisslerRUN chown -R {username}:{username} /usr/local/share 917305a9a5dSAndrew Geissler 918ab4fee83SJonathan Doman# Update library cache 919ab4fee83SJonathan DomanRUN ldconfig 920ab4fee83SJonathan Doman 92102871c91SPatrick Williams{proxy_cmd} 92202871c91SPatrick Williams 92302871c91SPatrick WilliamsRUN /bin/bash 92402871c91SPatrick Williams""" 92502871c91SPatrick Williams 926a18d9c57SPatrick Williams# Do the final docker build 927ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 928ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 929ee3c9eebSPatrick Williams 93000536fbeSPatrick Williams# Print the tag of the final image. 93100536fbeSPatrick Williamsprint(docker_final_img_name) 932