102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1050837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1150837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1250837436SPatrick Williams# (ex. Jenkins)> 1302871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1402871c91SPatrick Williams# repositories> 1502871c91SPatrick Williams# default is master, which will be used if input branch not 1602871c91SPatrick Williams# provided or not found 1702871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1802871c91SPatrick Williams# default ones in /etc/apt/sources.list> 1902871c91SPatrick Williams# default is empty, and no mirror is used. 2002871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2102871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2202871c91SPatrick Williams 2302871c91SPatrick Williamsimport os 2402871c91SPatrick Williamsimport sys 25b16f3e20SPatrick Williamsimport threading 26a18d9c57SPatrick Williamsfrom datetime import date 27a18d9c57SPatrick Williamsfrom hashlib import sha256 28e08ffba8SPatrick Williams 29e08ffba8SPatrick Williams# typing.Dict is used for type-hints. 30e08ffba8SPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional # noqa: F401 3102871c91SPatrick Williams 3241d86218SPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 3341d86218SPatrick Williams 34ee3c9eebSPatrick Williamstry: 35ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 36ee3c9eebSPatrick Williams from typing import TypedDict 3741d86218SPatrick Williamsexcept Exception: 38ee3c9eebSPatrick Williams 39ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 40ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 4141d86218SPatrick Williams def __init_subclass__(cls, **kwargs: Any) -> None: 42ee3c9eebSPatrick Williams super().__init_subclass__() 43ee3c9eebSPatrick Williams 44ee3c9eebSPatrick Williams 45ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 46aae36d18SPatrick Williamsprefix = "/usr/local" 4702871c91SPatrick Williamsproc_count = nproc().strip() 4802871c91SPatrick Williams 49ee3c9eebSPatrick Williams 50ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 51ee3c9eebSPatrick Williams """Package Definition for packages dictionary.""" 52ee3c9eebSPatrick Williams 53ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 54ee3c9eebSPatrick Williams rev: str 55ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 56ee3c9eebSPatrick Williams url: Callable[[str, str], str] 57ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 58ee3c9eebSPatrick Williams depends: Iterable[str] 59ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 60ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 61ee3c9eebSPatrick Williams build_type: str 62ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 63ee3c9eebSPatrick Williams build_steps: Iterable[str] 64ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 65ee3c9eebSPatrick Williams config_flags: Iterable[str] 66ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 67ee3c9eebSPatrick Williams config_env: Iterable[str] 68ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 69ee3c9eebSPatrick Williams # before config / build / install. 70ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 716bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 726bce2ca1SPatrick Williams custom_post_install: Iterable[str] 73ee3c9eebSPatrick Williams 74ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 75ee3c9eebSPatrick Williams __tag: str 76ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 77ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 78ee3c9eebSPatrick Williams 7902871c91SPatrick Williams 807204324cSPatrick Williams# Packages to include in image. 817204324cSPatrick Williamspackages = { 82ee3c9eebSPatrick Williams "boost": PackageDef( 8345bfd1ffSEd Tanous rev="1.81.0", 84ee3c9eebSPatrick Williams url=( 8545bfd1ffSEd Tanous lambda pkg, rev: f"https://boostorg.jfrog.io/artifactory/main/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.gz" # noqa: E501 862abc4a48SPatrick Williams ), 87ee3c9eebSPatrick Williams build_type="custom", 88ee3c9eebSPatrick Williams build_steps=[ 89e08ffba8SPatrick Williams ( 90e08ffba8SPatrick Williams "./bootstrap.sh" 91e08ffba8SPatrick Williams f" --prefix={prefix} --with-libraries=context,coroutine" 92e08ffba8SPatrick Williams ), 93aae36d18SPatrick Williams "./b2", 94aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 95aae36d18SPatrick Williams ], 96ee3c9eebSPatrick Williams ), 97ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 98c1977839SPatrick Williams rev="v1.3.2", 99ee3c9eebSPatrick Williams build_type="custom", 100ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 101ee3c9eebSPatrick Williams ), 102c7198558SEd Tanous "danmar/cppcheck": PackageDef( 103be4bd084SPatrick Williams rev="2.9", 104c7198558SEd Tanous build_type="cmake", 105c7198558SEd Tanous ), 106ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 107ee3c9eebSPatrick Williams rev="v1.9.1", 108ee3c9eebSPatrick Williams build_type="cmake", 109ee3c9eebSPatrick Williams config_flags=[ 110aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 111aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 112aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 113aae36d18SPatrick Williams ], 114ee3c9eebSPatrick Williams ), 115ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 116652d8aeaSWilliam A. Kennington III rev="9.1.0", 117ee3c9eebSPatrick Williams build_type="cmake", 118ee3c9eebSPatrick Williams config_flags=[ 119aae36d18SPatrick Williams "-DFMT_DOC=OFF", 120aae36d18SPatrick Williams "-DFMT_TEST=OFF", 121aae36d18SPatrick Williams ], 122ee3c9eebSPatrick Williams ), 123ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 124c1977839SPatrick Williams rev="4.2.1", 125ee3c9eebSPatrick Williams build_type="custom", 126ee3c9eebSPatrick Williams build_steps=[ 127aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 128aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 129aae36d18SPatrick Williams ], 130ee3c9eebSPatrick Williams ), 131ed9414e8SPatrick Williams # release-1.12.1 132ee3c9eebSPatrick Williams "google/googletest": PackageDef( 133ed9414e8SPatrick Williams rev="58d77fa8070e8cec2dc1ed015d66b454c8d78850", 134ee3c9eebSPatrick Williams build_type="cmake", 1354dd32c02SWilliam A. Kennington III config_env=["CXXFLAGS=-std=c++20"], 136ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 137ee3c9eebSPatrick Williams ), 138ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 139c1977839SPatrick Williams rev="v3.11.2", 1406bce2ca1SPatrick Williams build_type="cmake", 1416bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1426bce2ca1SPatrick Williams custom_post_install=[ 143e08ffba8SPatrick Williams ( 144e08ffba8SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp" 145e08ffba8SPatrick Williams f" {prefix}/include/json.hpp" 146e08ffba8SPatrick Williams ), 147aae36d18SPatrick Williams ], 148ee3c9eebSPatrick Williams ), 149058e3a34SPrzemyslaw Czarnowski "json-c/json-c": PackageDef( 150058e3a34SPrzemyslaw Czarnowski rev="json-c-0.16-20220414", 151058e3a34SPrzemyslaw Czarnowski build_type="cmake", 152058e3a34SPrzemyslaw Czarnowski ), 15302871c91SPatrick Williams # Snapshot from 2019-05-24 154ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 155ee3c9eebSPatrick Williams rev="v1.15", 156ee3c9eebSPatrick Williams build_type="make", 157ee3c9eebSPatrick Williams ), 158ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 159ee3c9eebSPatrick Williams rev="LibVNCServer-0.9.13", 160ee3c9eebSPatrick Williams build_type="cmake", 161ee3c9eebSPatrick Williams ), 162ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 163c1977839SPatrick Williams rev="9.0.0", 164ee3c9eebSPatrick Williams build_type="cmake", 165ee3c9eebSPatrick Williams ), 166ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 167c1977839SPatrick Williams rev="v0.7", 168ee3c9eebSPatrick Williams build_type="cmake", 169ee3c9eebSPatrick Williams config_flags=[ 1700eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1710eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 172aae36d18SPatrick Williams ], 173ee3c9eebSPatrick Williams ), 174ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 175ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 176ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 177ee3c9eebSPatrick Williams build_type="meson", 178ee3c9eebSPatrick Williams config_flags=[ 179aae36d18SPatrick Williams "-Dexamples=false", 180aae36d18SPatrick Williams "-Dtests=disabled", 181aae36d18SPatrick Williams ], 182ee3c9eebSPatrick Williams ), 183ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 184ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 185ee3c9eebSPatrick Williams build_type="meson", 1864fe87776SWilliam A. Kennington III config_flags=["-Dgenerate_md=false"], 187ee3c9eebSPatrick Williams ), 188ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 189ee3c9eebSPatrick Williams depends=[ 19083394610SPatrick Williams "USCiLab/cereal", 19183394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 19283394610SPatrick Williams "openbmc/sdbusplus", 19383394610SPatrick Williams "openbmc/sdeventplus", 194aae36d18SPatrick Williams ], 195f79ce4c4SPatrick Williams build_type="meson", 196ee3c9eebSPatrick Williams config_flags=[ 1976c98f280SWilliam A. Kennington III "-Dlibonly=true", 1986c98f280SWilliam A. Kennington III "-Dtests=disabled", 1995eabdae9SPatrick Williams f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml", 200aae36d18SPatrick Williams ], 201ee3c9eebSPatrick Williams ), 202ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 203ee3c9eebSPatrick Williams depends=[ 20411e5762cSBrad Bishop "CLIUtils/CLI11", 20570af95caSPatrick Williams "boost", 20683394610SPatrick Williams "leethomason/tinyxml2", 20770af95caSPatrick Williams "openbmc/phosphor-dbus-interfaces", 20883394610SPatrick Williams "openbmc/phosphor-logging", 20983394610SPatrick Williams "openbmc/sdbusplus", 210aae36d18SPatrick Williams ], 2111197e359SBrad Bishop build_type="meson", 2121197e359SBrad Bishop config_flags=[ 2131197e359SBrad Bishop "-Dtests=disabled", 2141197e359SBrad Bishop ], 215ee3c9eebSPatrick Williams ), 2161c19e453SManojkiran Eda "openbmc/libpldm": PackageDef( 217ee3c9eebSPatrick Williams build_type="meson", 218ee3c9eebSPatrick Williams config_flags=[ 219aae36d18SPatrick Williams "-Doem-ibm=enabled", 220aae36d18SPatrick Williams "-Dtests=disabled", 221aae36d18SPatrick Williams ], 222ee3c9eebSPatrick Williams ), 223ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 224ee3c9eebSPatrick Williams build_type="meson", 225ee3c9eebSPatrick Williams custom_post_dl=[ 226aae36d18SPatrick Williams "cd tools", 227aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 228aae36d18SPatrick Williams "cd ..", 229aae36d18SPatrick Williams ], 230ee3c9eebSPatrick Williams config_flags=[ 231aae36d18SPatrick Williams "-Dexamples=disabled", 232aae36d18SPatrick Williams "-Dtests=disabled", 233aae36d18SPatrick Williams ], 234b16f3e20SPatrick Williams ), 235ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 23670af95caSPatrick Williams depends=[ 23770af95caSPatrick Williams "Naios/function2", 23870af95caSPatrick Williams "openbmc/stdplus", 23970af95caSPatrick Williams ], 240ee3c9eebSPatrick Williams build_type="meson", 241ee3c9eebSPatrick Williams config_flags=[ 242ee3c9eebSPatrick Williams "-Dexamples=false", 243ee3c9eebSPatrick Williams "-Dtests=disabled", 244ee3c9eebSPatrick Williams ], 245ee3c9eebSPatrick Williams ), 246ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 24770af95caSPatrick Williams depends=[ 24870af95caSPatrick Williams "fmtlib/fmt", 249ca1bf0c0SWilliam A. Kennington III "google/googletest", 250ca1bf0c0SWilliam A. Kennington III "Naios/function2", 25170af95caSPatrick Williams ], 252ee3c9eebSPatrick Williams build_type="meson", 253ee3c9eebSPatrick Williams config_flags=[ 254ee3c9eebSPatrick Williams "-Dexamples=false", 255ee3c9eebSPatrick Williams "-Dtests=disabled", 256ca1bf0c0SWilliam A. Kennington III "-Dgtest=enabled", 257ee3c9eebSPatrick Williams ], 258ee3c9eebSPatrick Williams ), 259ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 26002871c91SPatrick Williams 26102871c91SPatrick Williams# Define common flags used for builds 26202871c91SPatrick Williamsconfigure_flags = " ".join( 26302871c91SPatrick Williams [ 26402871c91SPatrick Williams f"--prefix={prefix}", 26502871c91SPatrick Williams ] 26602871c91SPatrick Williams) 26702871c91SPatrick Williamscmake_flags = " ".join( 26802871c91SPatrick Williams [ 26902871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 2700f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 27102871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 2720f2086b3SPatrick Williams "-GNinja", 2730f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 27402871c91SPatrick Williams ] 27502871c91SPatrick Williams) 27602871c91SPatrick Williamsmeson_flags = " ".join( 27702871c91SPatrick Williams [ 27802871c91SPatrick Williams "--wrap-mode=nodownload", 27902871c91SPatrick Williams f"-Dprefix={prefix}", 28002871c91SPatrick Williams ] 28102871c91SPatrick Williams) 28202871c91SPatrick Williams 283ee3c9eebSPatrick Williams 284ee3c9eebSPatrick Williamsclass Package(threading.Thread): 285ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 286ee3c9eebSPatrick Williams 287ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 288ee3c9eebSPatrick Williams Package.generate_all(). 289ee3c9eebSPatrick Williams """ 290ee3c9eebSPatrick Williams 291ee3c9eebSPatrick Williams # Copy the packages dictionary. 292ee3c9eebSPatrick Williams packages = packages.copy() 293ee3c9eebSPatrick Williams 294ee3c9eebSPatrick Williams # Lock used for thread-safety. 295ee3c9eebSPatrick Williams lock = threading.Lock() 296ee3c9eebSPatrick Williams 297ee3c9eebSPatrick Williams def __init__(self, pkg: str): 298ee3c9eebSPatrick Williams """pkg - The name of this package (ex. foo/bar )""" 299ee3c9eebSPatrick Williams super(Package, self).__init__() 300ee3c9eebSPatrick Williams 301ee3c9eebSPatrick Williams self.package = pkg 302ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 303ee3c9eebSPatrick Williams 304ee3c9eebSPatrick Williams # Reference to this package's 305ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 306ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 307ee3c9eebSPatrick Williams 308ee3c9eebSPatrick Williams def run(self) -> None: 309ee3c9eebSPatrick Williams """Thread 'run' function. Builds the Docker stage.""" 310ee3c9eebSPatrick Williams 311ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 312ee3c9eebSPatrick Williams self._update_rev() 313ee3c9eebSPatrick Williams 314ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 315ee3c9eebSPatrick Williams # This section is locked because we are looking into another 316ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 317ee3c9eebSPatrick Williams Package.lock.acquire() 318ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 319ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 320ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 321ee3c9eebSPatrick Williams ] 322ee3c9eebSPatrick Williams Package.lock.release() 323ee3c9eebSPatrick Williams 324ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 325ee3c9eebSPatrick Williams # for the "COPY" commands. 326ee3c9eebSPatrick Williams for deppkg in deps: 327ee3c9eebSPatrick Williams deppkg.join() 328ee3c9eebSPatrick Williams 329ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 330ee3c9eebSPatrick Williams dockerfile = f""" 331ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 332ee3c9eebSPatrick Williams{self._df_copycmds()} 333ee3c9eebSPatrick Williams{self._df_build()} 334ee3c9eebSPatrick Williams""" 335ee3c9eebSPatrick Williams 336ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 337ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 338ee3c9eebSPatrick Williams # which can be accessed by other threads. 339ee3c9eebSPatrick Williams Package.lock.acquire() 340ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 341ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 342ee3c9eebSPatrick Williams Package.lock.release() 343ee3c9eebSPatrick Williams 344ee3c9eebSPatrick Williams # Do the build / save any exceptions. 345ee3c9eebSPatrick Williams try: 346ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 347ee3c9eebSPatrick Williams except Exception as e: 348ee3c9eebSPatrick Williams self.exception = e 349ee3c9eebSPatrick Williams 350ee3c9eebSPatrick Williams @classmethod 351ee3c9eebSPatrick Williams def generate_all(cls) -> None: 352ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 353ee3c9eebSPatrick Williams 354ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 355ee3c9eebSPatrick Williams package 'depends' specifications. 356ee3c9eebSPatrick Williams """ 357ee3c9eebSPatrick Williams 358ee3c9eebSPatrick Williams # Create a Package for each defined package. 359ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 360ee3c9eebSPatrick Williams 361ee3c9eebSPatrick Williams # Start building them all. 3626dbd7807SPatrick Williams # This section is locked because threads depend on each other, 3636dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 3646dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 3656dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 3666dbd7807SPatrick Williams Package.lock.acquire() 367ee3c9eebSPatrick Williams for t in pkg_threads: 368ee3c9eebSPatrick Williams t.start() 3696dbd7807SPatrick Williams Package.lock.release() 370ee3c9eebSPatrick Williams 371ee3c9eebSPatrick Williams # Wait for completion. 372ee3c9eebSPatrick Williams for t in pkg_threads: 373ee3c9eebSPatrick Williams t.join() 374ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 375ee3c9eebSPatrick Williams if t.exception: 376ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 377ee3c9eebSPatrick Williams raise t.exception 378ee3c9eebSPatrick Williams 379ee3c9eebSPatrick Williams @staticmethod 380ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 381ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 382ee3c9eebSPatrick Williams into the final image. 383ee3c9eebSPatrick Williams """ 384ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 385ee3c9eebSPatrick Williams 386ee3c9eebSPatrick Williams @classmethod 387ee3c9eebSPatrick Williams def depcache(cls) -> str: 388ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 389ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 390ee3c9eebSPatrick Williams """ 391ee3c9eebSPatrick Williams 392ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 393ee3c9eebSPatrick Williams depcache = "" 394ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 395ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 396ee3c9eebSPatrick Williams return depcache 397ee3c9eebSPatrick Williams 398ee3c9eebSPatrick Williams def _update_rev(self) -> None: 399ee3c9eebSPatrick Williams """Look up the HEAD for missing a static rev.""" 400ee3c9eebSPatrick Williams 401ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 402ee3c9eebSPatrick Williams return 403ee3c9eebSPatrick Williams 40465b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 40565b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 40665b21fb9SPatrick Williams print( 40765b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 40865b21fb9SPatrick Williams file=sys.stderr, 40965b21fb9SPatrick Williams ) 41065b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 41165b21fb9SPatrick Williams return 41265b21fb9SPatrick Williams 413ee3c9eebSPatrick Williams # Ask Github for all the branches. 41405fb2a0aSPatrick Williams lookup = git( 41505fb2a0aSPatrick Williams "ls-remote", "--heads", f"https://github.com/{self.package}" 41605fb2a0aSPatrick Williams ) 417ee3c9eebSPatrick Williams 418ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 419ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 420ee3c9eebSPatrick Williams Package.lock.acquire() 421ee3c9eebSPatrick Williams for line in lookup.split("\n"): 422ee3c9eebSPatrick Williams if f"refs/heads/{branch}" in line: 423ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 424c7d73646SPatrick Williams elif ( 425c7d73646SPatrick Williams "refs/heads/master" in line or "refs/heads/main" in line 426c7d73646SPatrick Williams ) and "rev" not in self.pkg_def: 427ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 428ee3c9eebSPatrick Williams Package.lock.release() 429ee3c9eebSPatrick Williams 430ee3c9eebSPatrick Williams def _stagename(self) -> str: 431ee3c9eebSPatrick Williams """Create a name for the Docker stage associated with this pkg.""" 432ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 433ee3c9eebSPatrick Williams 434ee3c9eebSPatrick Williams def _url(self) -> str: 435ee3c9eebSPatrick Williams """Get the URL for this package.""" 436ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 437ee3c9eebSPatrick Williams 438ee3c9eebSPatrick Williams # If the lambda exists, call it. 439ee3c9eebSPatrick Williams if "url" in self.pkg_def: 440ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 441ee3c9eebSPatrick Williams 442ee3c9eebSPatrick Williams # Default to the github archive URL. 443ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 444ee3c9eebSPatrick Williams 445ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 446ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 447ee3c9eebSPatrick Williams 448ee3c9eebSPatrick Williams url = self._url() 449ee3c9eebSPatrick Williams if ".tar." not in url: 450ee3c9eebSPatrick Williams raise NotImplementedError( 451ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 452ee3c9eebSPatrick Williams ) 453ee3c9eebSPatrick Williams 454ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 455ee3c9eebSPatrick Williams 456ee3c9eebSPatrick Williams if url.endswith(".bz2"): 457ee3c9eebSPatrick Williams cmd += "j" 458ee3c9eebSPatrick Williams elif url.endswith(".gz"): 459ee3c9eebSPatrick Williams cmd += "z" 460ee3c9eebSPatrick Williams else: 461ee3c9eebSPatrick Williams raise NotImplementedError( 462ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 463ee3c9eebSPatrick Williams ) 464ee3c9eebSPatrick Williams 465ee3c9eebSPatrick Williams return cmd 466ee3c9eebSPatrick Williams 467ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 468ee3c9eebSPatrick Williams """Formulate the command necessary to 'cd' into the source dir.""" 469ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 470ee3c9eebSPatrick Williams 471ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 472ee3c9eebSPatrick Williams """Formulate the dockerfile snippet necessary to COPY all depends.""" 473ee3c9eebSPatrick Williams 474ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 475ee3c9eebSPatrick Williams return "" 476ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 477ee3c9eebSPatrick Williams 478ee3c9eebSPatrick Williams @staticmethod 479ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 480ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 481ee3c9eebSPatrick Williams packages into a Docker stage. 482ee3c9eebSPatrick Williams """ 483ee3c9eebSPatrick Williams 484ee3c9eebSPatrick Williams copy_cmds = "" 485ee3c9eebSPatrick Williams 486ee3c9eebSPatrick Williams # Sort the packages for consistency. 487ee3c9eebSPatrick Williams for p in sorted(pkgs): 488ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 489ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 490ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 491ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 492ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 493ee3c9eebSPatrick Williams 494ee3c9eebSPatrick Williams return copy_cmds 495ee3c9eebSPatrick Williams 496ee3c9eebSPatrick Williams def _df_build(self) -> str: 497ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 498ee3c9eebSPatrick Williams install a package into a Docker stage. 499ee3c9eebSPatrick Williams """ 500ee3c9eebSPatrick Williams 501ee3c9eebSPatrick Williams # Download and extract source. 502ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 503ee3c9eebSPatrick Williams 504ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 505ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 506ee3c9eebSPatrick Williams if custom_post_dl: 507ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 508ee3c9eebSPatrick Williams 509ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 510ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 511ee3c9eebSPatrick Williams if build_type == "autoconf": 512ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 513ee3c9eebSPatrick Williams elif build_type == "cmake": 514ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 515ee3c9eebSPatrick Williams elif build_type == "custom": 516ee3c9eebSPatrick Williams result += self._cmd_build_custom() 517ee3c9eebSPatrick Williams elif build_type == "make": 518ee3c9eebSPatrick Williams result += self._cmd_build_make() 519ee3c9eebSPatrick Williams elif build_type == "meson": 520ee3c9eebSPatrick Williams result += self._cmd_build_meson() 521ee3c9eebSPatrick Williams else: 522ee3c9eebSPatrick Williams raise NotImplementedError( 523ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 524ee3c9eebSPatrick Williams ) 525ee3c9eebSPatrick Williams 5266bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5276bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5286bce2ca1SPatrick Williams if custom_post_install: 5296bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5306bce2ca1SPatrick Williams 531ee3c9eebSPatrick Williams return result 532ee3c9eebSPatrick Williams 533ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 534ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 535ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 536ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 537ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 538ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 539ee3c9eebSPatrick Williams return result 540ee3c9eebSPatrick Williams 541ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 542ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 543ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 544ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 545ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 546ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 547ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 548ee3c9eebSPatrick Williams result += "cd .." 549ee3c9eebSPatrick Williams return result 550ee3c9eebSPatrick Williams 551ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 552ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 553ee3c9eebSPatrick Williams 554ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 555ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 556ee3c9eebSPatrick Williams 557ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 558ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 559ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 560ee3c9eebSPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 561ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 562ee3c9eebSPatrick Williams return result 563ee3c9eebSPatrick Williams 564ee3c9eebSPatrick Williams 565ee3c9eebSPatrick Williamsclass Docker: 566ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 567ee3c9eebSPatrick Williams 568ee3c9eebSPatrick Williams @staticmethod 569ee3c9eebSPatrick Williams def timestamp() -> str: 570ee3c9eebSPatrick Williams """Generate a timestamp for today using the ISO week.""" 571ee3c9eebSPatrick Williams today = date.today().isocalendar() 572ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 573ee3c9eebSPatrick Williams 574ee3c9eebSPatrick Williams @staticmethod 57541d86218SPatrick Williams def tagname(pkgname: Optional[str], dockerfile: str) -> str: 576ee3c9eebSPatrick Williams """Generate a tag name for a package using a hash of the Dockerfile.""" 577ee3c9eebSPatrick Williams result = docker_image_name 578ee3c9eebSPatrick Williams if pkgname: 579ee3c9eebSPatrick Williams result += "-" + pkgname 580ee3c9eebSPatrick Williams 581ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 582ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 583ee3c9eebSPatrick Williams 584ee3c9eebSPatrick Williams return result 585ee3c9eebSPatrick Williams 586ee3c9eebSPatrick Williams @staticmethod 587ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 58822e6110bSAndrew Geissler """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 589ee3c9eebSPatrick Williams 590ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 591ee3c9eebSPatrick Williams if not force_build: 592ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 59305fb2a0aSPatrick Williams print( 59405fb2a0aSPatrick Williams f"Image {tag} already exists. Skipping.", file=sys.stderr 59505fb2a0aSPatrick Williams ) 596ee3c9eebSPatrick Williams return 597ee3c9eebSPatrick Williams 598ee3c9eebSPatrick Williams # Build it. 599ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 600ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 601ee3c9eebSPatrick Williams # progress but not polute stdout. Later on we output the final 602ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 603ee3c9eebSPatrick Williams # 604ee3c9eebSPatrick Williams # Other unusual flags: 605ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 606ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 607ee3c9eebSPatrick Williams docker.build( 608ee3c9eebSPatrick Williams proxy_args, 609ee3c9eebSPatrick Williams "--network=host", 610ee3c9eebSPatrick Williams "--force-rm", 611ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 612ee3c9eebSPatrick Williams "-t", 613ee3c9eebSPatrick Williams tag, 614ee3c9eebSPatrick Williams "-", 615ee3c9eebSPatrick Williams _in=dockerfile, 616ee3c9eebSPatrick Williams _out=( 617ee3c9eebSPatrick Williams lambda line: print( 618ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 619ee3c9eebSPatrick Williams ) 620ee3c9eebSPatrick Williams ), 621ee3c9eebSPatrick Williams ) 622ee3c9eebSPatrick Williams 623ee3c9eebSPatrick Williams 624ee3c9eebSPatrick Williams# Read a bunch of environment variables. 62505fb2a0aSPatrick Williamsdocker_image_name = os.environ.get( 62605fb2a0aSPatrick Williams "DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test" 62705fb2a0aSPatrick Williams) 628ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 629ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 6305b08dc6bSPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:kinetic") 631ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 632ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 633ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 634ee3c9eebSPatrick Williams 63565b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 63665b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 63765b21fb9SPatrick Williams 638d0dabc3eSAndrew Geissler# Ensure appropriate docker build output to see progress and identify 639d0dabc3eSAndrew Geissler# any issues 640d0dabc3eSAndrew Geissleros.environ["BUILDKIT_PROGRESS"] = "plain" 641d0dabc3eSAndrew Geissler 642ee3c9eebSPatrick Williams# Set up some common variables. 643ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 644ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 645ee3c9eebSPatrick Williamsgid = os.getgid() 646ee3c9eebSPatrick Williamsuid = os.getuid() 647ee3c9eebSPatrick Williams 6486825a018SJosh Lehan# Use well-known constants if user is root 6496825a018SJosh Lehanif username == "root": 6506825a018SJosh Lehan homedir = "/root" 6516825a018SJosh Lehan gid = 0 6526825a018SJosh Lehan uid = 0 6536825a018SJosh Lehan 654ee3c9eebSPatrick Williams# Determine the architecture for Docker. 655ee3c9eebSPatrick Williamsarch = uname("-m").strip() 656ee3c9eebSPatrick Williamsif arch == "ppc64le": 657ee3c9eebSPatrick Williams docker_base = "ppc64le/" 658ee3c9eebSPatrick Williamselif arch == "x86_64": 659ee3c9eebSPatrick Williams docker_base = "" 660051b05b7SThang Q. Nguyenelif arch == "aarch64": 661f98f1a8dSThang Q. Nguyen docker_base = "arm64v8/" 662ee3c9eebSPatrick Williamselse: 663ee3c9eebSPatrick Williams print( 664ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 665ee3c9eebSPatrick Williams file=sys.stderr, 666ee3c9eebSPatrick Williams ) 667ee3c9eebSPatrick Williams sys.exit(1) 668ee3c9eebSPatrick Williams 66902871c91SPatrick Williams# Special flags if setting up a deb mirror. 67002871c91SPatrick Williamsmirror = "" 67102871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 67202871c91SPatrick Williams mirror = f""" 673e08ffba8SPatrick WilliamsRUN echo "deb {ubuntu_mirror} \ 674e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME) \ 675e08ffba8SPatrick Williams main restricted universe multiverse" > /etc/apt/sources.list && \\ 676e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 677e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-updates \ 678e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 679e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 680e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-security \ 681e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 682e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 683e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-proposed \ 684e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 685e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 686e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-backports \ 687e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list 68802871c91SPatrick Williams""" 68902871c91SPatrick Williams 69002871c91SPatrick Williams# Special flags for proxying. 69102871c91SPatrick Williamsproxy_cmd = "" 69234ec77e8SAdrian Ambrożewiczproxy_keyserver = "" 69302871c91SPatrick Williamsproxy_args = [] 69402871c91SPatrick Williamsif http_proxy: 69502871c91SPatrick Williams proxy_cmd = f""" 69602871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 69702871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 69802871c91SPatrick Williams""" 69934ec77e8SAdrian Ambrożewicz proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 70034ec77e8SAdrian Ambrożewicz 70102871c91SPatrick Williams proxy_args.extend( 70202871c91SPatrick Williams [ 70302871c91SPatrick Williams "--build-arg", 70402871c91SPatrick Williams f"http_proxy={http_proxy}", 70502871c91SPatrick Williams "--build-arg", 706d461cd6aSLei YU f"https_proxy={http_proxy}", 70702871c91SPatrick Williams ] 70802871c91SPatrick Williams ) 70902871c91SPatrick Williams 710ee3c9eebSPatrick Williams# Create base Dockerfile. 711a18d9c57SPatrick Williamsdockerfile_base = f""" 712a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 71302871c91SPatrick Williams 71402871c91SPatrick Williams{mirror} 71502871c91SPatrick Williams 71602871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 71702871c91SPatrick Williams 7188949d3c3SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/" 71902871c91SPatrick Williams 720bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 721bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 722bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 723bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 724bb16ac14SPatrick Williams# the update. 725bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 726bb16ac14SPatrick Williams 72702871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 72802871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 72902871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 73050837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 731f79ce4c4SPatrick Williams ( apt-get install gpgv ubuntu-dbgsym-keyring || \ 73250837436SPatrick Williams ( apt-get install -yy dirmngr && \ 73350837436SPatrick Williams apt-key adv --keyserver keyserver.ubuntu.com \ 73434ec77e8SAdrian Ambrożewicz {proxy_keyserver} \ 73550837436SPatrick Williams --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 73602871c91SPatrick Williams 73702871c91SPatrick Williams# Parse the current repo list into a debug repo list 738e08ffba8SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' \ 739e08ffba8SPatrick Williams /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 74002871c91SPatrick Williams 74102871c91SPatrick Williams# Remove non-existent debug repos 74241d86218SPatrick WilliamsRUN sed -i '/-\\(backports\\|security\\) /d' /etc/apt/sources.list.d/debug.list 74302871c91SPatrick Williams 74402871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 74502871c91SPatrick Williams 74602871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 74702871c91SPatrick Williams autoconf \ 74802871c91SPatrick Williams autoconf-archive \ 749af49ed51SAndrew Geissler bison \ 75027a646b6SPatrick Williams clang-15 \ 75127a646b6SPatrick Williams clang-format-15 \ 75227a646b6SPatrick Williams clang-tidy-15 \ 75327a646b6SPatrick Williams clang-tools-15 \ 754af49ed51SAndrew Geissler cmake \ 755af49ed51SAndrew Geissler curl \ 756af49ed51SAndrew Geissler dbus \ 757af49ed51SAndrew Geissler device-tree-compiler \ 758af49ed51SAndrew Geissler flex \ 759af49ed51SAndrew Geissler g++-12 \ 760af49ed51SAndrew Geissler gcc-12 \ 761af49ed51SAndrew Geissler git \ 76202871c91SPatrick Williams iproute2 \ 763af49ed51SAndrew Geissler iputils-ping \ 764524a331cSManojkiran Eda libaudit-dev \ 765af49ed51SAndrew Geissler libc6-dbg \ 766af49ed51SAndrew Geissler libc6-dev \ 767af49ed51SAndrew Geissler libconfig++-dev \ 768af49ed51SAndrew Geissler libcryptsetup-dev \ 769af49ed51SAndrew Geissler libdbus-1-dev \ 770af49ed51SAndrew Geissler libevdev-dev \ 771af49ed51SAndrew Geissler libgpiod-dev \ 772af49ed51SAndrew Geissler libi2c-dev \ 773af49ed51SAndrew Geissler libjpeg-dev \ 774af49ed51SAndrew Geissler libjson-perl \ 775af49ed51SAndrew Geissler libldap2-dev \ 776af49ed51SAndrew Geissler libmimetic-dev \ 77702871c91SPatrick Williams libnl-3-dev \ 77802871c91SPatrick Williams libnl-genl-3-dev \ 77902871c91SPatrick Williams libpam0g-dev \ 78002871c91SPatrick Williams libpciaccess-dev \ 781af49ed51SAndrew Geissler libperlio-gzip-perl \ 782af49ed51SAndrew Geissler libpng-dev \ 783af49ed51SAndrew Geissler libprotobuf-dev \ 784af49ed51SAndrew Geissler libsnmp-dev \ 785af49ed51SAndrew Geissler libssl-dev \ 786af49ed51SAndrew Geissler libsystemd-dev \ 787af49ed51SAndrew Geissler libtool \ 788af49ed51SAndrew Geissler liburing2-dbgsym \ 789af49ed51SAndrew Geissler liburing-dev \ 79002871c91SPatrick Williams libxml2-utils \ 7910eedeedaSPatrick Williams libxml-simple-perl \ 792af49ed51SAndrew Geissler ninja-build \ 793af49ed51SAndrew Geissler npm \ 794af49ed51SAndrew Geissler pkg-config \ 795af49ed51SAndrew Geissler protobuf-compiler \ 796af49ed51SAndrew Geissler python3 \ 797af49ed51SAndrew Geissler python3-dev\ 798af49ed51SAndrew Geissler python3-git \ 799af49ed51SAndrew Geissler python3-mako \ 800af49ed51SAndrew Geissler python3-pip \ 801af49ed51SAndrew Geissler python3-setuptools \ 802af49ed51SAndrew Geissler python3-socks \ 803af49ed51SAndrew Geissler python3-yaml \ 8049adf68d6SJohn Wedig rsync \ 805af49ed51SAndrew Geissler shellcheck \ 806af49ed51SAndrew Geissler sudo \ 807af49ed51SAndrew Geissler systemd \ 808af49ed51SAndrew Geissler valgrind \ 809af49ed51SAndrew Geissler valgrind-dbg \ 810b565f825SAndrew Geissler vim \ 811af49ed51SAndrew Geissler wget \ 812af49ed51SAndrew Geissler xxd 81302871c91SPatrick Williams 8145b08dc6bSPatrick Williams# Kinetic comes with GCC-12, so skip this. 8155b08dc6bSPatrick Williams#RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 12 \ 8165b08dc6bSPatrick Williams# --slave /usr/bin/g++ g++ /usr/bin/g++-12 \ 8175b08dc6bSPatrick Williams# --slave /usr/bin/gcov gcov /usr/bin/gcov-12 \ 8185b08dc6bSPatrick Williams# --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 \ 8195b08dc6bSPatrick Williams# --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 8205b08dc6bSPatrick Williams#RUN update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 12 82102871c91SPatrick Williams 82227a646b6SPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 1000 \ 82327a646b6SPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-15 \ 82427a646b6SPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-15 \ 82527a646b6SPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-15 \ 826e08ffba8SPatrick Williams --slave /usr/bin/run-clang-tidy run-clang-tidy.py \ 827e08ffba8SPatrick Williams /usr/bin/run-clang-tidy-15 \ 82827a646b6SPatrick Williams --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-15 82902871c91SPatrick Williams 83050837436SPatrick Williams""" 83150837436SPatrick Williams 83250837436SPatrick Williamsif is_automated_ci_build: 83350837436SPatrick Williams dockerfile_base += f""" 83450837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 83550837436SPatrick Williams# to re-run `apt-get update` daily. 836ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 83750837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 83850837436SPatrick Williams 83950837436SPatrick Williams""" 84050837436SPatrick Williams 84141d86218SPatrick Williamsdockerfile_base += """ 842*818023dfSPatrick WilliamsRUN pip3 install \ 843*818023dfSPatrick Williams beautysh \ 844*818023dfSPatrick Williams black \ 845*818023dfSPatrick Williams codespell \ 846*818023dfSPatrick Williams flake8 \ 847*818023dfSPatrick Williams gitlint \ 848*818023dfSPatrick Williams inflection \ 849*818023dfSPatrick Williams isort \ 850*818023dfSPatrick Williams jsonschema \ 851*818023dfSPatrick Williams meson==0.63.0 \ 852*818023dfSPatrick Williams protobuf \ 853*818023dfSPatrick Williams requests 854b08ddf77SPatrick Williams 855b08ddf77SPatrick WilliamsRUN npm install -g \ 856b08ddf77SPatrick Williams eslint@latest eslint-plugin-json@latest \ 8577d41f6d2SPatrick Williams markdownlint-cli@latest \ 858b08ddf77SPatrick Williams prettier@latest 859fb9948a3SEd Tanous""" 860fb9948a3SEd Tanous 861ee3c9eebSPatrick Williams# Build the base and stage docker images. 862ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 863ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 864ee3c9eebSPatrick WilliamsPackage.generate_all() 86502871c91SPatrick Williams 866ee3c9eebSPatrick Williams# Create the final Dockerfile. 867a18d9c57SPatrick Williamsdockerfile = f""" 86802871c91SPatrick Williams# Build the final output image 869a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 870ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 87102871c91SPatrick Williams 87202871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 87302871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 87402871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 875ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 87602871c91SPatrick Williams 87702871c91SPatrick Williams# Final configuration for the workspace 8786825a018SJosh LehanRUN grep -q {gid} /etc/group || groupadd -f -g {gid} {username} 87902871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 880e08ffba8SPatrick WilliamsRUN grep -q {uid} /etc/passwd || \ 881e08ffba8SPatrick Williams useradd -d {homedir} -m -u {uid} -g {gid} {username} 88202871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 88302871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 88402871c91SPatrick Williams 885305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 886305a9a5dSAndrew Geissler# and data installs 8877bb00b13SAndrew GeisslerRUN chown -R {username}:{username} /usr/local/share 888305a9a5dSAndrew Geissler 88902871c91SPatrick Williams{proxy_cmd} 89002871c91SPatrick Williams 89102871c91SPatrick WilliamsRUN /bin/bash 89202871c91SPatrick Williams""" 89302871c91SPatrick Williams 894a18d9c57SPatrick Williams# Do the final docker build 895ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 896ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 897ee3c9eebSPatrick Williams 89800536fbeSPatrick Williams# Print the tag of the final image. 89900536fbeSPatrick Williamsprint(docker_final_img_name) 900