102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1050837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1150837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1250837436SPatrick Williams# (ex. Jenkins)> 1302871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1402871c91SPatrick Williams# repositories> 1502871c91SPatrick Williams# default is master, which will be used if input branch not 1602871c91SPatrick Williams# provided or not found 1702871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1802871c91SPatrick Williams# default ones in /etc/apt/sources.list> 1902871c91SPatrick Williams# default is empty, and no mirror is used. 2002871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2102871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2202871c91SPatrick Williams 2302871c91SPatrick Williamsimport os 2402871c91SPatrick Williamsimport sys 25b16f3e20SPatrick Williamsimport threading 26a18d9c57SPatrick Williamsfrom datetime import date 27a18d9c57SPatrick Williamsfrom hashlib import sha256 28ee3c9eebSPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional 2902871c91SPatrick Williams 3041d86218SPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 3141d86218SPatrick Williams 32ee3c9eebSPatrick Williamstry: 33ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 34ee3c9eebSPatrick Williams from typing import TypedDict 3541d86218SPatrick Williamsexcept Exception: 36ee3c9eebSPatrick Williams 37ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 38ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 3941d86218SPatrick Williams def __init_subclass__(cls, **kwargs: Any) -> None: 40ee3c9eebSPatrick Williams super().__init_subclass__() 41ee3c9eebSPatrick Williams 42ee3c9eebSPatrick Williams 43ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 44aae36d18SPatrick Williamsprefix = "/usr/local" 4502871c91SPatrick Williamsproc_count = nproc().strip() 4602871c91SPatrick Williams 47ee3c9eebSPatrick Williams 48ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 49ee3c9eebSPatrick Williams """Package Definition for packages dictionary.""" 50ee3c9eebSPatrick Williams 51ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 52ee3c9eebSPatrick Williams rev: str 53ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 54ee3c9eebSPatrick Williams url: Callable[[str, str], str] 55ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 56ee3c9eebSPatrick Williams depends: Iterable[str] 57ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 58ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 59ee3c9eebSPatrick Williams build_type: str 60ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 61ee3c9eebSPatrick Williams build_steps: Iterable[str] 62ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 63ee3c9eebSPatrick Williams config_flags: Iterable[str] 64ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 65ee3c9eebSPatrick Williams config_env: Iterable[str] 66ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 67ee3c9eebSPatrick Williams # before config / build / install. 68ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 696bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 706bce2ca1SPatrick Williams custom_post_install: Iterable[str] 71ee3c9eebSPatrick Williams 72ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 73ee3c9eebSPatrick Williams __tag: str 74ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 75ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 76ee3c9eebSPatrick Williams 7702871c91SPatrick Williams 787204324cSPatrick Williams# Packages to include in image. 797204324cSPatrick Williamspackages = { 80ee3c9eebSPatrick Williams "boost": PackageDef( 81c1977839SPatrick Williams rev="1.80.0", 82ee3c9eebSPatrick Williams url=( 835f2549eaSPatrick Williams lambda pkg, rev: f"https://downloads.yoctoproject.org/mirror/sources/{pkg}_{rev.replace('.', '_')}.tar.bz2" 842abc4a48SPatrick Williams ), 85ee3c9eebSPatrick Williams build_type="custom", 86ee3c9eebSPatrick Williams build_steps=[ 87aae36d18SPatrick Williams f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 88aae36d18SPatrick Williams "./b2", 89aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 90aae36d18SPatrick Williams ], 91ee3c9eebSPatrick Williams ), 92ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 93c1977839SPatrick Williams rev="v1.3.2", 94ee3c9eebSPatrick Williams build_type="custom", 95ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 96ee3c9eebSPatrick Williams ), 97c7198558SEd Tanous "danmar/cppcheck": PackageDef( 98be4bd084SPatrick Williams rev="2.9", 99c7198558SEd Tanous build_type="cmake", 100c7198558SEd Tanous ), 101ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 102ee3c9eebSPatrick Williams rev="v1.9.1", 103ee3c9eebSPatrick Williams build_type="cmake", 104ee3c9eebSPatrick Williams config_flags=[ 105aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 106aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 107aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 108aae36d18SPatrick Williams ], 109ee3c9eebSPatrick Williams ), 110ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 111652d8aeaSWilliam A. Kennington III rev="9.1.0", 112ee3c9eebSPatrick Williams build_type="cmake", 113ee3c9eebSPatrick Williams config_flags=[ 114aae36d18SPatrick Williams "-DFMT_DOC=OFF", 115aae36d18SPatrick Williams "-DFMT_TEST=OFF", 116aae36d18SPatrick Williams ], 117ee3c9eebSPatrick Williams ), 118ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 119c1977839SPatrick Williams rev="4.2.1", 120ee3c9eebSPatrick Williams build_type="custom", 121ee3c9eebSPatrick Williams build_steps=[ 122aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 123aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 124aae36d18SPatrick Williams ], 125ee3c9eebSPatrick Williams ), 126ed9414e8SPatrick Williams # release-1.12.1 127ee3c9eebSPatrick Williams "google/googletest": PackageDef( 128ed9414e8SPatrick Williams rev="58d77fa8070e8cec2dc1ed015d66b454c8d78850", 129ee3c9eebSPatrick Williams build_type="cmake", 1304dd32c02SWilliam A. Kennington III config_env=["CXXFLAGS=-std=c++20"], 131ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 132ee3c9eebSPatrick Williams ), 133ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 134c1977839SPatrick Williams rev="v3.11.2", 1356bce2ca1SPatrick Williams build_type="cmake", 1366bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1376bce2ca1SPatrick Williams custom_post_install=[ 138aae36d18SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 139aae36d18SPatrick Williams ], 140ee3c9eebSPatrick Williams ), 14102871c91SPatrick Williams # Snapshot from 2019-05-24 142ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 143ee3c9eebSPatrick Williams rev="v1.15", 144ee3c9eebSPatrick Williams build_type="make", 145ee3c9eebSPatrick Williams ), 146*001055b8SPatrick Williams # dev-6.0 2022-11-28 147ee3c9eebSPatrick Williams "openbmc/linux": PackageDef( 148*001055b8SPatrick Williams rev="1b16243b004ce4d977a9f3b9d9e715cf5028f867", 149ee3c9eebSPatrick Williams build_type="custom", 150ee3c9eebSPatrick Williams build_steps=[ 151aae36d18SPatrick Williams f"make -j{proc_count} defconfig", 152aae36d18SPatrick Williams f"make INSTALL_HDR_PATH={prefix} headers_install", 153aae36d18SPatrick Williams ], 154ee3c9eebSPatrick Williams ), 155ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 156ee3c9eebSPatrick Williams rev="LibVNCServer-0.9.13", 157ee3c9eebSPatrick Williams build_type="cmake", 158ee3c9eebSPatrick Williams ), 159ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 160c1977839SPatrick Williams rev="9.0.0", 161ee3c9eebSPatrick Williams build_type="cmake", 162ee3c9eebSPatrick Williams ), 16302871c91SPatrick Williams # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 164ee3c9eebSPatrick Williams "CPPAlliance/url": PackageDef( 165ab640cdaSEd Tanous rev="d740a92d38e3a8f4d5b2153f53b82f1c98e312ab", 166eed466e3SEd Tanous build_type="custom", 167eed466e3SEd Tanous build_steps=[f"cp -a include/** {prefix}/include/"], 168ee3c9eebSPatrick Williams ), 169ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 170c1977839SPatrick Williams rev="v0.7", 171ee3c9eebSPatrick Williams build_type="cmake", 172ee3c9eebSPatrick Williams config_flags=[ 1730eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1740eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 175aae36d18SPatrick Williams ], 176ee3c9eebSPatrick Williams ), 177ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 178ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 179ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 180ee3c9eebSPatrick Williams build_type="meson", 181ee3c9eebSPatrick Williams config_flags=[ 182aae36d18SPatrick Williams "-Dexamples=false", 183aae36d18SPatrick Williams "-Dtests=disabled", 184aae36d18SPatrick Williams ], 185ee3c9eebSPatrick Williams ), 186ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 187ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 188ee3c9eebSPatrick Williams build_type="meson", 1894fe87776SWilliam A. Kennington III config_flags=["-Dgenerate_md=false"], 190ee3c9eebSPatrick Williams ), 191ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 192ee3c9eebSPatrick Williams depends=[ 19383394610SPatrick Williams "USCiLab/cereal", 19483394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 19583394610SPatrick Williams "openbmc/sdbusplus", 19683394610SPatrick Williams "openbmc/sdeventplus", 197aae36d18SPatrick Williams ], 198f79ce4c4SPatrick Williams build_type="meson", 199ee3c9eebSPatrick Williams config_flags=[ 2006c98f280SWilliam A. Kennington III "-Dlibonly=true", 2016c98f280SWilliam A. Kennington III "-Dtests=disabled", 2025eabdae9SPatrick Williams f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml", 203aae36d18SPatrick Williams ], 204ee3c9eebSPatrick Williams ), 205ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 206ee3c9eebSPatrick Williams depends=[ 20711e5762cSBrad Bishop "CLIUtils/CLI11", 20870af95caSPatrick Williams "boost", 20983394610SPatrick Williams "leethomason/tinyxml2", 21070af95caSPatrick Williams "openbmc/phosphor-dbus-interfaces", 21183394610SPatrick Williams "openbmc/phosphor-logging", 21283394610SPatrick Williams "openbmc/sdbusplus", 213aae36d18SPatrick Williams ], 2141197e359SBrad Bishop build_type="meson", 2151197e359SBrad Bishop config_flags=[ 2161197e359SBrad Bishop "-Dtests=disabled", 2171197e359SBrad Bishop ], 218ee3c9eebSPatrick Williams ), 2191c19e453SManojkiran Eda "openbmc/libpldm": PackageDef( 220ee3c9eebSPatrick Williams build_type="meson", 221ee3c9eebSPatrick Williams config_flags=[ 222aae36d18SPatrick Williams "-Doem-ibm=enabled", 223aae36d18SPatrick Williams "-Dtests=disabled", 224aae36d18SPatrick Williams ], 225ee3c9eebSPatrick Williams ), 226ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 227ee3c9eebSPatrick Williams build_type="meson", 228ee3c9eebSPatrick Williams custom_post_dl=[ 229aae36d18SPatrick Williams "cd tools", 230aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 231aae36d18SPatrick Williams "cd ..", 232aae36d18SPatrick Williams ], 233ee3c9eebSPatrick Williams config_flags=[ 234aae36d18SPatrick Williams "-Dexamples=disabled", 235aae36d18SPatrick Williams "-Dtests=disabled", 236aae36d18SPatrick Williams ], 237b16f3e20SPatrick Williams ), 238ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 23970af95caSPatrick Williams depends=[ 24070af95caSPatrick Williams "Naios/function2", 24170af95caSPatrick Williams "openbmc/stdplus", 24270af95caSPatrick Williams ], 243ee3c9eebSPatrick Williams build_type="meson", 244ee3c9eebSPatrick Williams config_flags=[ 245ee3c9eebSPatrick Williams "-Dexamples=false", 246ee3c9eebSPatrick Williams "-Dtests=disabled", 247ee3c9eebSPatrick Williams ], 248ee3c9eebSPatrick Williams ), 249ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 25070af95caSPatrick Williams depends=[ 25170af95caSPatrick Williams "fmtlib/fmt", 252ca1bf0c0SWilliam A. Kennington III "google/googletest", 253ca1bf0c0SWilliam A. Kennington III "Naios/function2", 25470af95caSPatrick Williams ], 255ee3c9eebSPatrick Williams build_type="meson", 256ee3c9eebSPatrick Williams config_flags=[ 257ee3c9eebSPatrick Williams "-Dexamples=false", 258ee3c9eebSPatrick Williams "-Dtests=disabled", 259ca1bf0c0SWilliam A. Kennington III "-Dgtest=enabled", 260ee3c9eebSPatrick Williams ], 261ee3c9eebSPatrick Williams ), 262ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 26302871c91SPatrick Williams 26402871c91SPatrick Williams# Define common flags used for builds 26502871c91SPatrick Williamsconfigure_flags = " ".join( 26602871c91SPatrick Williams [ 26702871c91SPatrick Williams f"--prefix={prefix}", 26802871c91SPatrick Williams ] 26902871c91SPatrick Williams) 27002871c91SPatrick Williamscmake_flags = " ".join( 27102871c91SPatrick Williams [ 27202871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 2730f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 27402871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 2750f2086b3SPatrick Williams "-GNinja", 2760f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 27702871c91SPatrick Williams ] 27802871c91SPatrick Williams) 27902871c91SPatrick Williamsmeson_flags = " ".join( 28002871c91SPatrick Williams [ 28102871c91SPatrick Williams "--wrap-mode=nodownload", 28202871c91SPatrick Williams f"-Dprefix={prefix}", 28302871c91SPatrick Williams ] 28402871c91SPatrick Williams) 28502871c91SPatrick Williams 286ee3c9eebSPatrick Williams 287ee3c9eebSPatrick Williamsclass Package(threading.Thread): 288ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 289ee3c9eebSPatrick Williams 290ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 291ee3c9eebSPatrick Williams Package.generate_all(). 292ee3c9eebSPatrick Williams """ 293ee3c9eebSPatrick Williams 294ee3c9eebSPatrick Williams # Copy the packages dictionary. 295ee3c9eebSPatrick Williams packages = packages.copy() 296ee3c9eebSPatrick Williams 297ee3c9eebSPatrick Williams # Lock used for thread-safety. 298ee3c9eebSPatrick Williams lock = threading.Lock() 299ee3c9eebSPatrick Williams 300ee3c9eebSPatrick Williams def __init__(self, pkg: str): 301ee3c9eebSPatrick Williams """pkg - The name of this package (ex. foo/bar )""" 302ee3c9eebSPatrick Williams super(Package, self).__init__() 303ee3c9eebSPatrick Williams 304ee3c9eebSPatrick Williams self.package = pkg 305ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 306ee3c9eebSPatrick Williams 307ee3c9eebSPatrick Williams # Reference to this package's 308ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 309ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 310ee3c9eebSPatrick Williams 311ee3c9eebSPatrick Williams def run(self) -> None: 312ee3c9eebSPatrick Williams """Thread 'run' function. Builds the Docker stage.""" 313ee3c9eebSPatrick Williams 314ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 315ee3c9eebSPatrick Williams self._update_rev() 316ee3c9eebSPatrick Williams 317ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 318ee3c9eebSPatrick Williams # This section is locked because we are looking into another 319ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 320ee3c9eebSPatrick Williams Package.lock.acquire() 321ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 322ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 323ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 324ee3c9eebSPatrick Williams ] 325ee3c9eebSPatrick Williams Package.lock.release() 326ee3c9eebSPatrick Williams 327ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 328ee3c9eebSPatrick Williams # for the "COPY" commands. 329ee3c9eebSPatrick Williams for deppkg in deps: 330ee3c9eebSPatrick Williams deppkg.join() 331ee3c9eebSPatrick Williams 332ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 333ee3c9eebSPatrick Williams dockerfile = f""" 334ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 335ee3c9eebSPatrick Williams{self._df_copycmds()} 336ee3c9eebSPatrick Williams{self._df_build()} 337ee3c9eebSPatrick Williams""" 338ee3c9eebSPatrick Williams 339ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 340ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 341ee3c9eebSPatrick Williams # which can be accessed by other threads. 342ee3c9eebSPatrick Williams Package.lock.acquire() 343ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 344ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 345ee3c9eebSPatrick Williams Package.lock.release() 346ee3c9eebSPatrick Williams 347ee3c9eebSPatrick Williams # Do the build / save any exceptions. 348ee3c9eebSPatrick Williams try: 349ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 350ee3c9eebSPatrick Williams except Exception as e: 351ee3c9eebSPatrick Williams self.exception = e 352ee3c9eebSPatrick Williams 353ee3c9eebSPatrick Williams @classmethod 354ee3c9eebSPatrick Williams def generate_all(cls) -> None: 355ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 356ee3c9eebSPatrick Williams 357ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 358ee3c9eebSPatrick Williams package 'depends' specifications. 359ee3c9eebSPatrick Williams """ 360ee3c9eebSPatrick Williams 361ee3c9eebSPatrick Williams # Create a Package for each defined package. 362ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 363ee3c9eebSPatrick Williams 364ee3c9eebSPatrick Williams # Start building them all. 3656dbd7807SPatrick Williams # This section is locked because threads depend on each other, 3666dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 3676dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 3686dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 3696dbd7807SPatrick Williams Package.lock.acquire() 370ee3c9eebSPatrick Williams for t in pkg_threads: 371ee3c9eebSPatrick Williams t.start() 3726dbd7807SPatrick Williams Package.lock.release() 373ee3c9eebSPatrick Williams 374ee3c9eebSPatrick Williams # Wait for completion. 375ee3c9eebSPatrick Williams for t in pkg_threads: 376ee3c9eebSPatrick Williams t.join() 377ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 378ee3c9eebSPatrick Williams if t.exception: 379ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 380ee3c9eebSPatrick Williams raise t.exception 381ee3c9eebSPatrick Williams 382ee3c9eebSPatrick Williams @staticmethod 383ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 384ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 385ee3c9eebSPatrick Williams into the final image. 386ee3c9eebSPatrick Williams """ 387ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 388ee3c9eebSPatrick Williams 389ee3c9eebSPatrick Williams @classmethod 390ee3c9eebSPatrick Williams def depcache(cls) -> str: 391ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 392ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 393ee3c9eebSPatrick Williams """ 394ee3c9eebSPatrick Williams 395ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 396ee3c9eebSPatrick Williams depcache = "" 397ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 398ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 399ee3c9eebSPatrick Williams return depcache 400ee3c9eebSPatrick Williams 401ee3c9eebSPatrick Williams def _update_rev(self) -> None: 402ee3c9eebSPatrick Williams """Look up the HEAD for missing a static rev.""" 403ee3c9eebSPatrick Williams 404ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 405ee3c9eebSPatrick Williams return 406ee3c9eebSPatrick Williams 40765b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 40865b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 40965b21fb9SPatrick Williams print( 41065b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 41165b21fb9SPatrick Williams file=sys.stderr, 41265b21fb9SPatrick Williams ) 41365b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 41465b21fb9SPatrick Williams return 41565b21fb9SPatrick Williams 416ee3c9eebSPatrick Williams # Ask Github for all the branches. 41705fb2a0aSPatrick Williams lookup = git( 41805fb2a0aSPatrick Williams "ls-remote", "--heads", f"https://github.com/{self.package}" 41905fb2a0aSPatrick Williams ) 420ee3c9eebSPatrick Williams 421ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 422ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 423ee3c9eebSPatrick Williams Package.lock.acquire() 424ee3c9eebSPatrick Williams for line in lookup.split("\n"): 425ee3c9eebSPatrick Williams if f"refs/heads/{branch}" in line: 426ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 427c7d73646SPatrick Williams elif ( 428c7d73646SPatrick Williams "refs/heads/master" in line or "refs/heads/main" in line 429c7d73646SPatrick Williams ) and "rev" not in self.pkg_def: 430ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 431ee3c9eebSPatrick Williams Package.lock.release() 432ee3c9eebSPatrick Williams 433ee3c9eebSPatrick Williams def _stagename(self) -> str: 434ee3c9eebSPatrick Williams """Create a name for the Docker stage associated with this pkg.""" 435ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 436ee3c9eebSPatrick Williams 437ee3c9eebSPatrick Williams def _url(self) -> str: 438ee3c9eebSPatrick Williams """Get the URL for this package.""" 439ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 440ee3c9eebSPatrick Williams 441ee3c9eebSPatrick Williams # If the lambda exists, call it. 442ee3c9eebSPatrick Williams if "url" in self.pkg_def: 443ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 444ee3c9eebSPatrick Williams 445ee3c9eebSPatrick Williams # Default to the github archive URL. 446ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 447ee3c9eebSPatrick Williams 448ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 449ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 450ee3c9eebSPatrick Williams 451ee3c9eebSPatrick Williams url = self._url() 452ee3c9eebSPatrick Williams if ".tar." not in url: 453ee3c9eebSPatrick Williams raise NotImplementedError( 454ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 455ee3c9eebSPatrick Williams ) 456ee3c9eebSPatrick Williams 457ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 458ee3c9eebSPatrick Williams 459ee3c9eebSPatrick Williams if url.endswith(".bz2"): 460ee3c9eebSPatrick Williams cmd += "j" 461ee3c9eebSPatrick Williams elif url.endswith(".gz"): 462ee3c9eebSPatrick Williams cmd += "z" 463ee3c9eebSPatrick Williams else: 464ee3c9eebSPatrick Williams raise NotImplementedError( 465ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 466ee3c9eebSPatrick Williams ) 467ee3c9eebSPatrick Williams 468ee3c9eebSPatrick Williams return cmd 469ee3c9eebSPatrick Williams 470ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 471ee3c9eebSPatrick Williams """Formulate the command necessary to 'cd' into the source dir.""" 472ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 473ee3c9eebSPatrick Williams 474ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 475ee3c9eebSPatrick Williams """Formulate the dockerfile snippet necessary to COPY all depends.""" 476ee3c9eebSPatrick Williams 477ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 478ee3c9eebSPatrick Williams return "" 479ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 480ee3c9eebSPatrick Williams 481ee3c9eebSPatrick Williams @staticmethod 482ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 483ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 484ee3c9eebSPatrick Williams packages into a Docker stage. 485ee3c9eebSPatrick Williams """ 486ee3c9eebSPatrick Williams 487ee3c9eebSPatrick Williams copy_cmds = "" 488ee3c9eebSPatrick Williams 489ee3c9eebSPatrick Williams # Sort the packages for consistency. 490ee3c9eebSPatrick Williams for p in sorted(pkgs): 491ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 492ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 493ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 494ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 495ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 496ee3c9eebSPatrick Williams 497ee3c9eebSPatrick Williams return copy_cmds 498ee3c9eebSPatrick Williams 499ee3c9eebSPatrick Williams def _df_build(self) -> str: 500ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 501ee3c9eebSPatrick Williams install a package into a Docker stage. 502ee3c9eebSPatrick Williams """ 503ee3c9eebSPatrick Williams 504ee3c9eebSPatrick Williams # Download and extract source. 505ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 506ee3c9eebSPatrick Williams 507ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 508ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 509ee3c9eebSPatrick Williams if custom_post_dl: 510ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 511ee3c9eebSPatrick Williams 512ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 513ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 514ee3c9eebSPatrick Williams if build_type == "autoconf": 515ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 516ee3c9eebSPatrick Williams elif build_type == "cmake": 517ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 518ee3c9eebSPatrick Williams elif build_type == "custom": 519ee3c9eebSPatrick Williams result += self._cmd_build_custom() 520ee3c9eebSPatrick Williams elif build_type == "make": 521ee3c9eebSPatrick Williams result += self._cmd_build_make() 522ee3c9eebSPatrick Williams elif build_type == "meson": 523ee3c9eebSPatrick Williams result += self._cmd_build_meson() 524ee3c9eebSPatrick Williams else: 525ee3c9eebSPatrick Williams raise NotImplementedError( 526ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 527ee3c9eebSPatrick Williams ) 528ee3c9eebSPatrick Williams 5296bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5306bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5316bce2ca1SPatrick Williams if custom_post_install: 5326bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5336bce2ca1SPatrick Williams 534ee3c9eebSPatrick Williams return result 535ee3c9eebSPatrick Williams 536ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 537ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 538ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 539ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 540ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 541ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 542ee3c9eebSPatrick Williams return result 543ee3c9eebSPatrick Williams 544ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 545ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 546ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 547ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 548ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 549ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 550ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 551ee3c9eebSPatrick Williams result += "cd .." 552ee3c9eebSPatrick Williams return result 553ee3c9eebSPatrick Williams 554ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 555ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 556ee3c9eebSPatrick Williams 557ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 558ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 559ee3c9eebSPatrick Williams 560ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 561ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 562ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 563ee3c9eebSPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 564ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 565ee3c9eebSPatrick Williams return result 566ee3c9eebSPatrick Williams 567ee3c9eebSPatrick Williams 568ee3c9eebSPatrick Williamsclass Docker: 569ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 570ee3c9eebSPatrick Williams 571ee3c9eebSPatrick Williams @staticmethod 572ee3c9eebSPatrick Williams def timestamp() -> str: 573ee3c9eebSPatrick Williams """Generate a timestamp for today using the ISO week.""" 574ee3c9eebSPatrick Williams today = date.today().isocalendar() 575ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 576ee3c9eebSPatrick Williams 577ee3c9eebSPatrick Williams @staticmethod 57841d86218SPatrick Williams def tagname(pkgname: Optional[str], dockerfile: str) -> str: 579ee3c9eebSPatrick Williams """Generate a tag name for a package using a hash of the Dockerfile.""" 580ee3c9eebSPatrick Williams result = docker_image_name 581ee3c9eebSPatrick Williams if pkgname: 582ee3c9eebSPatrick Williams result += "-" + pkgname 583ee3c9eebSPatrick Williams 584ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 585ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 586ee3c9eebSPatrick Williams 587ee3c9eebSPatrick Williams return result 588ee3c9eebSPatrick Williams 589ee3c9eebSPatrick Williams @staticmethod 590ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 591ee3c9eebSPatrick Williams """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 592ee3c9eebSPatrick Williams 593ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 594ee3c9eebSPatrick Williams if not force_build: 595ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 59605fb2a0aSPatrick Williams print( 59705fb2a0aSPatrick Williams f"Image {tag} already exists. Skipping.", file=sys.stderr 59805fb2a0aSPatrick Williams ) 599ee3c9eebSPatrick Williams return 600ee3c9eebSPatrick Williams 601ee3c9eebSPatrick Williams # Build it. 602ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 603ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 604ee3c9eebSPatrick Williams # progress but not polute stdout. Later on we output the final 605ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 606ee3c9eebSPatrick Williams # 607ee3c9eebSPatrick Williams # Other unusual flags: 608ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 609ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 610ee3c9eebSPatrick Williams docker.build( 611ee3c9eebSPatrick Williams proxy_args, 612ee3c9eebSPatrick Williams "--network=host", 613ee3c9eebSPatrick Williams "--force-rm", 614ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 615ee3c9eebSPatrick Williams "-t", 616ee3c9eebSPatrick Williams tag, 617ee3c9eebSPatrick Williams "-", 618ee3c9eebSPatrick Williams _in=dockerfile, 619ee3c9eebSPatrick Williams _out=( 620ee3c9eebSPatrick Williams lambda line: print( 621ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 622ee3c9eebSPatrick Williams ) 623ee3c9eebSPatrick Williams ), 624ee3c9eebSPatrick Williams ) 625ee3c9eebSPatrick Williams 626ee3c9eebSPatrick Williams 627ee3c9eebSPatrick Williams# Read a bunch of environment variables. 62805fb2a0aSPatrick Williamsdocker_image_name = os.environ.get( 62905fb2a0aSPatrick Williams "DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test" 63005fb2a0aSPatrick Williams) 631ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 632ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 6335b08dc6bSPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:kinetic") 634ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 635ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 636ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 637ee3c9eebSPatrick Williams 63865b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 63965b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 64065b21fb9SPatrick Williams 641ee3c9eebSPatrick Williams# Set up some common variables. 642ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 643ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 644ee3c9eebSPatrick Williamsgid = os.getgid() 645ee3c9eebSPatrick Williamsuid = os.getuid() 646ee3c9eebSPatrick Williams 6476825a018SJosh Lehan# Use well-known constants if user is root 6486825a018SJosh Lehanif username == "root": 6496825a018SJosh Lehan homedir = "/root" 6506825a018SJosh Lehan gid = 0 6516825a018SJosh Lehan uid = 0 6526825a018SJosh Lehan 653ee3c9eebSPatrick Williams# Determine the architecture for Docker. 654ee3c9eebSPatrick Williamsarch = uname("-m").strip() 655ee3c9eebSPatrick Williamsif arch == "ppc64le": 656ee3c9eebSPatrick Williams docker_base = "ppc64le/" 657ee3c9eebSPatrick Williamselif arch == "x86_64": 658ee3c9eebSPatrick Williams docker_base = "" 659051b05b7SThang Q. Nguyenelif arch == "aarch64": 660f98f1a8dSThang Q. Nguyen docker_base = "arm64v8/" 661ee3c9eebSPatrick Williamselse: 662ee3c9eebSPatrick Williams print( 663ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 664ee3c9eebSPatrick Williams file=sys.stderr, 665ee3c9eebSPatrick Williams ) 666ee3c9eebSPatrick Williams sys.exit(1) 667ee3c9eebSPatrick Williams 66802871c91SPatrick Williams# Special flags if setting up a deb mirror. 66902871c91SPatrick Williamsmirror = "" 67002871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 67102871c91SPatrick Williams mirror = f""" 67202871c91SPatrick WilliamsRUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 67302871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 67402871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 67502871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 67602871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 67702871c91SPatrick Williams""" 67802871c91SPatrick Williams 67902871c91SPatrick Williams# Special flags for proxying. 68002871c91SPatrick Williamsproxy_cmd = "" 68134ec77e8SAdrian Ambrożewiczproxy_keyserver = "" 68202871c91SPatrick Williamsproxy_args = [] 68302871c91SPatrick Williamsif http_proxy: 68402871c91SPatrick Williams proxy_cmd = f""" 68502871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 68602871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 68702871c91SPatrick Williams""" 68834ec77e8SAdrian Ambrożewicz proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 68934ec77e8SAdrian Ambrożewicz 69002871c91SPatrick Williams proxy_args.extend( 69102871c91SPatrick Williams [ 69202871c91SPatrick Williams "--build-arg", 69302871c91SPatrick Williams f"http_proxy={http_proxy}", 69402871c91SPatrick Williams "--build-arg", 695d461cd6aSLei YU f"https_proxy={http_proxy}", 69602871c91SPatrick Williams ] 69702871c91SPatrick Williams ) 69802871c91SPatrick Williams 699ee3c9eebSPatrick Williams# Create base Dockerfile. 700a18d9c57SPatrick Williamsdockerfile_base = f""" 701a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 70202871c91SPatrick Williams 70302871c91SPatrick Williams{mirror} 70402871c91SPatrick Williams 70502871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 70602871c91SPatrick Williams 7078949d3c3SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/" 70802871c91SPatrick Williams 709bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 710bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 711bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 712bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 713bb16ac14SPatrick Williams# the update. 714bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 715bb16ac14SPatrick Williams 71602871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 71702871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 71802871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 71950837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 720f79ce4c4SPatrick Williams ( apt-get install gpgv ubuntu-dbgsym-keyring || \ 72150837436SPatrick Williams ( apt-get install -yy dirmngr && \ 72250837436SPatrick Williams apt-key adv --keyserver keyserver.ubuntu.com \ 72334ec77e8SAdrian Ambrożewicz {proxy_keyserver} \ 72450837436SPatrick Williams --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 72502871c91SPatrick Williams 72602871c91SPatrick Williams# Parse the current repo list into a debug repo list 72702871c91SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 72802871c91SPatrick Williams 72902871c91SPatrick Williams# Remove non-existent debug repos 73041d86218SPatrick WilliamsRUN sed -i '/-\\(backports\\|security\\) /d' /etc/apt/sources.list.d/debug.list 73102871c91SPatrick Williams 73202871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 73302871c91SPatrick Williams 73402871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 735274e3a9eSPatrick Williams gcc-12 \ 736274e3a9eSPatrick Williams g++-12 \ 73702871c91SPatrick Williams libc6-dbg \ 73802871c91SPatrick Williams libc6-dev \ 73902871c91SPatrick Williams libtool \ 74002871c91SPatrick Williams bison \ 74102871c91SPatrick Williams libdbus-1-dev \ 74202871c91SPatrick Williams flex \ 74302871c91SPatrick Williams cmake \ 74402871c91SPatrick Williams python3 \ 74502871c91SPatrick Williams python3-dev\ 74602871c91SPatrick Williams python3-yaml \ 74702871c91SPatrick Williams python3-mako \ 74802871c91SPatrick Williams python3-pip \ 74902871c91SPatrick Williams python3-setuptools \ 75002871c91SPatrick Williams python3-git \ 75102871c91SPatrick Williams python3-socks \ 75202871c91SPatrick Williams pkg-config \ 75302871c91SPatrick Williams autoconf \ 75402871c91SPatrick Williams autoconf-archive \ 75502871c91SPatrick Williams libsystemd-dev \ 75602871c91SPatrick Williams systemd \ 75702871c91SPatrick Williams libssl-dev \ 75802871c91SPatrick Williams libevdev-dev \ 75902871c91SPatrick Williams libjpeg-dev \ 76002871c91SPatrick Williams libpng-dev \ 76102871c91SPatrick Williams ninja-build \ 76202871c91SPatrick Williams sudo \ 76302871c91SPatrick Williams curl \ 76402871c91SPatrick Williams git \ 76502871c91SPatrick Williams dbus \ 76602871c91SPatrick Williams iputils-ping \ 76727a646b6SPatrick Williams clang-15 \ 76827a646b6SPatrick Williams clang-format-15 \ 76927a646b6SPatrick Williams clang-tidy-15 \ 77027a646b6SPatrick Williams clang-tools-15 \ 77102871c91SPatrick Williams shellcheck \ 77202871c91SPatrick Williams npm \ 77302871c91SPatrick Williams iproute2 \ 77402871c91SPatrick Williams libnl-3-dev \ 77502871c91SPatrick Williams libnl-genl-3-dev \ 77602871c91SPatrick Williams libconfig++-dev \ 77702871c91SPatrick Williams libsnmp-dev \ 77802871c91SPatrick Williams valgrind \ 77902871c91SPatrick Williams valgrind-dbg \ 78002871c91SPatrick Williams libpam0g-dev \ 78102871c91SPatrick Williams xxd \ 78202871c91SPatrick Williams libi2c-dev \ 78302871c91SPatrick Williams wget \ 78402871c91SPatrick Williams libldap2-dev \ 78502871c91SPatrick Williams libprotobuf-dev \ 786dafe7529SWilliam A. Kennington III liburing-dev \ 7878949d3c3SPatrick Williams liburing2-dbgsym \ 78802871c91SPatrick Williams libperlio-gzip-perl \ 78902871c91SPatrick Williams libjson-perl \ 79002871c91SPatrick Williams protobuf-compiler \ 79102871c91SPatrick Williams libgpiod-dev \ 79202871c91SPatrick Williams device-tree-compiler \ 79302871c91SPatrick Williams libpciaccess-dev \ 79402871c91SPatrick Williams libmimetic-dev \ 79502871c91SPatrick Williams libxml2-utils \ 7960eedeedaSPatrick Williams libxml-simple-perl \ 7979adf68d6SJohn Wedig rsync \ 7989adf68d6SJohn Wedig libcryptsetup-dev 79902871c91SPatrick Williams 80087111bb7SManojkiran EdaRUN npm install -g eslint@latest eslint-plugin-json@latest 80187111bb7SManojkiran Eda 8025b08dc6bSPatrick Williams# Kinetic comes with GCC-12, so skip this. 8035b08dc6bSPatrick Williams#RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 12 \ 8045b08dc6bSPatrick Williams# --slave /usr/bin/g++ g++ /usr/bin/g++-12 \ 8055b08dc6bSPatrick Williams# --slave /usr/bin/gcov gcov /usr/bin/gcov-12 \ 8065b08dc6bSPatrick Williams# --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 \ 8075b08dc6bSPatrick Williams# --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 8085b08dc6bSPatrick Williams#RUN update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 12 80902871c91SPatrick Williams 81027a646b6SPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 1000 \ 81127a646b6SPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-15 \ 81227a646b6SPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-15 \ 81327a646b6SPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-15 \ 81427a646b6SPatrick Williams --slave /usr/bin/run-clang-tidy run-clang-tidy.py /usr/bin/run-clang-tidy-15 \ 81527a646b6SPatrick Williams --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-15 81602871c91SPatrick Williams 81750837436SPatrick Williams""" 81850837436SPatrick Williams 81950837436SPatrick Williamsif is_automated_ci_build: 82050837436SPatrick Williams dockerfile_base += f""" 82150837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 82250837436SPatrick Williams# to re-run `apt-get update` daily. 823ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 82450837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 82550837436SPatrick Williams 82650837436SPatrick Williams""" 82750837436SPatrick Williams 82841d86218SPatrick Williamsdockerfile_base += """ 82902871c91SPatrick WilliamsRUN pip3 install inflection 83002871c91SPatrick WilliamsRUN pip3 install pycodestyle 83102871c91SPatrick WilliamsRUN pip3 install jsonschema 832fb612a51SMichael ShenRUN pip3 install meson==0.63.0 8335278efb6SEd TanousRUN pip3 install packaging 83402871c91SPatrick WilliamsRUN pip3 install protobuf 835e6f120aaSManojkiran EdaRUN pip3 install codespell 836ca8c4a8bSEd TanousRUN pip3 install requests 837c2e7c938SPatrick WilliamsRUN pip3 install gitlint 838fb9948a3SEd Tanous""" 839fb9948a3SEd Tanous 840ee3c9eebSPatrick Williams# Build the base and stage docker images. 841ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 842ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 843ee3c9eebSPatrick WilliamsPackage.generate_all() 84402871c91SPatrick Williams 845ee3c9eebSPatrick Williams# Create the final Dockerfile. 846a18d9c57SPatrick Williamsdockerfile = f""" 84702871c91SPatrick Williams# Build the final output image 848a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 849ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 85002871c91SPatrick Williams 85102871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 85202871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 85302871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 854ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 85502871c91SPatrick Williams 85602871c91SPatrick Williams# Final configuration for the workspace 8576825a018SJosh LehanRUN grep -q {gid} /etc/group || groupadd -f -g {gid} {username} 85802871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 85902871c91SPatrick WilliamsRUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 86002871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 86102871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 86202871c91SPatrick Williams 863305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 864305a9a5dSAndrew Geissler# and data installs 8657bb00b13SAndrew GeisslerRUN chown -R {username}:{username} /usr/local/share 866305a9a5dSAndrew Geissler 86702871c91SPatrick Williams{proxy_cmd} 86802871c91SPatrick Williams 86902871c91SPatrick WilliamsRUN /bin/bash 87002871c91SPatrick Williams""" 87102871c91SPatrick Williams 872a18d9c57SPatrick Williams# Do the final docker build 873ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 874ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 875ee3c9eebSPatrick Williams 87600536fbeSPatrick Williams# Print the tag of the final image. 87700536fbeSPatrick Williamsprint(docker_final_img_name) 878