102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1050837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1150837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1250837436SPatrick Williams# (ex. Jenkins)> 1302871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1402871c91SPatrick Williams# repositories> 1502871c91SPatrick Williams# default is master, which will be used if input branch not 1602871c91SPatrick Williams# provided or not found 1702871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1802871c91SPatrick Williams# default ones in /etc/apt/sources.list> 1902871c91SPatrick Williams# default is empty, and no mirror is used. 2002871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2102871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2202871c91SPatrick Williams 2302871c91SPatrick Williamsimport os 2402871c91SPatrick Williamsimport sys 25b16f3e20SPatrick Williamsimport threading 26a18d9c57SPatrick Williamsfrom datetime import date 27a18d9c57SPatrick Williamsfrom hashlib import sha256 28ee3c9eebSPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 29ee3c9eebSPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional 3002871c91SPatrick Williams 31ee3c9eebSPatrick Williamstry: 32ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 33ee3c9eebSPatrick Williams from typing import TypedDict 34ee3c9eebSPatrick Williamsexcept: 35ee3c9eebSPatrick Williams 36ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 37ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 38ee3c9eebSPatrick Williams def __init_subclass__(cls, **kwargs): 39ee3c9eebSPatrick Williams super().__init_subclass__() 40ee3c9eebSPatrick Williams 41ee3c9eebSPatrick Williams 42ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 43aae36d18SPatrick Williamsprefix = "/usr/local" 4402871c91SPatrick Williamsproc_count = nproc().strip() 4502871c91SPatrick Williams 46ee3c9eebSPatrick Williams 47ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 48ee3c9eebSPatrick Williams """ Package Definition for packages dictionary. """ 49ee3c9eebSPatrick Williams 50ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 51ee3c9eebSPatrick Williams rev: str 52ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 53ee3c9eebSPatrick Williams url: Callable[[str, str], str] 54ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 55ee3c9eebSPatrick Williams depends: Iterable[str] 56ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 57ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 58ee3c9eebSPatrick Williams build_type: str 59ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 60ee3c9eebSPatrick Williams build_steps: Iterable[str] 61ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 62ee3c9eebSPatrick Williams config_flags: Iterable[str] 63ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 64ee3c9eebSPatrick Williams config_env: Iterable[str] 65ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 66ee3c9eebSPatrick Williams # before config / build / install. 67ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 686bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 696bce2ca1SPatrick Williams custom_post_install: Iterable[str] 70ee3c9eebSPatrick Williams 71ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 72ee3c9eebSPatrick Williams __tag: str 73ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 74ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 75ee3c9eebSPatrick Williams 7602871c91SPatrick Williams 777204324cSPatrick Williams# Packages to include in image. 787204324cSPatrick Williamspackages = { 79ee3c9eebSPatrick Williams "boost": PackageDef( 806c55c580SEd Tanous rev="1.79.0", 81ee3c9eebSPatrick Williams url=( 825f2549eaSPatrick Williams lambda pkg, rev: f"https://downloads.yoctoproject.org/mirror/sources/{pkg}_{rev.replace('.', '_')}.tar.bz2" 832abc4a48SPatrick Williams ), 84ee3c9eebSPatrick Williams build_type="custom", 85ee3c9eebSPatrick Williams build_steps=[ 86aae36d18SPatrick Williams f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 87aae36d18SPatrick Williams "./b2", 88aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 89aae36d18SPatrick Williams ], 90ee3c9eebSPatrick Williams ), 91ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 923f8b5294SPatrick Williams rev="3e4d1b84cab4891368d2179a61a7ba06a5693e7f", 93ee3c9eebSPatrick Williams build_type="custom", 94ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 95ee3c9eebSPatrick Williams ), 96c7198558SEd Tanous "danmar/cppcheck": PackageDef( 97c7198558SEd Tanous rev="27578e9c4c1f90c62b6938867735a054082e178e", 98c7198558SEd Tanous build_type="cmake", 99c7198558SEd Tanous ), 100ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 101ee3c9eebSPatrick Williams rev="v1.9.1", 102ee3c9eebSPatrick Williams build_type="cmake", 103ee3c9eebSPatrick Williams config_flags=[ 104aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 105aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 106aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 107aae36d18SPatrick Williams ], 108ee3c9eebSPatrick Williams ), 109ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 11063075013SWilly Tu rev="8.1.1", 111ee3c9eebSPatrick Williams build_type="cmake", 112ee3c9eebSPatrick Williams config_flags=[ 113aae36d18SPatrick Williams "-DFMT_DOC=OFF", 114aae36d18SPatrick Williams "-DFMT_TEST=OFF", 115aae36d18SPatrick Williams ], 116ee3c9eebSPatrick Williams ), 117ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 11896b43455SWilliam A. Kennington III rev="4.1.0", 119ee3c9eebSPatrick Williams build_type="custom", 120ee3c9eebSPatrick Williams build_steps=[ 121aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 122aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 123aae36d18SPatrick Williams ], 124ee3c9eebSPatrick Williams ), 125ed9414e8SPatrick Williams # release-1.12.1 126ee3c9eebSPatrick Williams "google/googletest": PackageDef( 127ed9414e8SPatrick Williams rev="58d77fa8070e8cec2dc1ed015d66b454c8d78850", 128ee3c9eebSPatrick Williams build_type="cmake", 1294dd32c02SWilliam A. Kennington III config_env=["CXXFLAGS=-std=c++20"], 130ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 131ee3c9eebSPatrick Williams ), 13202871c91SPatrick Williams # Release 2020-08-06 133ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 1347d1b2a1dSEd Tanous rev="v3.10.4", 1356bce2ca1SPatrick Williams build_type="cmake", 1366bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1376bce2ca1SPatrick Williams custom_post_install=[ 138aae36d18SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 139aae36d18SPatrick Williams ], 140ee3c9eebSPatrick Williams ), 14102871c91SPatrick Williams # Snapshot from 2019-05-24 142ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 143ee3c9eebSPatrick Williams rev="v1.15", 144ee3c9eebSPatrick Williams build_type="make", 145ee3c9eebSPatrick Williams ), 1460eedeedaSPatrick Williams # dev-5.8 2021-01-11 147ee3c9eebSPatrick Williams "openbmc/linux": PackageDef( 148ee3c9eebSPatrick Williams rev="3cc95ae40716e56f81b69615781f54c78079042d", 149ee3c9eebSPatrick Williams build_type="custom", 150ee3c9eebSPatrick Williams build_steps=[ 151aae36d18SPatrick Williams f"make -j{proc_count} defconfig", 152aae36d18SPatrick Williams f"make INSTALL_HDR_PATH={prefix} headers_install", 153aae36d18SPatrick Williams ], 154ee3c9eebSPatrick Williams ), 1550eedeedaSPatrick Williams # Snapshot from 2020-06-13 156ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 157ee3c9eebSPatrick Williams rev="LibVNCServer-0.9.13", 158ee3c9eebSPatrick Williams build_type="cmake", 159ee3c9eebSPatrick Williams ), 1600eedeedaSPatrick Williams # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_8.0.0.bb 161ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 162ee3c9eebSPatrick Williams rev="8.0.0", 163ee3c9eebSPatrick Williams build_type="cmake", 164ee3c9eebSPatrick Williams ), 16502871c91SPatrick Williams # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 166ee3c9eebSPatrick Williams "CPPAlliance/url": PackageDef( 167ab640cdaSEd Tanous rev="d740a92d38e3a8f4d5b2153f53b82f1c98e312ab", 168eed466e3SEd Tanous build_type="custom", 169eed466e3SEd Tanous build_steps=[f"cp -a include/** {prefix}/include/"], 170ee3c9eebSPatrick Williams ), 17168992895SPatrick Williams # version from meta-openembedded/meta-oe/dynamic-layers/networking-layer/recipes-devools/valijson/valijson_0.6.bb 172ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 17368992895SPatrick Williams rev="v0.6", 174ee3c9eebSPatrick Williams build_type="cmake", 175ee3c9eebSPatrick Williams config_flags=[ 1760eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1770eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 178aae36d18SPatrick Williams ], 179ee3c9eebSPatrick Williams ), 180ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 181ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 182ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 183ee3c9eebSPatrick Williams build_type="meson", 184ee3c9eebSPatrick Williams config_flags=[ 185aae36d18SPatrick Williams "-Dexamples=false", 186aae36d18SPatrick Williams "-Dtests=disabled", 187aae36d18SPatrick Williams ], 188ee3c9eebSPatrick Williams ), 189ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 190ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 191ee3c9eebSPatrick Williams build_type="meson", 1924fe87776SWilliam A. Kennington III config_flags=["-Dgenerate_md=false"], 193ee3c9eebSPatrick Williams ), 194ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 195ee3c9eebSPatrick Williams depends=[ 19683394610SPatrick Williams "USCiLab/cereal", 19783394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 19883394610SPatrick Williams "openbmc/sdbusplus", 19983394610SPatrick Williams "openbmc/sdeventplus", 200aae36d18SPatrick Williams ], 201f79ce4c4SPatrick Williams build_type="meson", 202ee3c9eebSPatrick Williams config_flags=[ 2035eabdae9SPatrick Williams f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml", 204aae36d18SPatrick Williams ], 205ee3c9eebSPatrick Williams ), 206ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 207ee3c9eebSPatrick Williams depends=[ 20883394610SPatrick Williams "boost", 20911e5762cSBrad Bishop "CLIUtils/CLI11", 21083394610SPatrick Williams "leethomason/tinyxml2", 21183394610SPatrick Williams "openbmc/phosphor-logging", 21283394610SPatrick Williams "openbmc/sdbusplus", 213aae36d18SPatrick Williams ], 2141197e359SBrad Bishop build_type="meson", 2151197e359SBrad Bishop config_flags=[ 2161197e359SBrad Bishop "-Dtests=disabled", 2171197e359SBrad Bishop ], 218ee3c9eebSPatrick Williams ), 219ee3c9eebSPatrick Williams "openbmc/pldm": PackageDef( 220ee3c9eebSPatrick Williams depends=[ 22183394610SPatrick Williams "CLIUtils/CLI11", 22283394610SPatrick Williams "boost", 22383394610SPatrick Williams "nlohmann/json", 22483394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 22583394610SPatrick Williams "openbmc/phosphor-logging", 22683394610SPatrick Williams "openbmc/sdbusplus", 22783394610SPatrick Williams "openbmc/sdeventplus", 228aae36d18SPatrick Williams ], 229ee3c9eebSPatrick Williams build_type="meson", 230ee3c9eebSPatrick Williams config_flags=[ 231aae36d18SPatrick Williams "-Dlibpldm-only=enabled", 232aae36d18SPatrick Williams "-Doem-ibm=enabled", 233aae36d18SPatrick Williams "-Dtests=disabled", 234aae36d18SPatrick Williams ], 235ee3c9eebSPatrick Williams ), 236ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 237ee3c9eebSPatrick Williams build_type="meson", 238ee3c9eebSPatrick Williams custom_post_dl=[ 239aae36d18SPatrick Williams "cd tools", 240aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 241aae36d18SPatrick Williams "cd ..", 242aae36d18SPatrick Williams ], 243ee3c9eebSPatrick Williams config_flags=[ 244aae36d18SPatrick Williams "-Dexamples=disabled", 245aae36d18SPatrick Williams "-Dtests=disabled", 246aae36d18SPatrick Williams ], 247b16f3e20SPatrick Williams ), 248ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 249ee3c9eebSPatrick Williams depends=["Naios/function2", "openbmc/stdplus"], 250ee3c9eebSPatrick Williams build_type="meson", 251ee3c9eebSPatrick Williams config_flags=[ 252ee3c9eebSPatrick Williams "-Dexamples=false", 253ee3c9eebSPatrick Williams "-Dtests=disabled", 254ee3c9eebSPatrick Williams ], 255ee3c9eebSPatrick Williams ), 256ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 257*d342ed5eSWilliam A. Kennington III depends=["fmtlib/fmt", "Naios/function2", "google/googletest"], 258ee3c9eebSPatrick Williams build_type="meson", 259ee3c9eebSPatrick Williams config_flags=[ 260ee3c9eebSPatrick Williams "-Dexamples=false", 261ee3c9eebSPatrick Williams "-Dtests=disabled", 262ee3c9eebSPatrick Williams ], 263ee3c9eebSPatrick Williams ), 264ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 26502871c91SPatrick Williams 26602871c91SPatrick Williams# Define common flags used for builds 26702871c91SPatrick Williamsconfigure_flags = " ".join( 26802871c91SPatrick Williams [ 26902871c91SPatrick Williams f"--prefix={prefix}", 27002871c91SPatrick Williams ] 27102871c91SPatrick Williams) 27202871c91SPatrick Williamscmake_flags = " ".join( 27302871c91SPatrick Williams [ 27402871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 2750f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 27602871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 2770f2086b3SPatrick Williams "-GNinja", 2780f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 27902871c91SPatrick Williams ] 28002871c91SPatrick Williams) 28102871c91SPatrick Williamsmeson_flags = " ".join( 28202871c91SPatrick Williams [ 28302871c91SPatrick Williams "--wrap-mode=nodownload", 28402871c91SPatrick Williams f"-Dprefix={prefix}", 28502871c91SPatrick Williams ] 28602871c91SPatrick Williams) 28702871c91SPatrick Williams 288ee3c9eebSPatrick Williams 289ee3c9eebSPatrick Williamsclass Package(threading.Thread): 290ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 291ee3c9eebSPatrick Williams 292ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 293ee3c9eebSPatrick Williams Package.generate_all(). 294ee3c9eebSPatrick Williams """ 295ee3c9eebSPatrick Williams 296ee3c9eebSPatrick Williams # Copy the packages dictionary. 297ee3c9eebSPatrick Williams packages = packages.copy() 298ee3c9eebSPatrick Williams 299ee3c9eebSPatrick Williams # Lock used for thread-safety. 300ee3c9eebSPatrick Williams lock = threading.Lock() 301ee3c9eebSPatrick Williams 302ee3c9eebSPatrick Williams def __init__(self, pkg: str): 303ee3c9eebSPatrick Williams """ pkg - The name of this package (ex. foo/bar ) """ 304ee3c9eebSPatrick Williams super(Package, self).__init__() 305ee3c9eebSPatrick Williams 306ee3c9eebSPatrick Williams self.package = pkg 307ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 308ee3c9eebSPatrick Williams 309ee3c9eebSPatrick Williams # Reference to this package's 310ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 311ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 312ee3c9eebSPatrick Williams 313ee3c9eebSPatrick Williams def run(self) -> None: 314ee3c9eebSPatrick Williams """ Thread 'run' function. Builds the Docker stage. """ 315ee3c9eebSPatrick Williams 316ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 317ee3c9eebSPatrick Williams self._update_rev() 318ee3c9eebSPatrick Williams 319ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 320ee3c9eebSPatrick Williams # This section is locked because we are looking into another 321ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 322ee3c9eebSPatrick Williams Package.lock.acquire() 323ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 324ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 325ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 326ee3c9eebSPatrick Williams ] 327ee3c9eebSPatrick Williams Package.lock.release() 328ee3c9eebSPatrick Williams 329ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 330ee3c9eebSPatrick Williams # for the "COPY" commands. 331ee3c9eebSPatrick Williams for deppkg in deps: 332ee3c9eebSPatrick Williams deppkg.join() 333ee3c9eebSPatrick Williams 334ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 335ee3c9eebSPatrick Williams dockerfile = f""" 336ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 337ee3c9eebSPatrick Williams{self._df_copycmds()} 338ee3c9eebSPatrick Williams{self._df_build()} 339ee3c9eebSPatrick Williams""" 340ee3c9eebSPatrick Williams 341ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 342ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 343ee3c9eebSPatrick Williams # which can be accessed by other threads. 344ee3c9eebSPatrick Williams Package.lock.acquire() 345ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 346ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 347ee3c9eebSPatrick Williams Package.lock.release() 348ee3c9eebSPatrick Williams 349ee3c9eebSPatrick Williams # Do the build / save any exceptions. 350ee3c9eebSPatrick Williams try: 351ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 352ee3c9eebSPatrick Williams except Exception as e: 353ee3c9eebSPatrick Williams self.exception = e 354ee3c9eebSPatrick Williams 355ee3c9eebSPatrick Williams @classmethod 356ee3c9eebSPatrick Williams def generate_all(cls) -> None: 357ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 358ee3c9eebSPatrick Williams 359ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 360ee3c9eebSPatrick Williams package 'depends' specifications. 361ee3c9eebSPatrick Williams """ 362ee3c9eebSPatrick Williams 363ee3c9eebSPatrick Williams # Create a Package for each defined package. 364ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 365ee3c9eebSPatrick Williams 366ee3c9eebSPatrick Williams # Start building them all. 3676dbd7807SPatrick Williams # This section is locked because threads depend on each other, 3686dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 3696dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 3706dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 3716dbd7807SPatrick Williams Package.lock.acquire() 372ee3c9eebSPatrick Williams for t in pkg_threads: 373ee3c9eebSPatrick Williams t.start() 3746dbd7807SPatrick Williams Package.lock.release() 375ee3c9eebSPatrick Williams 376ee3c9eebSPatrick Williams # Wait for completion. 377ee3c9eebSPatrick Williams for t in pkg_threads: 378ee3c9eebSPatrick Williams t.join() 379ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 380ee3c9eebSPatrick Williams if t.exception: 381ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 382ee3c9eebSPatrick Williams raise t.exception 383ee3c9eebSPatrick Williams 384ee3c9eebSPatrick Williams @staticmethod 385ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 386ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 387ee3c9eebSPatrick Williams into the final image. 388ee3c9eebSPatrick Williams """ 389ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 390ee3c9eebSPatrick Williams 391ee3c9eebSPatrick Williams @classmethod 392ee3c9eebSPatrick Williams def depcache(cls) -> str: 393ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 394ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 395ee3c9eebSPatrick Williams """ 396ee3c9eebSPatrick Williams 397ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 398ee3c9eebSPatrick Williams depcache = "" 399ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 400ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 401ee3c9eebSPatrick Williams return depcache 402ee3c9eebSPatrick Williams 403ee3c9eebSPatrick Williams def _update_rev(self) -> None: 404ee3c9eebSPatrick Williams """ Look up the HEAD for missing a static rev. """ 405ee3c9eebSPatrick Williams 406ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 407ee3c9eebSPatrick Williams return 408ee3c9eebSPatrick Williams 40965b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 41065b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 41165b21fb9SPatrick Williams print( 41265b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 41365b21fb9SPatrick Williams file=sys.stderr, 41465b21fb9SPatrick Williams ) 41565b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 41665b21fb9SPatrick Williams return 41765b21fb9SPatrick Williams 418ee3c9eebSPatrick Williams # Ask Github for all the branches. 419ee3c9eebSPatrick Williams lookup = git("ls-remote", "--heads", f"https://github.com/{self.package}") 420ee3c9eebSPatrick Williams 421ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 422ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 423ee3c9eebSPatrick Williams Package.lock.acquire() 424ee3c9eebSPatrick Williams for line in lookup.split("\n"): 425ee3c9eebSPatrick Williams if f"refs/heads/{branch}" in line: 426ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 427ee3c9eebSPatrick Williams elif f"refs/heads/master" in line and "rev" not in self.pkg_def: 428ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 429ee3c9eebSPatrick Williams Package.lock.release() 430ee3c9eebSPatrick Williams 431ee3c9eebSPatrick Williams def _stagename(self) -> str: 432ee3c9eebSPatrick Williams """ Create a name for the Docker stage associated with this pkg. """ 433ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 434ee3c9eebSPatrick Williams 435ee3c9eebSPatrick Williams def _url(self) -> str: 436ee3c9eebSPatrick Williams """ Get the URL for this package. """ 437ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 438ee3c9eebSPatrick Williams 439ee3c9eebSPatrick Williams # If the lambda exists, call it. 440ee3c9eebSPatrick Williams if "url" in self.pkg_def: 441ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 442ee3c9eebSPatrick Williams 443ee3c9eebSPatrick Williams # Default to the github archive URL. 444ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 445ee3c9eebSPatrick Williams 446ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 447ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 448ee3c9eebSPatrick Williams 449ee3c9eebSPatrick Williams url = self._url() 450ee3c9eebSPatrick Williams if ".tar." not in url: 451ee3c9eebSPatrick Williams raise NotImplementedError( 452ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 453ee3c9eebSPatrick Williams ) 454ee3c9eebSPatrick Williams 455ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 456ee3c9eebSPatrick Williams 457ee3c9eebSPatrick Williams if url.endswith(".bz2"): 458ee3c9eebSPatrick Williams cmd += "j" 459ee3c9eebSPatrick Williams elif url.endswith(".gz"): 460ee3c9eebSPatrick Williams cmd += "z" 461ee3c9eebSPatrick Williams else: 462ee3c9eebSPatrick Williams raise NotImplementedError( 463ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 464ee3c9eebSPatrick Williams ) 465ee3c9eebSPatrick Williams 466ee3c9eebSPatrick Williams return cmd 467ee3c9eebSPatrick Williams 468ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 469ee3c9eebSPatrick Williams """ Formulate the command necessary to 'cd' into the source dir. """ 470ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 471ee3c9eebSPatrick Williams 472ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 473ee3c9eebSPatrick Williams """ Formulate the dockerfile snippet necessary to COPY all depends. """ 474ee3c9eebSPatrick Williams 475ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 476ee3c9eebSPatrick Williams return "" 477ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 478ee3c9eebSPatrick Williams 479ee3c9eebSPatrick Williams @staticmethod 480ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 481ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 482ee3c9eebSPatrick Williams packages into a Docker stage. 483ee3c9eebSPatrick Williams """ 484ee3c9eebSPatrick Williams 485ee3c9eebSPatrick Williams copy_cmds = "" 486ee3c9eebSPatrick Williams 487ee3c9eebSPatrick Williams # Sort the packages for consistency. 488ee3c9eebSPatrick Williams for p in sorted(pkgs): 489ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 490ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 491ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 492ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 493ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 494ee3c9eebSPatrick Williams 495ee3c9eebSPatrick Williams return copy_cmds 496ee3c9eebSPatrick Williams 497ee3c9eebSPatrick Williams def _df_build(self) -> str: 498ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 499ee3c9eebSPatrick Williams install a package into a Docker stage. 500ee3c9eebSPatrick Williams """ 501ee3c9eebSPatrick Williams 502ee3c9eebSPatrick Williams # Download and extract source. 503ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 504ee3c9eebSPatrick Williams 505ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 506ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 507ee3c9eebSPatrick Williams if custom_post_dl: 508ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 509ee3c9eebSPatrick Williams 510ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 511ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 512ee3c9eebSPatrick Williams if build_type == "autoconf": 513ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 514ee3c9eebSPatrick Williams elif build_type == "cmake": 515ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 516ee3c9eebSPatrick Williams elif build_type == "custom": 517ee3c9eebSPatrick Williams result += self._cmd_build_custom() 518ee3c9eebSPatrick Williams elif build_type == "make": 519ee3c9eebSPatrick Williams result += self._cmd_build_make() 520ee3c9eebSPatrick Williams elif build_type == "meson": 521ee3c9eebSPatrick Williams result += self._cmd_build_meson() 522ee3c9eebSPatrick Williams else: 523ee3c9eebSPatrick Williams raise NotImplementedError( 524ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 525ee3c9eebSPatrick Williams ) 526ee3c9eebSPatrick Williams 5276bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5286bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5296bce2ca1SPatrick Williams if custom_post_install: 5306bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5316bce2ca1SPatrick Williams 532ee3c9eebSPatrick Williams return result 533ee3c9eebSPatrick Williams 534ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 535ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 536ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 537ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 538ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 539ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 540ee3c9eebSPatrick Williams return result 541ee3c9eebSPatrick Williams 542ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 543ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 544ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 545ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 546ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 547ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 548ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 549ee3c9eebSPatrick Williams result += "cd .." 550ee3c9eebSPatrick Williams return result 551ee3c9eebSPatrick Williams 552ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 553ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 554ee3c9eebSPatrick Williams 555ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 556ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 557ee3c9eebSPatrick Williams 558ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 559ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 560ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 561ee3c9eebSPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 562ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 563ee3c9eebSPatrick Williams return result 564ee3c9eebSPatrick Williams 565ee3c9eebSPatrick Williams 566ee3c9eebSPatrick Williamsclass Docker: 567ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 568ee3c9eebSPatrick Williams 569ee3c9eebSPatrick Williams @staticmethod 570ee3c9eebSPatrick Williams def timestamp() -> str: 571ee3c9eebSPatrick Williams """ Generate a timestamp for today using the ISO week. """ 572ee3c9eebSPatrick Williams today = date.today().isocalendar() 573ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 574ee3c9eebSPatrick Williams 575ee3c9eebSPatrick Williams @staticmethod 576ee3c9eebSPatrick Williams def tagname(pkgname: str, dockerfile: str) -> str: 577ee3c9eebSPatrick Williams """ Generate a tag name for a package using a hash of the Dockerfile. """ 578ee3c9eebSPatrick Williams result = docker_image_name 579ee3c9eebSPatrick Williams if pkgname: 580ee3c9eebSPatrick Williams result += "-" + pkgname 581ee3c9eebSPatrick Williams 582ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 583ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 584ee3c9eebSPatrick Williams 585ee3c9eebSPatrick Williams return result 586ee3c9eebSPatrick Williams 587ee3c9eebSPatrick Williams @staticmethod 588ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 589ee3c9eebSPatrick Williams """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 590ee3c9eebSPatrick Williams 591ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 592ee3c9eebSPatrick Williams if not force_build: 593ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 594ee3c9eebSPatrick Williams print(f"Image {tag} already exists. Skipping.", file=sys.stderr) 595ee3c9eebSPatrick Williams return 596ee3c9eebSPatrick Williams 597ee3c9eebSPatrick Williams # Build it. 598ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 599ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 600ee3c9eebSPatrick Williams # progress but not polute stdout. Later on we output the final 601ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 602ee3c9eebSPatrick Williams # 603ee3c9eebSPatrick Williams # Other unusual flags: 604ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 605ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 606ee3c9eebSPatrick Williams docker.build( 607ee3c9eebSPatrick Williams proxy_args, 608ee3c9eebSPatrick Williams "--network=host", 609ee3c9eebSPatrick Williams "--force-rm", 610ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 611ee3c9eebSPatrick Williams "-t", 612ee3c9eebSPatrick Williams tag, 613ee3c9eebSPatrick Williams "-", 614ee3c9eebSPatrick Williams _in=dockerfile, 615ee3c9eebSPatrick Williams _out=( 616ee3c9eebSPatrick Williams lambda line: print( 617ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 618ee3c9eebSPatrick Williams ) 619ee3c9eebSPatrick Williams ), 620ee3c9eebSPatrick Williams ) 621ee3c9eebSPatrick Williams 622ee3c9eebSPatrick Williams 623ee3c9eebSPatrick Williams# Read a bunch of environment variables. 624ee3c9eebSPatrick Williamsdocker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 625ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 626ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 6278949d3c3SPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:jammy") 628ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 629ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 630ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 631ee3c9eebSPatrick Williams 63265b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 63365b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 63465b21fb9SPatrick Williams 635ee3c9eebSPatrick Williams# Set up some common variables. 636ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 637ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 638ee3c9eebSPatrick Williamsgid = os.getgid() 639ee3c9eebSPatrick Williamsuid = os.getuid() 640ee3c9eebSPatrick Williams 6416825a018SJosh Lehan# Use well-known constants if user is root 6426825a018SJosh Lehanif username == "root": 6436825a018SJosh Lehan homedir = "/root" 6446825a018SJosh Lehan gid = 0 6456825a018SJosh Lehan uid = 0 6466825a018SJosh Lehan 647ee3c9eebSPatrick Williams# Determine the architecture for Docker. 648ee3c9eebSPatrick Williamsarch = uname("-m").strip() 649ee3c9eebSPatrick Williamsif arch == "ppc64le": 650ee3c9eebSPatrick Williams docker_base = "ppc64le/" 651ee3c9eebSPatrick Williamselif arch == "x86_64": 652ee3c9eebSPatrick Williams docker_base = "" 653051b05b7SThang Q. Nguyenelif arch == "aarch64": 654f98f1a8dSThang Q. Nguyen docker_base = "arm64v8/" 655ee3c9eebSPatrick Williamselse: 656ee3c9eebSPatrick Williams print( 657ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 658ee3c9eebSPatrick Williams file=sys.stderr, 659ee3c9eebSPatrick Williams ) 660ee3c9eebSPatrick Williams sys.exit(1) 661ee3c9eebSPatrick Williams 66202871c91SPatrick Williams# Special flags if setting up a deb mirror. 66302871c91SPatrick Williamsmirror = "" 66402871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 66502871c91SPatrick Williams mirror = f""" 66602871c91SPatrick WilliamsRUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 66702871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 66802871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 66902871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 67002871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 67102871c91SPatrick Williams""" 67202871c91SPatrick Williams 67302871c91SPatrick Williams# Special flags for proxying. 67402871c91SPatrick Williamsproxy_cmd = "" 67534ec77e8SAdrian Ambrożewiczproxy_keyserver = "" 67602871c91SPatrick Williamsproxy_args = [] 67702871c91SPatrick Williamsif http_proxy: 67802871c91SPatrick Williams proxy_cmd = f""" 67902871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 68002871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 68102871c91SPatrick Williams""" 68234ec77e8SAdrian Ambrożewicz proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 68334ec77e8SAdrian Ambrożewicz 68402871c91SPatrick Williams proxy_args.extend( 68502871c91SPatrick Williams [ 68602871c91SPatrick Williams "--build-arg", 68702871c91SPatrick Williams f"http_proxy={http_proxy}", 68802871c91SPatrick Williams "--build-arg", 689d461cd6aSLei YU f"https_proxy={http_proxy}", 69002871c91SPatrick Williams ] 69102871c91SPatrick Williams ) 69202871c91SPatrick Williams 693ee3c9eebSPatrick Williams# Create base Dockerfile. 694a18d9c57SPatrick Williamsdockerfile_base = f""" 695a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 69602871c91SPatrick Williams 69702871c91SPatrick Williams{mirror} 69802871c91SPatrick Williams 69902871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 70002871c91SPatrick Williams 7018949d3c3SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/" 70202871c91SPatrick Williams 703bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 704bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 705bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 706bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 707bb16ac14SPatrick Williams# the update. 708bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 709bb16ac14SPatrick Williams 71002871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 71102871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 71202871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 71350837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 714f79ce4c4SPatrick Williams ( apt-get install gpgv ubuntu-dbgsym-keyring || \ 71550837436SPatrick Williams ( apt-get install -yy dirmngr && \ 71650837436SPatrick Williams apt-key adv --keyserver keyserver.ubuntu.com \ 71734ec77e8SAdrian Ambrożewicz {proxy_keyserver} \ 71850837436SPatrick Williams --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 71902871c91SPatrick Williams 72002871c91SPatrick Williams# Parse the current repo list into a debug repo list 72102871c91SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 72202871c91SPatrick Williams 72302871c91SPatrick Williams# Remove non-existent debug repos 72402871c91SPatrick WilliamsRUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 72502871c91SPatrick Williams 72602871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 72702871c91SPatrick Williams 72802871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 729274e3a9eSPatrick Williams gcc-12 \ 730274e3a9eSPatrick Williams g++-12 \ 73102871c91SPatrick Williams libc6-dbg \ 73202871c91SPatrick Williams libc6-dev \ 73302871c91SPatrick Williams libtool \ 73402871c91SPatrick Williams bison \ 73502871c91SPatrick Williams libdbus-1-dev \ 73602871c91SPatrick Williams flex \ 73702871c91SPatrick Williams cmake \ 73802871c91SPatrick Williams python3 \ 73902871c91SPatrick Williams python3-dev\ 74002871c91SPatrick Williams python3-yaml \ 74102871c91SPatrick Williams python3-mako \ 74202871c91SPatrick Williams python3-pip \ 74302871c91SPatrick Williams python3-setuptools \ 74402871c91SPatrick Williams python3-git \ 74502871c91SPatrick Williams python3-socks \ 74602871c91SPatrick Williams pkg-config \ 74702871c91SPatrick Williams autoconf \ 74802871c91SPatrick Williams autoconf-archive \ 74902871c91SPatrick Williams libsystemd-dev \ 75002871c91SPatrick Williams systemd \ 75102871c91SPatrick Williams libssl-dev \ 75202871c91SPatrick Williams libevdev-dev \ 75302871c91SPatrick Williams libjpeg-dev \ 75402871c91SPatrick Williams libpng-dev \ 75502871c91SPatrick Williams ninja-build \ 75602871c91SPatrick Williams sudo \ 75702871c91SPatrick Williams curl \ 75802871c91SPatrick Williams git \ 75902871c91SPatrick Williams dbus \ 76002871c91SPatrick Williams iputils-ping \ 7610ff14b2aSJian Zhang clang-14 \ 7620ff14b2aSJian Zhang clang-format-14 \ 7630ff14b2aSJian Zhang clang-tidy-14 \ 7640ff14b2aSJian Zhang clang-tools-14 \ 76502871c91SPatrick Williams shellcheck \ 76602871c91SPatrick Williams npm \ 76702871c91SPatrick Williams iproute2 \ 76802871c91SPatrick Williams libnl-3-dev \ 76902871c91SPatrick Williams libnl-genl-3-dev \ 77002871c91SPatrick Williams libconfig++-dev \ 77102871c91SPatrick Williams libsnmp-dev \ 77202871c91SPatrick Williams valgrind \ 77302871c91SPatrick Williams valgrind-dbg \ 77402871c91SPatrick Williams libpam0g-dev \ 77502871c91SPatrick Williams xxd \ 77602871c91SPatrick Williams libi2c-dev \ 77702871c91SPatrick Williams wget \ 77802871c91SPatrick Williams libldap2-dev \ 77902871c91SPatrick Williams libprotobuf-dev \ 780dafe7529SWilliam A. Kennington III liburing-dev \ 7818949d3c3SPatrick Williams liburing2-dbgsym \ 78202871c91SPatrick Williams libperlio-gzip-perl \ 78302871c91SPatrick Williams libjson-perl \ 78402871c91SPatrick Williams protobuf-compiler \ 78502871c91SPatrick Williams libgpiod-dev \ 78602871c91SPatrick Williams device-tree-compiler \ 78702871c91SPatrick Williams libpciaccess-dev \ 78802871c91SPatrick Williams libmimetic-dev \ 78902871c91SPatrick Williams libxml2-utils \ 7900eedeedaSPatrick Williams libxml-simple-perl \ 7919adf68d6SJohn Wedig rsync \ 7929adf68d6SJohn Wedig libcryptsetup-dev 79302871c91SPatrick Williams 7943ea4cd7eSPatrick Williams# Apply autoconf-archive-v2022.02.11 file ax_cxx_compile_stdcxx for C++20. 7953ea4cd7eSPatrick WilliamsRUN curl "http://git.savannah.gnu.org/gitweb/?p=autoconf-archive.git;a=blob_plain;f=m4/ax_cxx_compile_stdcxx.m4;hb=3311b6bdeff883c6a13952594a9dcb60bce6ba80" \ 7963ea4cd7eSPatrick Williams > /usr/share/aclocal/ax_cxx_compile_stdcxx.m4 7973ea4cd7eSPatrick Williams 79887111bb7SManojkiran EdaRUN npm install -g eslint@latest eslint-plugin-json@latest 79987111bb7SManojkiran Eda 800274e3a9eSPatrick WilliamsRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 12 \ 801274e3a9eSPatrick Williams --slave /usr/bin/g++ g++ /usr/bin/g++-12 \ 802274e3a9eSPatrick Williams --slave /usr/bin/gcov gcov /usr/bin/gcov-12 \ 803274e3a9eSPatrick Williams --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 \ 804274e3a9eSPatrick Williams --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 80502871c91SPatrick Williams 8060ff14b2aSJian ZhangRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 1000 \ 8070ff14b2aSJian Zhang --slave /usr/bin/clang++ clang++ /usr/bin/clang++-14 \ 8080ff14b2aSJian Zhang --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-14 \ 8090ff14b2aSJian Zhang --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-14 \ 8100ff14b2aSJian Zhang --slave /usr/bin/run-clang-tidy run-clang-tidy.py /usr/bin/run-clang-tidy-14 \ 8110ff14b2aSJian Zhang --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-14 81202871c91SPatrick Williams 81350837436SPatrick Williams""" 81450837436SPatrick Williams 81550837436SPatrick Williamsif is_automated_ci_build: 81650837436SPatrick Williams dockerfile_base += f""" 81750837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 81850837436SPatrick Williams# to re-run `apt-get update` daily. 819ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 82050837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 82150837436SPatrick Williams 82250837436SPatrick Williams""" 82350837436SPatrick Williams 82450837436SPatrick Williamsdockerfile_base += f""" 82502871c91SPatrick WilliamsRUN pip3 install inflection 82602871c91SPatrick WilliamsRUN pip3 install pycodestyle 82702871c91SPatrick WilliamsRUN pip3 install jsonschema 828fb612a51SMichael ShenRUN pip3 install meson==0.63.0 82902871c91SPatrick WilliamsRUN pip3 install protobuf 830e6f120aaSManojkiran EdaRUN pip3 install codespell 831ca8c4a8bSEd TanousRUN pip3 install requests 832a18d9c57SPatrick Williams""" 83302871c91SPatrick Williams 834fb9948a3SEd Tanous# Note, we use sha1s here because the newest gitlint release doesn't include 835fb9948a3SEd Tanous# some features we need. Next time they release, we can rely on a direct 836fb9948a3SEd Tanous# release tag 837fb9948a3SEd Tanousdockerfile_base += f""" 838fb9948a3SEd TanousRUN pip3 install git+https://github.com/jorisroovers/gitlint.git@8ede310d62d5794efa7518b235f899f8a8ad6a68\#subdirectory=gitlint-core 839fb9948a3SEd TanousRUN pip3 install git+https://github.com/jorisroovers/gitlint.git@8ede310d62d5794efa7518b235f899f8a8ad6a68 840fb9948a3SEd Tanous""" 841fb9948a3SEd Tanous 842ee3c9eebSPatrick Williams# Build the base and stage docker images. 843ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 844ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 845ee3c9eebSPatrick WilliamsPackage.generate_all() 84602871c91SPatrick Williams 847ee3c9eebSPatrick Williams# Create the final Dockerfile. 848a18d9c57SPatrick Williamsdockerfile = f""" 84902871c91SPatrick Williams# Build the final output image 850a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 851ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 85202871c91SPatrick Williams 85302871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 85402871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 85502871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 856ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 85702871c91SPatrick Williams 85802871c91SPatrick Williams# Final configuration for the workspace 8596825a018SJosh LehanRUN grep -q {gid} /etc/group || groupadd -f -g {gid} {username} 86002871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 86102871c91SPatrick WilliamsRUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 86202871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 86302871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 86402871c91SPatrick Williams 865305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 866305a9a5dSAndrew Geissler# and data installs 8677bb00b13SAndrew GeisslerRUN chown -R {username}:{username} /usr/local/share 868305a9a5dSAndrew Geissler 86902871c91SPatrick Williams{proxy_cmd} 87002871c91SPatrick Williams 87102871c91SPatrick WilliamsRUN /bin/bash 87202871c91SPatrick Williams""" 87302871c91SPatrick Williams 874a18d9c57SPatrick Williams# Do the final docker build 875ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 876ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 877ee3c9eebSPatrick Williams 87800536fbeSPatrick Williams# Print the tag of the final image. 87900536fbeSPatrick Williamsprint(docker_final_img_name) 880