102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1050837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1150837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1250837436SPatrick Williams# (ex. Jenkins)> 1302871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1402871c91SPatrick Williams# repositories> 1502871c91SPatrick Williams# default is master, which will be used if input branch not 1602871c91SPatrick Williams# provided or not found 1702871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1802871c91SPatrick Williams# default ones in /etc/apt/sources.list> 1902871c91SPatrick Williams# default is empty, and no mirror is used. 2002871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2102871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2202871c91SPatrick Williams 2302871c91SPatrick Williamsimport os 2402871c91SPatrick Williamsimport sys 25b16f3e20SPatrick Williamsimport threading 26a18d9c57SPatrick Williamsfrom datetime import date 27a18d9c57SPatrick Williamsfrom hashlib import sha256 28e08ffba8SPatrick Williams 29e08ffba8SPatrick Williams# typing.Dict is used for type-hints. 30e08ffba8SPatrick Williamsfrom typing import Any, Callable, Dict, Iterable, Optional # noqa: F401 3102871c91SPatrick Williams 3241d86218SPatrick Williamsfrom sh import docker, git, nproc, uname # type: ignore 3341d86218SPatrick Williams 34ee3c9eebSPatrick Williamstry: 35ee3c9eebSPatrick Williams # Python before 3.8 doesn't have TypedDict, so reroute to standard 'dict'. 36ee3c9eebSPatrick Williams from typing import TypedDict 3741d86218SPatrick Williamsexcept Exception: 38ee3c9eebSPatrick Williams 39ee3c9eebSPatrick Williams class TypedDict(dict): # type: ignore 40ee3c9eebSPatrick Williams # We need to do this to eat the 'total' argument. 4141d86218SPatrick Williams def __init_subclass__(cls, **kwargs: Any) -> None: 42ee3c9eebSPatrick Williams super().__init_subclass__() 43ee3c9eebSPatrick Williams 44ee3c9eebSPatrick Williams 45ee3c9eebSPatrick Williams# Declare some variables used in package definitions. 46aae36d18SPatrick Williamsprefix = "/usr/local" 4702871c91SPatrick Williamsproc_count = nproc().strip() 4802871c91SPatrick Williams 49ee3c9eebSPatrick Williams 50ee3c9eebSPatrick Williamsclass PackageDef(TypedDict, total=False): 51ee3c9eebSPatrick Williams """Package Definition for packages dictionary.""" 52ee3c9eebSPatrick Williams 53ee3c9eebSPatrick Williams # rev [optional]: Revision of package to use. 54ee3c9eebSPatrick Williams rev: str 55ee3c9eebSPatrick Williams # url [optional]: lambda function to create URL: (package, rev) -> url. 56ee3c9eebSPatrick Williams url: Callable[[str, str], str] 57ee3c9eebSPatrick Williams # depends [optional]: List of package dependencies. 58ee3c9eebSPatrick Williams depends: Iterable[str] 59ee3c9eebSPatrick Williams # build_type [required]: Build type used for package. 60ee3c9eebSPatrick Williams # Currently supported: autoconf, cmake, custom, make, meson 61ee3c9eebSPatrick Williams build_type: str 62ee3c9eebSPatrick Williams # build_steps [optional]: Steps to run for 'custom' build_type. 63ee3c9eebSPatrick Williams build_steps: Iterable[str] 64ee3c9eebSPatrick Williams # config_flags [optional]: List of options to pass configuration tool. 65ee3c9eebSPatrick Williams config_flags: Iterable[str] 66ee3c9eebSPatrick Williams # config_env [optional]: List of environment variables to set for config. 67ee3c9eebSPatrick Williams config_env: Iterable[str] 68ee3c9eebSPatrick Williams # custom_post_dl [optional]: List of steps to run after download, but 69ee3c9eebSPatrick Williams # before config / build / install. 70ee3c9eebSPatrick Williams custom_post_dl: Iterable[str] 716bce2ca1SPatrick Williams # custom_post_install [optional]: List of steps to run after install. 726bce2ca1SPatrick Williams custom_post_install: Iterable[str] 73ee3c9eebSPatrick Williams 74ee3c9eebSPatrick Williams # __tag [private]: Generated Docker tag name for package stage. 75ee3c9eebSPatrick Williams __tag: str 76ee3c9eebSPatrick Williams # __package [private]: Package object associated with this package. 77ee3c9eebSPatrick Williams __package: Any # Type is Package, but not defined yet. 78ee3c9eebSPatrick Williams 7902871c91SPatrick Williams 807204324cSPatrick Williams# Packages to include in image. 817204324cSPatrick Williamspackages = { 82ee3c9eebSPatrick Williams "boost": PackageDef( 8378173cd4SEd Tanous rev="1.82.0", 84ee3c9eebSPatrick Williams url=( 8545bfd1ffSEd Tanous lambda pkg, rev: f"https://boostorg.jfrog.io/artifactory/main/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.gz" # noqa: E501 862abc4a48SPatrick Williams ), 87ee3c9eebSPatrick Williams build_type="custom", 88ee3c9eebSPatrick Williams build_steps=[ 89e08ffba8SPatrick Williams ( 90782f41f4SBrad Bishop "curl --remote-name" 91876ea1efSPatrick Williams " https://github.com/williamspatrick/beast/commit/98f8b1fbd059a35754c2c7b2841769cf8d021272.patch" 92876ea1efSPatrick Williams " && patch -p2 <" 93876ea1efSPatrick Williams " 98f8b1fbd059a35754c2c7b2841769cf8d021272.patch &&" 94e08ffba8SPatrick Williams " ./bootstrap.sh" 95e08ffba8SPatrick Williams f" --prefix={prefix} --with-libraries=context,coroutine" 96e08ffba8SPatrick Williams ), 97aae36d18SPatrick Williams "./b2", 98aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 99aae36d18SPatrick Williams ], 100ee3c9eebSPatrick Williams ), 101ee3c9eebSPatrick Williams "USCiLab/cereal": PackageDef( 102c1977839SPatrick Williams rev="v1.3.2", 103ee3c9eebSPatrick Williams build_type="custom", 104ee3c9eebSPatrick Williams build_steps=[f"cp -a include/cereal/ {prefix}/include/"], 105ee3c9eebSPatrick Williams ), 106c7198558SEd Tanous "danmar/cppcheck": PackageDef( 107be4bd084SPatrick Williams rev="2.9", 108c7198558SEd Tanous build_type="cmake", 109c7198558SEd Tanous ), 110ee3c9eebSPatrick Williams "CLIUtils/CLI11": PackageDef( 111ee3c9eebSPatrick Williams rev="v1.9.1", 112ee3c9eebSPatrick Williams build_type="cmake", 113ee3c9eebSPatrick Williams config_flags=[ 114aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 115aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 116aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 117aae36d18SPatrick Williams ], 118ee3c9eebSPatrick Williams ), 119ee3c9eebSPatrick Williams "fmtlib/fmt": PackageDef( 120652d8aeaSWilliam A. Kennington III rev="9.1.0", 121ee3c9eebSPatrick Williams build_type="cmake", 122ee3c9eebSPatrick Williams config_flags=[ 123aae36d18SPatrick Williams "-DFMT_DOC=OFF", 124aae36d18SPatrick Williams "-DFMT_TEST=OFF", 125aae36d18SPatrick Williams ], 126ee3c9eebSPatrick Williams ), 127ee3c9eebSPatrick Williams "Naios/function2": PackageDef( 128c1977839SPatrick Williams rev="4.2.1", 129ee3c9eebSPatrick Williams build_type="custom", 130ee3c9eebSPatrick Williams build_steps=[ 131aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 132aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 133aae36d18SPatrick Williams ], 134ee3c9eebSPatrick Williams ), 135ed9414e8SPatrick Williams # release-1.12.1 136ee3c9eebSPatrick Williams "google/googletest": PackageDef( 137ed9414e8SPatrick Williams rev="58d77fa8070e8cec2dc1ed015d66b454c8d78850", 138ee3c9eebSPatrick Williams build_type="cmake", 1394dd32c02SWilliam A. Kennington III config_env=["CXXFLAGS=-std=c++20"], 140ee3c9eebSPatrick Williams config_flags=["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 141ee3c9eebSPatrick Williams ), 142ee3c9eebSPatrick Williams "nlohmann/json": PackageDef( 143c1977839SPatrick Williams rev="v3.11.2", 1446bce2ca1SPatrick Williams build_type="cmake", 1456bce2ca1SPatrick Williams config_flags=["-DJSON_BuildTests=OFF"], 1466bce2ca1SPatrick Williams custom_post_install=[ 147e08ffba8SPatrick Williams ( 148e08ffba8SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp" 149e08ffba8SPatrick Williams f" {prefix}/include/json.hpp" 150e08ffba8SPatrick Williams ), 151aae36d18SPatrick Williams ], 152ee3c9eebSPatrick Williams ), 153058e3a34SPrzemyslaw Czarnowski "json-c/json-c": PackageDef( 154058e3a34SPrzemyslaw Czarnowski rev="json-c-0.16-20220414", 155058e3a34SPrzemyslaw Czarnowski build_type="cmake", 156058e3a34SPrzemyslaw Czarnowski ), 15702871c91SPatrick Williams # Snapshot from 2019-05-24 158ee3c9eebSPatrick Williams "linux-test-project/lcov": PackageDef( 159ee3c9eebSPatrick Williams rev="v1.15", 160ee3c9eebSPatrick Williams build_type="make", 161ee3c9eebSPatrick Williams ), 162ee3c9eebSPatrick Williams "LibVNC/libvncserver": PackageDef( 163ee3c9eebSPatrick Williams rev="LibVNCServer-0.9.13", 164ee3c9eebSPatrick Williams build_type="cmake", 165ee3c9eebSPatrick Williams ), 166ee3c9eebSPatrick Williams "leethomason/tinyxml2": PackageDef( 167c1977839SPatrick Williams rev="9.0.0", 168ee3c9eebSPatrick Williams build_type="cmake", 169ee3c9eebSPatrick Williams ), 170ee3c9eebSPatrick Williams "tristanpenman/valijson": PackageDef( 171c1977839SPatrick Williams rev="v0.7", 172ee3c9eebSPatrick Williams build_type="cmake", 173ee3c9eebSPatrick Williams config_flags=[ 1740eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 1750eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 176aae36d18SPatrick Williams ], 177ee3c9eebSPatrick Williams ), 178ee3c9eebSPatrick Williams "open-power/pdbg": PackageDef(build_type="autoconf"), 179ee3c9eebSPatrick Williams "openbmc/gpioplus": PackageDef( 180ee3c9eebSPatrick Williams depends=["openbmc/stdplus"], 181ee3c9eebSPatrick Williams build_type="meson", 182ee3c9eebSPatrick Williams config_flags=[ 183aae36d18SPatrick Williams "-Dexamples=false", 184aae36d18SPatrick Williams "-Dtests=disabled", 185aae36d18SPatrick Williams ], 186ee3c9eebSPatrick Williams ), 187ee3c9eebSPatrick Williams "openbmc/phosphor-dbus-interfaces": PackageDef( 188ee3c9eebSPatrick Williams depends=["openbmc/sdbusplus"], 189ee3c9eebSPatrick Williams build_type="meson", 1904fe87776SWilliam A. Kennington III config_flags=["-Dgenerate_md=false"], 191ee3c9eebSPatrick Williams ), 192ee3c9eebSPatrick Williams "openbmc/phosphor-logging": PackageDef( 193ee3c9eebSPatrick Williams depends=[ 19483394610SPatrick Williams "USCiLab/cereal", 19583394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 19683394610SPatrick Williams "openbmc/sdbusplus", 19783394610SPatrick Williams "openbmc/sdeventplus", 198aae36d18SPatrick Williams ], 199f79ce4c4SPatrick Williams build_type="meson", 200ee3c9eebSPatrick Williams config_flags=[ 2016c98f280SWilliam A. Kennington III "-Dlibonly=true", 2026c98f280SWilliam A. Kennington III "-Dtests=disabled", 2035eabdae9SPatrick Williams f"-Dyamldir={prefix}/share/phosphor-dbus-yaml/yaml", 204aae36d18SPatrick Williams ], 205ee3c9eebSPatrick Williams ), 206ee3c9eebSPatrick Williams "openbmc/phosphor-objmgr": PackageDef( 207ee3c9eebSPatrick Williams depends=[ 20811e5762cSBrad Bishop "CLIUtils/CLI11", 20970af95caSPatrick Williams "boost", 21083394610SPatrick Williams "leethomason/tinyxml2", 21170af95caSPatrick Williams "openbmc/phosphor-dbus-interfaces", 21283394610SPatrick Williams "openbmc/phosphor-logging", 21383394610SPatrick Williams "openbmc/sdbusplus", 214aae36d18SPatrick Williams ], 2151197e359SBrad Bishop build_type="meson", 2161197e359SBrad Bishop config_flags=[ 2171197e359SBrad Bishop "-Dtests=disabled", 2181197e359SBrad Bishop ], 219ee3c9eebSPatrick Williams ), 2201c19e453SManojkiran Eda "openbmc/libpldm": PackageDef( 221ee3c9eebSPatrick Williams build_type="meson", 222ee3c9eebSPatrick Williams config_flags=[ 223*29d69bb0SAndrew Jeffery "-Dabi=deprecated,stable", 224aae36d18SPatrick Williams "-Doem-ibm=enabled", 225aae36d18SPatrick Williams "-Dtests=disabled", 226aae36d18SPatrick Williams ], 227ee3c9eebSPatrick Williams ), 228ee3c9eebSPatrick Williams "openbmc/sdbusplus": PackageDef( 229ee3c9eebSPatrick Williams build_type="meson", 230ee3c9eebSPatrick Williams custom_post_dl=[ 231aae36d18SPatrick Williams "cd tools", 232aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 233aae36d18SPatrick Williams "cd ..", 234aae36d18SPatrick Williams ], 235ee3c9eebSPatrick Williams config_flags=[ 236aae36d18SPatrick Williams "-Dexamples=disabled", 237aae36d18SPatrick Williams "-Dtests=disabled", 238aae36d18SPatrick Williams ], 239b16f3e20SPatrick Williams ), 240ee3c9eebSPatrick Williams "openbmc/sdeventplus": PackageDef( 24170af95caSPatrick Williams depends=[ 24270af95caSPatrick Williams "Naios/function2", 24370af95caSPatrick Williams "openbmc/stdplus", 24470af95caSPatrick Williams ], 245ee3c9eebSPatrick Williams build_type="meson", 246ee3c9eebSPatrick Williams config_flags=[ 247ee3c9eebSPatrick Williams "-Dexamples=false", 248ee3c9eebSPatrick Williams "-Dtests=disabled", 249ee3c9eebSPatrick Williams ], 250ee3c9eebSPatrick Williams ), 251ee3c9eebSPatrick Williams "openbmc/stdplus": PackageDef( 25270af95caSPatrick Williams depends=[ 25370af95caSPatrick Williams "fmtlib/fmt", 254ca1bf0c0SWilliam A. Kennington III "google/googletest", 255ca1bf0c0SWilliam A. Kennington III "Naios/function2", 25670af95caSPatrick Williams ], 257ee3c9eebSPatrick Williams build_type="meson", 258ee3c9eebSPatrick Williams config_flags=[ 259ee3c9eebSPatrick Williams "-Dexamples=false", 260ee3c9eebSPatrick Williams "-Dtests=disabled", 261ca1bf0c0SWilliam A. Kennington III "-Dgtest=enabled", 262ee3c9eebSPatrick Williams ], 263ee3c9eebSPatrick Williams ), 264ee3c9eebSPatrick Williams} # type: Dict[str, PackageDef] 26502871c91SPatrick Williams 26602871c91SPatrick Williams# Define common flags used for builds 26702871c91SPatrick Williamsconfigure_flags = " ".join( 26802871c91SPatrick Williams [ 26902871c91SPatrick Williams f"--prefix={prefix}", 27002871c91SPatrick Williams ] 27102871c91SPatrick Williams) 27202871c91SPatrick Williamscmake_flags = " ".join( 27302871c91SPatrick Williams [ 27402871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 2750f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 27602871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 2770f2086b3SPatrick Williams "-GNinja", 2780f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 27902871c91SPatrick Williams ] 28002871c91SPatrick Williams) 28102871c91SPatrick Williamsmeson_flags = " ".join( 28202871c91SPatrick Williams [ 28302871c91SPatrick Williams "--wrap-mode=nodownload", 28402871c91SPatrick Williams f"-Dprefix={prefix}", 28502871c91SPatrick Williams ] 28602871c91SPatrick Williams) 28702871c91SPatrick Williams 288ee3c9eebSPatrick Williams 289ee3c9eebSPatrick Williamsclass Package(threading.Thread): 290ee3c9eebSPatrick Williams """Class used to build the Docker stages for each package. 291ee3c9eebSPatrick Williams 292ee3c9eebSPatrick Williams Generally, this class should not be instantiated directly but through 293ee3c9eebSPatrick Williams Package.generate_all(). 294ee3c9eebSPatrick Williams """ 295ee3c9eebSPatrick Williams 296ee3c9eebSPatrick Williams # Copy the packages dictionary. 297ee3c9eebSPatrick Williams packages = packages.copy() 298ee3c9eebSPatrick Williams 299ee3c9eebSPatrick Williams # Lock used for thread-safety. 300ee3c9eebSPatrick Williams lock = threading.Lock() 301ee3c9eebSPatrick Williams 302ee3c9eebSPatrick Williams def __init__(self, pkg: str): 303ee3c9eebSPatrick Williams """pkg - The name of this package (ex. foo/bar )""" 304ee3c9eebSPatrick Williams super(Package, self).__init__() 305ee3c9eebSPatrick Williams 306ee3c9eebSPatrick Williams self.package = pkg 307ee3c9eebSPatrick Williams self.exception = None # type: Optional[Exception] 308ee3c9eebSPatrick Williams 309ee3c9eebSPatrick Williams # Reference to this package's 310ee3c9eebSPatrick Williams self.pkg_def = Package.packages[pkg] 311ee3c9eebSPatrick Williams self.pkg_def["__package"] = self 312ee3c9eebSPatrick Williams 313ee3c9eebSPatrick Williams def run(self) -> None: 314ee3c9eebSPatrick Williams """Thread 'run' function. Builds the Docker stage.""" 315ee3c9eebSPatrick Williams 316ee3c9eebSPatrick Williams # In case this package has no rev, fetch it from Github. 317ee3c9eebSPatrick Williams self._update_rev() 318ee3c9eebSPatrick Williams 319ee3c9eebSPatrick Williams # Find all the Package objects that this package depends on. 320ee3c9eebSPatrick Williams # This section is locked because we are looking into another 321ee3c9eebSPatrick Williams # package's PackageDef dict, which could be being modified. 322ee3c9eebSPatrick Williams Package.lock.acquire() 323ee3c9eebSPatrick Williams deps: Iterable[Package] = [ 324ee3c9eebSPatrick Williams Package.packages[deppkg]["__package"] 325ee3c9eebSPatrick Williams for deppkg in self.pkg_def.get("depends", []) 326ee3c9eebSPatrick Williams ] 327ee3c9eebSPatrick Williams Package.lock.release() 328ee3c9eebSPatrick Williams 329ee3c9eebSPatrick Williams # Wait until all the depends finish building. We need them complete 330ee3c9eebSPatrick Williams # for the "COPY" commands. 331ee3c9eebSPatrick Williams for deppkg in deps: 332ee3c9eebSPatrick Williams deppkg.join() 333ee3c9eebSPatrick Williams 334ee3c9eebSPatrick Williams # Generate this package's Dockerfile. 335ee3c9eebSPatrick Williams dockerfile = f""" 336ee3c9eebSPatrick WilliamsFROM {docker_base_img_name} 337ee3c9eebSPatrick Williams{self._df_copycmds()} 338ee3c9eebSPatrick Williams{self._df_build()} 339ee3c9eebSPatrick Williams""" 340ee3c9eebSPatrick Williams 341ee3c9eebSPatrick Williams # Generate the resulting tag name and save it to the PackageDef. 342ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef, 343ee3c9eebSPatrick Williams # which can be accessed by other threads. 344ee3c9eebSPatrick Williams Package.lock.acquire() 345ee3c9eebSPatrick Williams tag = Docker.tagname(self._stagename(), dockerfile) 346ee3c9eebSPatrick Williams self.pkg_def["__tag"] = tag 347ee3c9eebSPatrick Williams Package.lock.release() 348ee3c9eebSPatrick Williams 349ee3c9eebSPatrick Williams # Do the build / save any exceptions. 350ee3c9eebSPatrick Williams try: 351ee3c9eebSPatrick Williams Docker.build(self.package, tag, dockerfile) 352ee3c9eebSPatrick Williams except Exception as e: 353ee3c9eebSPatrick Williams self.exception = e 354ee3c9eebSPatrick Williams 355ee3c9eebSPatrick Williams @classmethod 356ee3c9eebSPatrick Williams def generate_all(cls) -> None: 357ee3c9eebSPatrick Williams """Ensure a Docker stage is created for all defined packages. 358ee3c9eebSPatrick Williams 359ee3c9eebSPatrick Williams These are done in parallel but with appropriate blocking per 360ee3c9eebSPatrick Williams package 'depends' specifications. 361ee3c9eebSPatrick Williams """ 362ee3c9eebSPatrick Williams 363ee3c9eebSPatrick Williams # Create a Package for each defined package. 364ee3c9eebSPatrick Williams pkg_threads = [Package(p) for p in cls.packages.keys()] 365ee3c9eebSPatrick Williams 366ee3c9eebSPatrick Williams # Start building them all. 3676dbd7807SPatrick Williams # This section is locked because threads depend on each other, 3686dbd7807SPatrick Williams # based on the packages, and they cannot 'join' on a thread 3696dbd7807SPatrick Williams # which is not yet started. Adding a lock here allows all the 3706dbd7807SPatrick Williams # threads to start before they 'join' their dependencies. 3716dbd7807SPatrick Williams Package.lock.acquire() 372ee3c9eebSPatrick Williams for t in pkg_threads: 373ee3c9eebSPatrick Williams t.start() 3746dbd7807SPatrick Williams Package.lock.release() 375ee3c9eebSPatrick Williams 376ee3c9eebSPatrick Williams # Wait for completion. 377ee3c9eebSPatrick Williams for t in pkg_threads: 378ee3c9eebSPatrick Williams t.join() 379ee3c9eebSPatrick Williams # Check if the thread saved off its own exception. 380ee3c9eebSPatrick Williams if t.exception: 381ee3c9eebSPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 382ee3c9eebSPatrick Williams raise t.exception 383ee3c9eebSPatrick Williams 384ee3c9eebSPatrick Williams @staticmethod 385ee3c9eebSPatrick Williams def df_all_copycmds() -> str: 386ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to copy all packages 387ee3c9eebSPatrick Williams into the final image. 388ee3c9eebSPatrick Williams """ 389ee3c9eebSPatrick Williams return Package.df_copycmds_set(Package.packages.keys()) 390ee3c9eebSPatrick Williams 391ee3c9eebSPatrick Williams @classmethod 392ee3c9eebSPatrick Williams def depcache(cls) -> str: 393ee3c9eebSPatrick Williams """Create the contents of the '/tmp/depcache'. 394ee3c9eebSPatrick Williams This file is a comma-separated list of "<pkg>:<rev>". 395ee3c9eebSPatrick Williams """ 396ee3c9eebSPatrick Williams 397ee3c9eebSPatrick Williams # This needs to be sorted for consistency. 398ee3c9eebSPatrick Williams depcache = "" 399ee3c9eebSPatrick Williams for pkg in sorted(cls.packages.keys()): 400ee3c9eebSPatrick Williams depcache += "%s:%s," % (pkg, cls.packages[pkg]["rev"]) 401ee3c9eebSPatrick Williams return depcache 402ee3c9eebSPatrick Williams 403ee3c9eebSPatrick Williams def _update_rev(self) -> None: 404ee3c9eebSPatrick Williams """Look up the HEAD for missing a static rev.""" 405ee3c9eebSPatrick Williams 406ee3c9eebSPatrick Williams if "rev" in self.pkg_def: 407ee3c9eebSPatrick Williams return 408ee3c9eebSPatrick Williams 40965b21fb9SPatrick Williams # Check if Jenkins/Gerrit gave us a revision and use it. 41065b21fb9SPatrick Williams if gerrit_project == self.package and gerrit_rev: 41165b21fb9SPatrick Williams print( 41265b21fb9SPatrick Williams f"Found Gerrit revision for {self.package}: {gerrit_rev}", 41365b21fb9SPatrick Williams file=sys.stderr, 41465b21fb9SPatrick Williams ) 41565b21fb9SPatrick Williams self.pkg_def["rev"] = gerrit_rev 41665b21fb9SPatrick Williams return 41765b21fb9SPatrick Williams 418ee3c9eebSPatrick Williams # Ask Github for all the branches. 41905fb2a0aSPatrick Williams lookup = git( 42005fb2a0aSPatrick Williams "ls-remote", "--heads", f"https://github.com/{self.package}" 42105fb2a0aSPatrick Williams ) 422ee3c9eebSPatrick Williams 423ee3c9eebSPatrick Williams # Find the branch matching {branch} (or fallback to master). 424ee3c9eebSPatrick Williams # This section is locked because we are modifying the PackageDef. 425ee3c9eebSPatrick Williams Package.lock.acquire() 426ee3c9eebSPatrick Williams for line in lookup.split("\n"): 427ee3c9eebSPatrick Williams if f"refs/heads/{branch}" in line: 428ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 429c7d73646SPatrick Williams elif ( 430c7d73646SPatrick Williams "refs/heads/master" in line or "refs/heads/main" in line 431c7d73646SPatrick Williams ) and "rev" not in self.pkg_def: 432ee3c9eebSPatrick Williams self.pkg_def["rev"] = line.split()[0] 433ee3c9eebSPatrick Williams Package.lock.release() 434ee3c9eebSPatrick Williams 435ee3c9eebSPatrick Williams def _stagename(self) -> str: 436ee3c9eebSPatrick Williams """Create a name for the Docker stage associated with this pkg.""" 437ee3c9eebSPatrick Williams return self.package.replace("/", "-").lower() 438ee3c9eebSPatrick Williams 439ee3c9eebSPatrick Williams def _url(self) -> str: 440ee3c9eebSPatrick Williams """Get the URL for this package.""" 441ee3c9eebSPatrick Williams rev = self.pkg_def["rev"] 442ee3c9eebSPatrick Williams 443ee3c9eebSPatrick Williams # If the lambda exists, call it. 444ee3c9eebSPatrick Williams if "url" in self.pkg_def: 445ee3c9eebSPatrick Williams return self.pkg_def["url"](self.package, rev) 446ee3c9eebSPatrick Williams 447ee3c9eebSPatrick Williams # Default to the github archive URL. 448ee3c9eebSPatrick Williams return f"https://github.com/{self.package}/archive/{rev}.tar.gz" 449ee3c9eebSPatrick Williams 450ee3c9eebSPatrick Williams def _cmd_download(self) -> str: 451ee3c9eebSPatrick Williams """Formulate the command necessary to download and unpack to source.""" 452ee3c9eebSPatrick Williams 453ee3c9eebSPatrick Williams url = self._url() 454ee3c9eebSPatrick Williams if ".tar." not in url: 455ee3c9eebSPatrick Williams raise NotImplementedError( 456ee3c9eebSPatrick Williams f"Unhandled download type for {self.package}: {url}" 457ee3c9eebSPatrick Williams ) 458ee3c9eebSPatrick Williams 459ee3c9eebSPatrick Williams cmd = f"curl -L {url} | tar -x" 460ee3c9eebSPatrick Williams 461ee3c9eebSPatrick Williams if url.endswith(".bz2"): 462ee3c9eebSPatrick Williams cmd += "j" 463ee3c9eebSPatrick Williams elif url.endswith(".gz"): 464ee3c9eebSPatrick Williams cmd += "z" 465ee3c9eebSPatrick Williams else: 466ee3c9eebSPatrick Williams raise NotImplementedError( 467ee3c9eebSPatrick Williams f"Unknown tar flags needed for {self.package}: {url}" 468ee3c9eebSPatrick Williams ) 469ee3c9eebSPatrick Williams 470ee3c9eebSPatrick Williams return cmd 471ee3c9eebSPatrick Williams 472ee3c9eebSPatrick Williams def _cmd_cd_srcdir(self) -> str: 473ee3c9eebSPatrick Williams """Formulate the command necessary to 'cd' into the source dir.""" 474ee3c9eebSPatrick Williams return f"cd {self.package.split('/')[-1]}*" 475ee3c9eebSPatrick Williams 476ee3c9eebSPatrick Williams def _df_copycmds(self) -> str: 477ee3c9eebSPatrick Williams """Formulate the dockerfile snippet necessary to COPY all depends.""" 478ee3c9eebSPatrick Williams 479ee3c9eebSPatrick Williams if "depends" not in self.pkg_def: 480ee3c9eebSPatrick Williams return "" 481ee3c9eebSPatrick Williams return Package.df_copycmds_set(self.pkg_def["depends"]) 482ee3c9eebSPatrick Williams 483ee3c9eebSPatrick Williams @staticmethod 484ee3c9eebSPatrick Williams def df_copycmds_set(pkgs: Iterable[str]) -> str: 485ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to COPY a set of 486ee3c9eebSPatrick Williams packages into a Docker stage. 487ee3c9eebSPatrick Williams """ 488ee3c9eebSPatrick Williams 489ee3c9eebSPatrick Williams copy_cmds = "" 490ee3c9eebSPatrick Williams 491ee3c9eebSPatrick Williams # Sort the packages for consistency. 492ee3c9eebSPatrick Williams for p in sorted(pkgs): 493ee3c9eebSPatrick Williams tag = Package.packages[p]["__tag"] 494ee3c9eebSPatrick Williams copy_cmds += f"COPY --from={tag} {prefix} {prefix}\n" 495ee3c9eebSPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 496ee3c9eebSPatrick Williams # https://github.com/moby/moby/issues/37965 497ee3c9eebSPatrick Williams copy_cmds += "RUN true\n" 498ee3c9eebSPatrick Williams 499ee3c9eebSPatrick Williams return copy_cmds 500ee3c9eebSPatrick Williams 501ee3c9eebSPatrick Williams def _df_build(self) -> str: 502ee3c9eebSPatrick Williams """Formulate the Dockerfile snippet necessary to download, build, and 503ee3c9eebSPatrick Williams install a package into a Docker stage. 504ee3c9eebSPatrick Williams """ 505ee3c9eebSPatrick Williams 506ee3c9eebSPatrick Williams # Download and extract source. 507ee3c9eebSPatrick Williams result = f"RUN {self._cmd_download()} && {self._cmd_cd_srcdir()} && " 508ee3c9eebSPatrick Williams 509ee3c9eebSPatrick Williams # Handle 'custom_post_dl' commands. 510ee3c9eebSPatrick Williams custom_post_dl = self.pkg_def.get("custom_post_dl") 511ee3c9eebSPatrick Williams if custom_post_dl: 512ee3c9eebSPatrick Williams result += " && ".join(custom_post_dl) + " && " 513ee3c9eebSPatrick Williams 514ee3c9eebSPatrick Williams # Build and install package based on 'build_type'. 515ee3c9eebSPatrick Williams build_type = self.pkg_def["build_type"] 516ee3c9eebSPatrick Williams if build_type == "autoconf": 517ee3c9eebSPatrick Williams result += self._cmd_build_autoconf() 518ee3c9eebSPatrick Williams elif build_type == "cmake": 519ee3c9eebSPatrick Williams result += self._cmd_build_cmake() 520ee3c9eebSPatrick Williams elif build_type == "custom": 521ee3c9eebSPatrick Williams result += self._cmd_build_custom() 522ee3c9eebSPatrick Williams elif build_type == "make": 523ee3c9eebSPatrick Williams result += self._cmd_build_make() 524ee3c9eebSPatrick Williams elif build_type == "meson": 525ee3c9eebSPatrick Williams result += self._cmd_build_meson() 526ee3c9eebSPatrick Williams else: 527ee3c9eebSPatrick Williams raise NotImplementedError( 528ee3c9eebSPatrick Williams f"Unhandled build type for {self.package}: {build_type}" 529ee3c9eebSPatrick Williams ) 530ee3c9eebSPatrick Williams 5316bce2ca1SPatrick Williams # Handle 'custom_post_install' commands. 5326bce2ca1SPatrick Williams custom_post_install = self.pkg_def.get("custom_post_install") 5336bce2ca1SPatrick Williams if custom_post_install: 5346bce2ca1SPatrick Williams result += " && " + " && ".join(custom_post_install) 5356bce2ca1SPatrick Williams 536ee3c9eebSPatrick Williams return result 537ee3c9eebSPatrick Williams 538ee3c9eebSPatrick Williams def _cmd_build_autoconf(self) -> str: 539ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 540ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 541ee3c9eebSPatrick Williams result = "./bootstrap.sh && " 542ee3c9eebSPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 543ee3c9eebSPatrick Williams result += f"make -j{proc_count} && make install" 544ee3c9eebSPatrick Williams return result 545ee3c9eebSPatrick Williams 546ee3c9eebSPatrick Williams def _cmd_build_cmake(self) -> str: 547ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 548ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 549ee3c9eebSPatrick Williams result = "mkdir builddir && cd builddir && " 550ee3c9eebSPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 551ee3c9eebSPatrick Williams result += "cmake --build . --target all && " 552ee3c9eebSPatrick Williams result += "cmake --build . --target install && " 553ee3c9eebSPatrick Williams result += "cd .." 554ee3c9eebSPatrick Williams return result 555ee3c9eebSPatrick Williams 556ee3c9eebSPatrick Williams def _cmd_build_custom(self) -> str: 557ee3c9eebSPatrick Williams return " && ".join(self.pkg_def.get("build_steps", [])) 558ee3c9eebSPatrick Williams 559ee3c9eebSPatrick Williams def _cmd_build_make(self) -> str: 560ee3c9eebSPatrick Williams return f"make -j{proc_count} && make install" 561ee3c9eebSPatrick Williams 562ee3c9eebSPatrick Williams def _cmd_build_meson(self) -> str: 563ee3c9eebSPatrick Williams options = " ".join(self.pkg_def.get("config_flags", [])) 564ee3c9eebSPatrick Williams env = " ".join(self.pkg_def.get("config_env", [])) 565ee3c9eebSPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 566ee3c9eebSPatrick Williams result += "ninja -C builddir && ninja -C builddir install" 567ee3c9eebSPatrick Williams return result 568ee3c9eebSPatrick Williams 569ee3c9eebSPatrick Williams 570ee3c9eebSPatrick Williamsclass Docker: 571ee3c9eebSPatrick Williams """Class to assist with Docker interactions. All methods are static.""" 572ee3c9eebSPatrick Williams 573ee3c9eebSPatrick Williams @staticmethod 574ee3c9eebSPatrick Williams def timestamp() -> str: 575ee3c9eebSPatrick Williams """Generate a timestamp for today using the ISO week.""" 576ee3c9eebSPatrick Williams today = date.today().isocalendar() 577ee3c9eebSPatrick Williams return f"{today[0]}-W{today[1]:02}" 578ee3c9eebSPatrick Williams 579ee3c9eebSPatrick Williams @staticmethod 58041d86218SPatrick Williams def tagname(pkgname: Optional[str], dockerfile: str) -> str: 581ee3c9eebSPatrick Williams """Generate a tag name for a package using a hash of the Dockerfile.""" 582ee3c9eebSPatrick Williams result = docker_image_name 583ee3c9eebSPatrick Williams if pkgname: 584ee3c9eebSPatrick Williams result += "-" + pkgname 585ee3c9eebSPatrick Williams 586ee3c9eebSPatrick Williams result += ":" + Docker.timestamp() 587ee3c9eebSPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 588ee3c9eebSPatrick Williams 589ee3c9eebSPatrick Williams return result 590ee3c9eebSPatrick Williams 591ee3c9eebSPatrick Williams @staticmethod 592ee3c9eebSPatrick Williams def build(pkg: str, tag: str, dockerfile: str) -> None: 59322e6110bSAndrew Geissler """Build a docker image using the Dockerfile and tagging it with 'tag'.""" 594ee3c9eebSPatrick Williams 595ee3c9eebSPatrick Williams # If we're not forcing builds, check if it already exists and skip. 596ee3c9eebSPatrick Williams if not force_build: 597ee3c9eebSPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 59805fb2a0aSPatrick Williams print( 59905fb2a0aSPatrick Williams f"Image {tag} already exists. Skipping.", file=sys.stderr 60005fb2a0aSPatrick Williams ) 601ee3c9eebSPatrick Williams return 602ee3c9eebSPatrick Williams 603ee3c9eebSPatrick Williams # Build it. 604ee3c9eebSPatrick Williams # Capture the output of the 'docker build' command and send it to 605ee3c9eebSPatrick Williams # stderr (prefixed with the package name). This allows us to see 606ee3c9eebSPatrick Williams # progress but not polute stdout. Later on we output the final 607ee3c9eebSPatrick Williams # docker tag to stdout and we want to keep that pristine. 608ee3c9eebSPatrick Williams # 609ee3c9eebSPatrick Williams # Other unusual flags: 610ee3c9eebSPatrick Williams # --no-cache: Bypass the Docker cache if 'force_build'. 611ee3c9eebSPatrick Williams # --force-rm: Clean up Docker processes if they fail. 612ee3c9eebSPatrick Williams docker.build( 613ee3c9eebSPatrick Williams proxy_args, 614ee3c9eebSPatrick Williams "--network=host", 615ee3c9eebSPatrick Williams "--force-rm", 616ee3c9eebSPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 617ee3c9eebSPatrick Williams "-t", 618ee3c9eebSPatrick Williams tag, 619ee3c9eebSPatrick Williams "-", 620ee3c9eebSPatrick Williams _in=dockerfile, 621ee3c9eebSPatrick Williams _out=( 622ee3c9eebSPatrick Williams lambda line: print( 623ee3c9eebSPatrick Williams pkg + ":", line, end="", file=sys.stderr, flush=True 624ee3c9eebSPatrick Williams ) 625ee3c9eebSPatrick Williams ), 626ee3c9eebSPatrick Williams ) 627ee3c9eebSPatrick Williams 628ee3c9eebSPatrick Williams 629ee3c9eebSPatrick Williams# Read a bunch of environment variables. 63005fb2a0aSPatrick Williamsdocker_image_name = os.environ.get( 63105fb2a0aSPatrick Williams "DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test" 63205fb2a0aSPatrick Williams) 633ee3c9eebSPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 634ee3c9eebSPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 6355e4d8402SPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:lunar") 636ee3c9eebSPatrick Williamsbranch = os.environ.get("BRANCH", "master") 637ee3c9eebSPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 638ee3c9eebSPatrick Williamshttp_proxy = os.environ.get("http_proxy") 639ee3c9eebSPatrick Williams 64065b21fb9SPatrick Williamsgerrit_project = os.environ.get("GERRIT_PROJECT") 64165b21fb9SPatrick Williamsgerrit_rev = os.environ.get("GERRIT_PATCHSET_REVISION") 64265b21fb9SPatrick Williams 643d0dabc3eSAndrew Geissler# Ensure appropriate docker build output to see progress and identify 644d0dabc3eSAndrew Geissler# any issues 645d0dabc3eSAndrew Geissleros.environ["BUILDKIT_PROGRESS"] = "plain" 646d0dabc3eSAndrew Geissler 647ee3c9eebSPatrick Williams# Set up some common variables. 648ee3c9eebSPatrick Williamsusername = os.environ.get("USER", "root") 649ee3c9eebSPatrick Williamshomedir = os.environ.get("HOME", "/root") 650ee3c9eebSPatrick Williamsgid = os.getgid() 651ee3c9eebSPatrick Williamsuid = os.getuid() 652ee3c9eebSPatrick Williams 6536825a018SJosh Lehan# Use well-known constants if user is root 6546825a018SJosh Lehanif username == "root": 6556825a018SJosh Lehan homedir = "/root" 6566825a018SJosh Lehan gid = 0 6576825a018SJosh Lehan uid = 0 6586825a018SJosh Lehan 659ee3c9eebSPatrick Williams# Determine the architecture for Docker. 660ee3c9eebSPatrick Williamsarch = uname("-m").strip() 661ee3c9eebSPatrick Williamsif arch == "ppc64le": 662ee3c9eebSPatrick Williams docker_base = "ppc64le/" 663ee3c9eebSPatrick Williamselif arch == "x86_64": 664ee3c9eebSPatrick Williams docker_base = "" 665051b05b7SThang Q. Nguyenelif arch == "aarch64": 666f98f1a8dSThang Q. Nguyen docker_base = "arm64v8/" 667ee3c9eebSPatrick Williamselse: 668ee3c9eebSPatrick Williams print( 669ee3c9eebSPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 670ee3c9eebSPatrick Williams file=sys.stderr, 671ee3c9eebSPatrick Williams ) 672ee3c9eebSPatrick Williams sys.exit(1) 673ee3c9eebSPatrick Williams 67402871c91SPatrick Williams# Special flags if setting up a deb mirror. 67502871c91SPatrick Williamsmirror = "" 67602871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 67702871c91SPatrick Williams mirror = f""" 678e08ffba8SPatrick WilliamsRUN echo "deb {ubuntu_mirror} \ 679e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME) \ 680e08ffba8SPatrick Williams main restricted universe multiverse" > /etc/apt/sources.list && \\ 681e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 682e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-updates \ 683e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 684e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 685e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-security \ 686e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 687e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 688e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-proposed \ 689e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list && \\ 690e08ffba8SPatrick Williams echo "deb {ubuntu_mirror} \ 691e08ffba8SPatrick Williams $(. /etc/os-release && echo $VERSION_CODENAME)-backports \ 692e08ffba8SPatrick Williams main restricted universe multiverse" >> /etc/apt/sources.list 69302871c91SPatrick Williams""" 69402871c91SPatrick Williams 69502871c91SPatrick Williams# Special flags for proxying. 69602871c91SPatrick Williamsproxy_cmd = "" 69734ec77e8SAdrian Ambrożewiczproxy_keyserver = "" 69802871c91SPatrick Williamsproxy_args = [] 69902871c91SPatrick Williamsif http_proxy: 70002871c91SPatrick Williams proxy_cmd = f""" 70102871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 70202871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 70302871c91SPatrick Williams""" 70434ec77e8SAdrian Ambrożewicz proxy_keyserver = f"--keyserver-options http-proxy={http_proxy}" 70534ec77e8SAdrian Ambrożewicz 70602871c91SPatrick Williams proxy_args.extend( 70702871c91SPatrick Williams [ 70802871c91SPatrick Williams "--build-arg", 70902871c91SPatrick Williams f"http_proxy={http_proxy}", 71002871c91SPatrick Williams "--build-arg", 711d461cd6aSLei YU f"https_proxy={http_proxy}", 71202871c91SPatrick Williams ] 71302871c91SPatrick Williams ) 71402871c91SPatrick Williams 715ee3c9eebSPatrick Williams# Create base Dockerfile. 716a18d9c57SPatrick Williamsdockerfile_base = f""" 717a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 71802871c91SPatrick Williams 71902871c91SPatrick Williams{mirror} 72002871c91SPatrick Williams 72102871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 72202871c91SPatrick Williams 7238949d3c3SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.10/site-packages/" 72402871c91SPatrick Williams 725bb16ac14SPatrick Williams# Sometimes the ubuntu key expires and we need a way to force an execution 726bb16ac14SPatrick Williams# of the apt-get commands for the dbgsym-keyring. When this happens we see 727bb16ac14SPatrick Williams# an error like: "Release: The following signatures were invalid:" 728bb16ac14SPatrick Williams# Insert a bogus echo that we can change here when we get this error to force 729bb16ac14SPatrick Williams# the update. 730bb16ac14SPatrick WilliamsRUN echo "ubuntu keyserver rev as of 2021-04-21" 731bb16ac14SPatrick Williams 73202871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 73302871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 73402871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 735575b5e4cSJagpal Singh Gill# Known issue with gpg to get keys via proxy - 736575b5e4cSJagpal Singh Gill# https://bugs.launchpad.net/ubuntu/+source/gnupg2/+bug/1788190, hence using 737575b5e4cSJagpal Singh Gill# curl to get keys. 73850837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 739f79ce4c4SPatrick Williams ( apt-get install gpgv ubuntu-dbgsym-keyring || \ 740575b5e4cSJagpal Singh Gill ( apt-get install -yy dirmngr curl && \ 741575b5e4cSJagpal Singh Gill curl -sSL \ 742575b5e4cSJagpal Singh Gill 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xF2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622' \ 743575b5e4cSJagpal Singh Gill | apt-key add - )) 74402871c91SPatrick Williams 74502871c91SPatrick Williams# Parse the current repo list into a debug repo list 746e08ffba8SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' \ 747e08ffba8SPatrick Williams /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 74802871c91SPatrick Williams 74902871c91SPatrick Williams# Remove non-existent debug repos 75041d86218SPatrick WilliamsRUN sed -i '/-\\(backports\\|security\\) /d' /etc/apt/sources.list.d/debug.list 75102871c91SPatrick Williams 75202871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 75302871c91SPatrick Williams 75402871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 75558f1915eSAndrew Jeffery abi-compliance-checker \ 7568b112068SAndrew Jeffery abi-dumper \ 75702871c91SPatrick Williams autoconf \ 75802871c91SPatrick Williams autoconf-archive \ 759af49ed51SAndrew Geissler bison \ 76064b6f9d2SPatrick Williams clang-16 \ 76164b6f9d2SPatrick Williams clang-format-16 \ 76264b6f9d2SPatrick Williams clang-tidy-16 \ 76364b6f9d2SPatrick Williams clang-tools-16 \ 764af49ed51SAndrew Geissler cmake \ 765af49ed51SAndrew Geissler curl \ 766af49ed51SAndrew Geissler dbus \ 767af49ed51SAndrew Geissler device-tree-compiler \ 768af49ed51SAndrew Geissler flex \ 769961f148bSPatrick Williams g++-13 \ 770961f148bSPatrick Williams gcc-13 \ 771af49ed51SAndrew Geissler git \ 77202871c91SPatrick Williams iproute2 \ 773af49ed51SAndrew Geissler iputils-ping \ 774524a331cSManojkiran Eda libaudit-dev \ 775af49ed51SAndrew Geissler libc6-dbg \ 776af49ed51SAndrew Geissler libc6-dev \ 777af49ed51SAndrew Geissler libconfig++-dev \ 778af49ed51SAndrew Geissler libcryptsetup-dev \ 779af49ed51SAndrew Geissler libdbus-1-dev \ 780af49ed51SAndrew Geissler libevdev-dev \ 781af49ed51SAndrew Geissler libgpiod-dev \ 782af49ed51SAndrew Geissler libi2c-dev \ 783af49ed51SAndrew Geissler libjpeg-dev \ 784af49ed51SAndrew Geissler libjson-perl \ 785af49ed51SAndrew Geissler libldap2-dev \ 786af49ed51SAndrew Geissler libmimetic-dev \ 78702871c91SPatrick Williams libnl-3-dev \ 78802871c91SPatrick Williams libnl-genl-3-dev \ 78902871c91SPatrick Williams libpam0g-dev \ 79002871c91SPatrick Williams libpciaccess-dev \ 791af49ed51SAndrew Geissler libperlio-gzip-perl \ 792af49ed51SAndrew Geissler libpng-dev \ 793af49ed51SAndrew Geissler libprotobuf-dev \ 794af49ed51SAndrew Geissler libsnmp-dev \ 795af49ed51SAndrew Geissler libssl-dev \ 796af49ed51SAndrew Geissler libsystemd-dev \ 797af49ed51SAndrew Geissler libtool \ 798af49ed51SAndrew Geissler liburing-dev \ 79902871c91SPatrick Williams libxml2-utils \ 8000eedeedaSPatrick Williams libxml-simple-perl \ 801af49ed51SAndrew Geissler ninja-build \ 802af49ed51SAndrew Geissler npm \ 803af49ed51SAndrew Geissler pkg-config \ 804af49ed51SAndrew Geissler protobuf-compiler \ 805af49ed51SAndrew Geissler python3 \ 806af49ed51SAndrew Geissler python3-dev\ 807af49ed51SAndrew Geissler python3-git \ 808af49ed51SAndrew Geissler python3-mako \ 809af49ed51SAndrew Geissler python3-pip \ 810af49ed51SAndrew Geissler python3-setuptools \ 811af49ed51SAndrew Geissler python3-socks \ 812af49ed51SAndrew Geissler python3-yaml \ 8139adf68d6SJohn Wedig rsync \ 814af49ed51SAndrew Geissler shellcheck \ 815af49ed51SAndrew Geissler sudo \ 816af49ed51SAndrew Geissler systemd \ 817af49ed51SAndrew Geissler valgrind \ 818af49ed51SAndrew Geissler valgrind-dbg \ 819b565f825SAndrew Geissler vim \ 820af49ed51SAndrew Geissler wget \ 821af49ed51SAndrew Geissler xxd 82202871c91SPatrick Williams 823961f148bSPatrick WilliamsRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 13 \ 824961f148bSPatrick Williams --slave /usr/bin/g++ g++ /usr/bin/g++-13 \ 825961f148bSPatrick Williams --slave /usr/bin/gcov gcov /usr/bin/gcov-13 \ 826961f148bSPatrick Williams --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-13 \ 827961f148bSPatrick Williams --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-13 828961f148bSPatrick WilliamsRUN update-alternatives --remove cpp /usr/bin/cpp && \ 829961f148bSPatrick Williams update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-13 13 83002871c91SPatrick Williams 83164b6f9d2SPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-16 1000 \ 83264b6f9d2SPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-16 \ 83364b6f9d2SPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-16 \ 83464b6f9d2SPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-16 \ 835e08ffba8SPatrick Williams --slave /usr/bin/run-clang-tidy run-clang-tidy.py \ 83664b6f9d2SPatrick Williams /usr/bin/run-clang-tidy-16 \ 83764b6f9d2SPatrick Williams --slave /usr/bin/scan-build scan-build /usr/bin/scan-build-16 83802871c91SPatrick Williams 83950837436SPatrick Williams""" 84050837436SPatrick Williams 84150837436SPatrick Williamsif is_automated_ci_build: 84250837436SPatrick Williams dockerfile_base += f""" 84350837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 84450837436SPatrick Williams# to re-run `apt-get update` daily. 845ee3c9eebSPatrick WilliamsRUN echo {Docker.timestamp()} 84650837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 84750837436SPatrick Williams 84850837436SPatrick Williams""" 84950837436SPatrick Williams 85041d86218SPatrick Williamsdockerfile_base += """ 8515e4d8402SPatrick WilliamsRUN pip3 install --break-system-packages \ 852818023dfSPatrick Williams beautysh \ 853818023dfSPatrick Williams black \ 854818023dfSPatrick Williams codespell \ 855818023dfSPatrick Williams flake8 \ 856818023dfSPatrick Williams gitlint \ 857818023dfSPatrick Williams inflection \ 858818023dfSPatrick Williams isort \ 859818023dfSPatrick Williams jsonschema \ 8600044f69cSPatrick Williams meson==1.0.1 \ 861818023dfSPatrick Williams protobuf \ 862818023dfSPatrick Williams requests 863b08ddf77SPatrick Williams 864b08ddf77SPatrick WilliamsRUN npm install -g \ 865b08ddf77SPatrick Williams eslint@latest eslint-plugin-json@latest \ 8667d41f6d2SPatrick Williams markdownlint-cli@latest \ 867b08ddf77SPatrick Williams prettier@latest 868fb9948a3SEd Tanous""" 869fb9948a3SEd Tanous 870ee3c9eebSPatrick Williams# Build the base and stage docker images. 871ee3c9eebSPatrick Williamsdocker_base_img_name = Docker.tagname("base", dockerfile_base) 872ee3c9eebSPatrick WilliamsDocker.build("base", docker_base_img_name, dockerfile_base) 873ee3c9eebSPatrick WilliamsPackage.generate_all() 87402871c91SPatrick Williams 875ee3c9eebSPatrick Williams# Create the final Dockerfile. 876a18d9c57SPatrick Williamsdockerfile = f""" 87702871c91SPatrick Williams# Build the final output image 878a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 879ee3c9eebSPatrick Williams{Package.df_all_copycmds()} 88002871c91SPatrick Williams 88102871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 88202871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 88302871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 884ee3c9eebSPatrick WilliamsRUN echo '{Package.depcache()}' > /tmp/depcache 88502871c91SPatrick Williams 88667cc0616SPatrick Williams# Ensure the group, user, and home directory are created (or rename them if 88767cc0616SPatrick Williams# they already exist). 88867cc0616SPatrick WilliamsRUN if grep -q ":{gid}:" /etc/group ; then \ 88967cc0616SPatrick Williams groupmod -n {username} $(awk -F : '{{ if ($3 == {gid}) {{ print $1 }} }}' /etc/group) ; \ 89067cc0616SPatrick Williams else \ 89167cc0616SPatrick Williams groupadd -f -g {gid} {username} ; \ 89267cc0616SPatrick Williams fi 89302871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 89467cc0616SPatrick WilliamsRUN if grep -q ":{uid}:" /etc/passwd ; then \ 89573b3ee91SPatrick Williams usermod -l {username} -d {homedir} -m $(awk -F : '{{ if ($3 == {uid}) {{ print $1 }} }}' /etc/passwd) ; \ 89667cc0616SPatrick Williams else \ 89767cc0616SPatrick Williams useradd -d {homedir} -m -u {uid} -g {gid} {username} ; \ 89867cc0616SPatrick Williams fi 89902871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 90002871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 90102871c91SPatrick Williams 902305a9a5dSAndrew Geissler# Ensure user has ability to write to /usr/local for different tool 903305a9a5dSAndrew Geissler# and data installs 9047bb00b13SAndrew GeisslerRUN chown -R {username}:{username} /usr/local/share 905305a9a5dSAndrew Geissler 90602871c91SPatrick Williams{proxy_cmd} 90702871c91SPatrick Williams 90802871c91SPatrick WilliamsRUN /bin/bash 90902871c91SPatrick Williams""" 91002871c91SPatrick Williams 911a18d9c57SPatrick Williams# Do the final docker build 912ee3c9eebSPatrick Williamsdocker_final_img_name = Docker.tagname(None, dockerfile) 913ee3c9eebSPatrick WilliamsDocker.build("final", docker_final_img_name, dockerfile) 914ee3c9eebSPatrick Williams 91500536fbeSPatrick Williams# Print the tag of the final image. 91600536fbeSPatrick Williamsprint(docker_final_img_name) 917