102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 9*50837436SPatrick Williams# default is ubuntu:focal 10*50837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 11*50837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 12*50837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 13*50837436SPatrick Williams# (ex. Jenkins)> 1402871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1502871c91SPatrick Williams# repositories> 1602871c91SPatrick Williams# default is master, which will be used if input branch not 1702871c91SPatrick Williams# provided or not found 1802871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1902871c91SPatrick Williams# default ones in /etc/apt/sources.list> 2002871c91SPatrick Williams# default is empty, and no mirror is used. 2102871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2202871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2302871c91SPatrick Williams 2402871c91SPatrick Williamsimport os 2502871c91SPatrick Williamsimport sys 26b16f3e20SPatrick Williamsimport threading 27a18d9c57SPatrick Williamsfrom datetime import date 28a18d9c57SPatrick Williamsfrom hashlib import sha256 2902871c91SPatrick Williamsfrom sh import docker, git, nproc, uname 3002871c91SPatrick Williams 3102871c91SPatrick Williams# Read a bunch of environment variables. 3202871c91SPatrick Williamsdocker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 33*50837436SPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 34*50837436SPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 3502871c91SPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:focal") 3602871c91SPatrick Williamsbranch = os.environ.get("BRANCH", "master") 3702871c91SPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 3802871c91SPatrick Williamshttp_proxy = os.environ.get("http_proxy") 39aae36d18SPatrick Williamsprefix = "/usr/local" 4002871c91SPatrick Williams 4102871c91SPatrick Williams# Set up some common variables. 4202871c91SPatrick Williamsproc_count = nproc().strip() 4302871c91SPatrick Williamsusername = os.environ.get("USER") 4402871c91SPatrick Williamshomedir = os.environ.get("HOME") 4502871c91SPatrick Williamsgid = os.getgid() 4602871c91SPatrick Williamsuid = os.getuid() 4702871c91SPatrick Williams 4802871c91SPatrick Williams# Determine the architecture for Docker. 4902871c91SPatrick Williamsarch = uname("-m").strip() 5002871c91SPatrick Williamsif arch == "ppc64le": 5102871c91SPatrick Williams docker_base = "ppc64le/" 5202871c91SPatrick Williamselif arch == "x86_64": 5302871c91SPatrick Williams docker_base = "" 5402871c91SPatrick Williamselse: 55b16f3e20SPatrick Williams print( 56b16f3e20SPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 57b16f3e20SPatrick Williams file=sys.stderr, 58b16f3e20SPatrick Williams ) 5902871c91SPatrick Williams sys.exit(1) 6002871c91SPatrick Williams 617204324cSPatrick Williams# Packages to include in image. 627204324cSPatrick Williamspackages = { 632abc4a48SPatrick Williams "boost": { 642abc4a48SPatrick Williams "rev": "1.74.0", 652abc4a48SPatrick Williams "url": ( 662abc4a48SPatrick Williams lambda pkg, rev: f"https://dl.bintray.com/boostorg/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.bz2" 672abc4a48SPatrick Williams ), 68aae36d18SPatrick Williams "build_type": "custom", 69aae36d18SPatrick Williams "build_steps": [ 70aae36d18SPatrick Williams f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 71aae36d18SPatrick Williams "./b2", 72aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 73aae36d18SPatrick Williams ], 742abc4a48SPatrick Williams }, 75aae36d18SPatrick Williams "USCiLab/cereal": { 76aae36d18SPatrick Williams "rev": "v1.3.0", 77aae36d18SPatrick Williams "build_type": "custom", 78aae36d18SPatrick Williams "build_steps": [f"cp -a include/cereal/ {prefix}/include/"], 79aae36d18SPatrick Williams }, 80aae36d18SPatrick Williams "catchorg/Catch2": { 81aae36d18SPatrick Williams "rev": "v2.12.2", 82aae36d18SPatrick Williams "build_type": "cmake", 83aae36d18SPatrick Williams "config_flags": ["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"], 84aae36d18SPatrick Williams }, 85aae36d18SPatrick Williams "CLIUtils/CLI11": { 86aae36d18SPatrick Williams "rev": "v1.9.0", 87aae36d18SPatrick Williams "build_type": "cmake", 88aae36d18SPatrick Williams "config_flags": [ 89aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 90aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 91aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 92aae36d18SPatrick Williams ], 93aae36d18SPatrick Williams }, 94aae36d18SPatrick Williams "fmtlib/fmt": { 95aae36d18SPatrick Williams "rev": "6.2.1", 96aae36d18SPatrick Williams "build_type": "cmake", 97aae36d18SPatrick Williams "config_flags": [ 98aae36d18SPatrick Williams "-DFMT_DOC=OFF", 99aae36d18SPatrick Williams "-DFMT_TEST=OFF", 100aae36d18SPatrick Williams ], 101aae36d18SPatrick Williams }, 10202871c91SPatrick Williams # Snapshot from 2020-01-03 103aae36d18SPatrick Williams "Naios/function2": { 104aae36d18SPatrick Williams "rev": "3a0746bf5f601dfed05330aefcb6854354fce07d", 105aae36d18SPatrick Williams "build_type": "custom", 106aae36d18SPatrick Williams "build_steps": [ 107aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 108aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 109aae36d18SPatrick Williams ], 110aae36d18SPatrick Williams }, 11102871c91SPatrick Williams # Snapshot from 2020-02-13 112aae36d18SPatrick Williams "google/googletest": { 113aae36d18SPatrick Williams "rev": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5", 114aae36d18SPatrick Williams "build_type": "cmake", 115aae36d18SPatrick Williams "config_env": ["CXXFLAGS=-std=c++17"], 116aae36d18SPatrick Williams "config_flags": ["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 117aae36d18SPatrick Williams }, 11802871c91SPatrick Williams # Release 2020-08-06 119aae36d18SPatrick Williams "nlohmann/json": { 120aae36d18SPatrick Williams "rev": "v3.9.1", 121aae36d18SPatrick Williams "build_type": "custom", 122aae36d18SPatrick Williams "build_steps": [ 123aae36d18SPatrick Williams f"mkdir {prefix}/include/nlohmann", 124aae36d18SPatrick Williams f"cp include/nlohmann/json.hpp {prefix}/include/nlohmann", 125aae36d18SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 126aae36d18SPatrick Williams ], 127aae36d18SPatrick Williams }, 12802871c91SPatrick Williams # Snapshot from 2019-05-24 129aae36d18SPatrick Williams "linux-test-project/lcov": { 130aae36d18SPatrick Williams "rev": "75fbae1cfc5027f818a0bb865bf6f96fab3202da", 131aae36d18SPatrick Williams "build_type": "make", 132aae36d18SPatrick Williams }, 13302871c91SPatrick Williams # dev-5.0 2019-05-03 134aae36d18SPatrick Williams "openbmc/linux": { 135aae36d18SPatrick Williams "rev": "8bf6567e77f7aa68975b7c9c6d044bba690bf327", 136aae36d18SPatrick Williams "build_type": "custom", 137aae36d18SPatrick Williams "build_steps": [ 138aae36d18SPatrick Williams f"make -j{proc_count} defconfig", 139aae36d18SPatrick Williams f"make INSTALL_HDR_PATH={prefix} headers_install", 140aae36d18SPatrick Williams ], 141aae36d18SPatrick Williams }, 14202871c91SPatrick Williams # Snapshot from 2019-09-03 143aae36d18SPatrick Williams "LibVNC/libvncserver": { 144aae36d18SPatrick Williams "rev": "1354f7f1bb6962dab209eddb9d6aac1f03408110", 145aae36d18SPatrick Williams "build_type": "cmake", 146aae36d18SPatrick Williams }, 147aae36d18SPatrick Williams "martinmoene/span-lite": { 148aae36d18SPatrick Williams "rev": "v0.7.0", 149aae36d18SPatrick Williams "build_type": "cmake", 150aae36d18SPatrick Williams "config_flags": [ 151aae36d18SPatrick Williams "-DSPAN_LITE_OPT_BUILD_TESTS=OFF", 152aae36d18SPatrick Williams ], 153aae36d18SPatrick Williams }, 15402871c91SPatrick Williams # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb 155aae36d18SPatrick Williams "leethomason/tinyxml2": { 156aae36d18SPatrick Williams "rev": "37bc3aca429f0164adf68c23444540b4a24b5778", 157aae36d18SPatrick Williams "build_type": "cmake", 158aae36d18SPatrick Williams }, 15902871c91SPatrick Williams # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 160aae36d18SPatrick Williams "CPPAlliance/url": { 161aae36d18SPatrick Williams "rev": "a56ae0df6d3078319755fbaa67822b4fa7fd352b", 162aae36d18SPatrick Williams "build_type": "cmake", 163aae36d18SPatrick Williams "config_flags": [ 164aae36d18SPatrick Williams "-DBOOST_URL_BUILD_EXAMPLES=OFF", 165aae36d18SPatrick Williams "-DBOOST_URL_BUILD_TESTS=OFF", 166aae36d18SPatrick Williams "-DBOOST_URL_STANDALONE=ON", 167aae36d18SPatrick Williams ], 168aae36d18SPatrick Williams }, 16902871c91SPatrick Williams # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb 170aae36d18SPatrick Williams "tristanpenman/valijson": { 171aae36d18SPatrick Williams "rev": "c2f22fddf599d04dc33fcd7ed257c698a05345d9", 172aae36d18SPatrick Williams "build_type": "cmake", 173aae36d18SPatrick Williams "config_flags": [ 174aae36d18SPatrick Williams "-DBUILD_TESTS=0", 175aae36d18SPatrick Williams "-DINSTALL_HEADERS=1", 176aae36d18SPatrick Williams ], 177aae36d18SPatrick Williams }, 17802871c91SPatrick Williams # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb 179aae36d18SPatrick Williams "nlohmann/fifo_map": { 180aae36d18SPatrick Williams "rev": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9", 181aae36d18SPatrick Williams "build_type": "custom", 182aae36d18SPatrick Williams "build_steps": [f"cp src/fifo_map.hpp {prefix}/include/"], 183aae36d18SPatrick Williams }, 184aae36d18SPatrick Williams "open-power/pdbg": {"build_type": "autoconf"}, 185aae36d18SPatrick Williams "openbmc/gpioplus": { 186aae36d18SPatrick Williams "depends": ["openbmc/stdplus"], 187aae36d18SPatrick Williams "build_type": "meson", 188aae36d18SPatrick Williams "config_flags": [ 189aae36d18SPatrick Williams "-Dexamples=false", 190aae36d18SPatrick Williams "-Dtests=disabled", 191aae36d18SPatrick Williams ], 192aae36d18SPatrick Williams }, 193aae36d18SPatrick Williams "openbmc/phosphor-dbus-interfaces": { 194aae36d18SPatrick Williams "depends": ["openbmc/sdbusplus"], 195aae36d18SPatrick Williams "build_type": "meson", 196aae36d18SPatrick Williams "config_flags": [ 197aae36d18SPatrick Williams "-Ddata_com_ibm=true", 198aae36d18SPatrick Williams "-Ddata_org_open_power=true", 199aae36d18SPatrick Williams ], 200aae36d18SPatrick Williams }, 20183394610SPatrick Williams "openbmc/phosphor-logging": { 20283394610SPatrick Williams "depends": [ 20383394610SPatrick Williams "USCiLab/cereal", 20483394610SPatrick Williams "nlohmann/fifo_map", 20583394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 20683394610SPatrick Williams "openbmc/sdbusplus", 20783394610SPatrick Williams "openbmc/sdeventplus", 208aae36d18SPatrick Williams ], 209aae36d18SPatrick Williams "build_type": "autoconf", 210aae36d18SPatrick Williams "config_flags": [ 211aae36d18SPatrick Williams "--enable-metadata-processing", 212aae36d18SPatrick Williams f"YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml", 213aae36d18SPatrick Williams ], 21483394610SPatrick Williams }, 21583394610SPatrick Williams "openbmc/phosphor-objmgr": { 21683394610SPatrick Williams "depends": [ 21783394610SPatrick Williams "boost", 21883394610SPatrick Williams "leethomason/tinyxml2", 21983394610SPatrick Williams "openbmc/phosphor-logging", 22083394610SPatrick Williams "openbmc/sdbusplus", 221aae36d18SPatrick Williams ], 222aae36d18SPatrick Williams "build_type": "autoconf", 22383394610SPatrick Williams }, 22483394610SPatrick Williams "openbmc/pldm": { 22583394610SPatrick Williams "depends": [ 22683394610SPatrick Williams "CLIUtils/CLI11", 22783394610SPatrick Williams "boost", 22883394610SPatrick Williams "nlohmann/json", 22983394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 23083394610SPatrick Williams "openbmc/phosphor-logging", 23183394610SPatrick Williams "openbmc/sdbusplus", 23283394610SPatrick Williams "openbmc/sdeventplus", 233aae36d18SPatrick Williams ], 234aae36d18SPatrick Williams "build_type": "meson", 235aae36d18SPatrick Williams "config_flags": [ 236aae36d18SPatrick Williams "-Dlibpldm-only=enabled", 237aae36d18SPatrick Williams "-Doem-ibm=enabled", 238aae36d18SPatrick Williams "-Dtests=disabled", 239aae36d18SPatrick Williams ], 24083394610SPatrick Williams }, 241aae36d18SPatrick Williams "openbmc/sdbusplus": { 242aae36d18SPatrick Williams "build_type": "meson", 243aae36d18SPatrick Williams "custom_post_dl": [ 244aae36d18SPatrick Williams "cd tools", 245aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 246aae36d18SPatrick Williams "cd ..", 247aae36d18SPatrick Williams ], 248aae36d18SPatrick Williams "config_flags": [ 249aae36d18SPatrick Williams "-Dexamples=disabled", 250aae36d18SPatrick Williams "-Dtests=disabled", 251aae36d18SPatrick Williams ], 252aae36d18SPatrick Williams }, 253aae36d18SPatrick Williams "openbmc/sdeventplus": { 254aae36d18SPatrick Williams "depends": ["Naios/function2", "openbmc/stdplus"], 255aae36d18SPatrick Williams "build_type": "meson", 256aae36d18SPatrick Williams "config_flags": [ 257aae36d18SPatrick Williams "-Dexamples=false", 258aae36d18SPatrick Williams "-Dtests=disabled", 259aae36d18SPatrick Williams ], 260aae36d18SPatrick Williams }, 261aae36d18SPatrick Williams "openbmc/stdplus": { 262aae36d18SPatrick Williams "depends": ["fmtlib/fmt", "martinmoene/span-lite"], 263aae36d18SPatrick Williams "build_type": "meson", 264aae36d18SPatrick Williams "config_flags": [ 265aae36d18SPatrick Williams "-Dexamples=false", 266aae36d18SPatrick Williams "-Dtests=disabled", 267aae36d18SPatrick Williams ], 268aae36d18SPatrick Williams }, 26902871c91SPatrick Williams} 27002871c91SPatrick Williams 2712abc4a48SPatrick Williams 2727204324cSPatrick Williamsdef pkg_rev(pkg): 2737204324cSPatrick Williams return packages[pkg]["rev"] 2747204324cSPatrick Williams 2752abc4a48SPatrick Williams 276e1398742SPatrick Williamsdef pkg_stagename(pkg): 277a18d9c57SPatrick Williams return pkg.replace("/", "-").lower() 2787204324cSPatrick Williams 2792abc4a48SPatrick Williams 2802abc4a48SPatrick Williamsdef pkg_url(pkg): 2812abc4a48SPatrick Williams if "url" in packages[pkg]: 2822abc4a48SPatrick Williams return packages[pkg]["url"](pkg, pkg_rev(pkg)) 2832abc4a48SPatrick Williams return f"https://github.com/{pkg}/archive/{pkg_rev(pkg)}.tar.gz" 2842abc4a48SPatrick Williams 2852abc4a48SPatrick Williams 286aae36d18SPatrick Williamsdef pkg_download(pkg): 287aae36d18SPatrick Williams url = pkg_url(pkg) 288aae36d18SPatrick Williams if ".tar." not in url: 289aae36d18SPatrick Williams raise NotImplementedError(f"Unhandled download type for {pkg}: {url}") 290aae36d18SPatrick Williams cmd = f"curl -L {url} | tar -x" 291aae36d18SPatrick Williams if url.endswith(".bz2"): 292aae36d18SPatrick Williams cmd += "j" 293aae36d18SPatrick Williams if url.endswith(".gz"): 294aae36d18SPatrick Williams cmd += "z" 295aae36d18SPatrick Williams return cmd 296aae36d18SPatrick Williams 297aae36d18SPatrick Williams 29883394610SPatrick Williamsdef pkg_copycmds(pkg=None): 29983394610SPatrick Williams pkgs = [] 30083394610SPatrick Williams if pkg: 30183394610SPatrick Williams if "depends" not in packages[pkg]: 30283394610SPatrick Williams return "" 30383394610SPatrick Williams pkgs = sorted(packages[pkg]["depends"]) 30483394610SPatrick Williams else: 30583394610SPatrick Williams pkgs = sorted(packages.keys()) 30683394610SPatrick Williams 30783394610SPatrick Williams copy_cmds = "" 30883394610SPatrick Williams for p in pkgs: 309a18d9c57SPatrick Williams copy_cmds += f"COPY --from={packages[p]['__tag']} {prefix} {prefix}\n" 31083394610SPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 31183394610SPatrick Williams # https://github.com/moby/moby/issues/37965 31283394610SPatrick Williams copy_cmds += "RUN true\n" 31383394610SPatrick Williams return copy_cmds 31483394610SPatrick Williams 31583394610SPatrick Williams 316aae36d18SPatrick Williamsdef pkg_cd_srcdir(pkg): 317aae36d18SPatrick Williams return f"cd {pkg.split('/')[-1]}* && " 318aae36d18SPatrick Williams 319aae36d18SPatrick Williams 320aae36d18SPatrick Williamsdef pkg_build(pkg): 321aae36d18SPatrick Williams result = f"RUN {pkg_download(pkg)} && " 322aae36d18SPatrick Williams result += pkg_cd_srcdir(pkg) 323aae36d18SPatrick Williams 324aae36d18SPatrick Williams if "custom_post_dl" in packages[pkg]: 325aae36d18SPatrick Williams result += " && ".join(packages[pkg]["custom_post_dl"]) + " && " 326aae36d18SPatrick Williams 327aae36d18SPatrick Williams build_type = packages[pkg]["build_type"] 328aae36d18SPatrick Williams if build_type == "autoconf": 329aae36d18SPatrick Williams result += pkg_build_autoconf(pkg) 330aae36d18SPatrick Williams elif build_type == "cmake": 331aae36d18SPatrick Williams result += pkg_build_cmake(pkg) 332aae36d18SPatrick Williams elif build_type == "custom": 333aae36d18SPatrick Williams result += pkg_build_custom(pkg) 334aae36d18SPatrick Williams elif build_type == "make": 335aae36d18SPatrick Williams result += pkg_build_make(pkg) 336aae36d18SPatrick Williams elif build_type == "meson": 337aae36d18SPatrick Williams result += pkg_build_meson(pkg) 338aae36d18SPatrick Williams else: 339aae36d18SPatrick Williams raise NotImplementedError( 340aae36d18SPatrick Williams f"Unhandled build type for {pkg}: {packages[pkg]['build_type']}" 341aae36d18SPatrick Williams ) 342aae36d18SPatrick Williams 343aae36d18SPatrick Williams return result 344aae36d18SPatrick Williams 345aae36d18SPatrick Williams 346aae36d18SPatrick Williamsdef pkg_build_autoconf(pkg): 347aae36d18SPatrick Williams options = " ".join(packages[pkg].get("config_flags", [])) 348aae36d18SPatrick Williams env = " ".join(packages[pkg].get("config_env", [])) 349aae36d18SPatrick Williams result = "./bootstrap.sh && " 350aae36d18SPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 351aae36d18SPatrick Williams result += f"make -j{proc_count} && " 352aae36d18SPatrick Williams result += "make install " 353aae36d18SPatrick Williams return result 354aae36d18SPatrick Williams 355aae36d18SPatrick Williams 356aae36d18SPatrick Williamsdef pkg_build_cmake(pkg): 357aae36d18SPatrick Williams options = " ".join(packages[pkg].get("config_flags", [])) 358aae36d18SPatrick Williams env = " ".join(packages[pkg].get("config_env", [])) 359aae36d18SPatrick Williams result = "mkdir builddir && cd builddir && " 360aae36d18SPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 3610f2086b3SPatrick Williams result += "cmake --build . --target all && " 3620f2086b3SPatrick Williams result += "cmake --build . --target install && " 363aae36d18SPatrick Williams result += "cd .. " 364aae36d18SPatrick Williams return result 365aae36d18SPatrick Williams 366aae36d18SPatrick Williams 367aae36d18SPatrick Williamsdef pkg_build_custom(pkg): 368aae36d18SPatrick Williams return " && ".join(packages[pkg].get("build_steps", [])) 369aae36d18SPatrick Williams 370aae36d18SPatrick Williams 371aae36d18SPatrick Williamsdef pkg_build_make(pkg): 372aae36d18SPatrick Williams result = f"make -j{proc_count} && " 373aae36d18SPatrick Williams result += "make install " 374aae36d18SPatrick Williams return result 375aae36d18SPatrick Williams 376aae36d18SPatrick Williams 377aae36d18SPatrick Williamsdef pkg_build_meson(pkg): 378aae36d18SPatrick Williams options = " ".join(packages[pkg].get("config_flags", [])) 379aae36d18SPatrick Williams env = " ".join(packages[pkg].get("config_env", [])) 380aae36d18SPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 381aae36d18SPatrick Williams result += "ninja -C builddir && ninja -C builddir install " 382aae36d18SPatrick Williams return result 383aae36d18SPatrick Williams 384aae36d18SPatrick Williams 385b16f3e20SPatrick Williamspkg_lock = threading.Lock() 3868751b80eSPatrick Williams 387b16f3e20SPatrick Williams 388b16f3e20SPatrick Williamsdef pkg_generate(pkg): 389b16f3e20SPatrick Williams class pkg_thread(threading.Thread): 390b16f3e20SPatrick Williams def run(self): 391b16f3e20SPatrick Williams pkg_lock.acquire() 392b16f3e20SPatrick Williams deps = [ 393b16f3e20SPatrick Williams packages[deppkg]["__thread"] 394b16f3e20SPatrick Williams for deppkg in sorted(packages[pkg].get("depends", [])) 395b16f3e20SPatrick Williams ] 396b16f3e20SPatrick Williams pkg_lock.release() 397b16f3e20SPatrick Williams for deppkg in deps: 398b16f3e20SPatrick Williams deppkg.join() 3998751b80eSPatrick Williams 400a18d9c57SPatrick Williams dockerfile = f""" 401a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 402a18d9c57SPatrick Williams{pkg_copycmds(pkg)} 403a18d9c57SPatrick Williams{pkg_build(pkg)} 404a18d9c57SPatrick Williams""" 405a18d9c57SPatrick Williams 406b16f3e20SPatrick Williams pkg_lock.acquire() 407a18d9c57SPatrick Williams tag = docker_img_tagname(pkg_stagename(pkg), dockerfile) 408a18d9c57SPatrick Williams packages[pkg]["__tag"] = tag 409b16f3e20SPatrick Williams pkg_lock.release() 4108751b80eSPatrick Williams 411b16f3e20SPatrick Williams try: 412b16f3e20SPatrick Williams self.exception = None 413b16f3e20SPatrick Williams docker_img_build(pkg, tag, dockerfile) 414b16f3e20SPatrick Williams except Exception as e: 415b16f3e20SPatrick Williams self.package = pkg 416b16f3e20SPatrick Williams self.exception = e 417b16f3e20SPatrick Williams 418b16f3e20SPatrick Williams packages[pkg]["__thread"] = pkg_thread() 4198751b80eSPatrick Williams 4208751b80eSPatrick Williams 4218751b80eSPatrick Williamsdef pkg_generate_packages(): 422b16f3e20SPatrick Williams for pkg in packages.keys(): 423a18d9c57SPatrick Williams pkg_generate(pkg) 424a18d9c57SPatrick Williams 425b16f3e20SPatrick Williams pkg_lock.acquire() 426b16f3e20SPatrick Williams pkg_threads = [packages[p]["__thread"] for p in packages.keys()] 427b16f3e20SPatrick Williams for t in pkg_threads: 428b16f3e20SPatrick Williams t.start() 429b16f3e20SPatrick Williams pkg_lock.release() 430b16f3e20SPatrick Williams 431b16f3e20SPatrick Williams for t in pkg_threads: 432b16f3e20SPatrick Williams t.join() 433b16f3e20SPatrick Williams if t.exception: 434b16f3e20SPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 435b16f3e20SPatrick Williams raise t.exception 436b16f3e20SPatrick Williams 437*50837436SPatrick Williamsdef timestamp(): 438*50837436SPatrick Williams today = date.today().isocalendar() 439*50837436SPatrick Williams return f"{today[0]}-W{today[1]:02}" 440a18d9c57SPatrick Williams 441a18d9c57SPatrick Williamsdef docker_img_tagname(pkgname, dockerfile): 442a18d9c57SPatrick Williams result = docker_image_name 443a18d9c57SPatrick Williams if pkgname: 444a18d9c57SPatrick Williams result += "-" + pkgname 445*50837436SPatrick Williams result += ":" + timestamp() 446a18d9c57SPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 4478751b80eSPatrick Williams return result 4488751b80eSPatrick Williams 4498751b80eSPatrick Williams 450b16f3e20SPatrick Williamsdef docker_img_build(pkg, tag, dockerfile): 451*50837436SPatrick Williams if not force_build and pkg != "final": 452*50837436SPatrick Williams # TODO: the 'final' is here because we do not tag the final image yet 453*50837436SPatrick Williams # so we always need to rebuild it. This will be changed in a future 454*50837436SPatrick Williams # commit so that we tag even the final image. 455*50837436SPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 456*50837436SPatrick Williams print(f"Image {tag} already exists. Skipping.", file=sys.stderr) 457*50837436SPatrick Williams return 458*50837436SPatrick Williams 459b16f3e20SPatrick Williams docker.build( 460a18d9c57SPatrick Williams proxy_args, 461a18d9c57SPatrick Williams "--network=host", 462a18d9c57SPatrick Williams "--force-rm", 463*50837436SPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 464a18d9c57SPatrick Williams "-t", 465a18d9c57SPatrick Williams tag, 466a18d9c57SPatrick Williams "-", 467a18d9c57SPatrick Williams _in=dockerfile, 468b16f3e20SPatrick Williams _out=( 469b16f3e20SPatrick Williams lambda line: print(pkg + ":", line, end="", file=sys.stderr, flush=True) 470b16f3e20SPatrick Williams ), 471b16f3e20SPatrick Williams ) 472a18d9c57SPatrick Williams 473a18d9c57SPatrick Williams 4747204324cSPatrick Williams# Look up the HEAD for missing a static rev. 47502871c91SPatrick Williamspkg_lookups = {} 4767204324cSPatrick Williamsfor pkg in packages.keys(): 4777204324cSPatrick Williams if "rev" in packages[pkg]: 4787204324cSPatrick Williams continue 47902871c91SPatrick Williams pkg_lookups[pkg] = git( 48002871c91SPatrick Williams "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True 48102871c91SPatrick Williams ) 48202871c91SPatrick Williamsfor pkg, result in pkg_lookups.items(): 48302871c91SPatrick Williams for line in result.stdout.decode().split("\n"): 48402871c91SPatrick Williams if f"refs/heads/{branch}" in line: 4857204324cSPatrick Williams packages[pkg]["rev"] = line.strip().split()[0] 4867204324cSPatrick Williams elif "refs/heads/master" in line and p not in packages: 4877204324cSPatrick Williams packages[pkg]["rev"] = line.strip().split()[0] 48802871c91SPatrick Williams 48902871c91SPatrick Williams# Create the contents of the '/tmp/depcache'. 49002871c91SPatrick Williams# This needs to be sorted for consistency. 49102871c91SPatrick Williamsdepcache = "" 4927204324cSPatrick Williamsfor pkg in sorted(packages.keys()): 4937204324cSPatrick Williams depcache += "%s:%s," % (pkg, pkg_rev(pkg)) 49402871c91SPatrick Williams 49502871c91SPatrick Williams# Define common flags used for builds 49602871c91SPatrick Williamsconfigure_flags = " ".join( 49702871c91SPatrick Williams [ 49802871c91SPatrick Williams f"--prefix={prefix}", 49902871c91SPatrick Williams ] 50002871c91SPatrick Williams) 50102871c91SPatrick Williamscmake_flags = " ".join( 50202871c91SPatrick Williams [ 50302871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 5040f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 50502871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 5060f2086b3SPatrick Williams "-GNinja", 5070f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 50802871c91SPatrick Williams ] 50902871c91SPatrick Williams) 51002871c91SPatrick Williamsmeson_flags = " ".join( 51102871c91SPatrick Williams [ 51202871c91SPatrick Williams "--wrap-mode=nodownload", 51302871c91SPatrick Williams f"-Dprefix={prefix}", 51402871c91SPatrick Williams ] 51502871c91SPatrick Williams) 51602871c91SPatrick Williams 51702871c91SPatrick Williams# Special flags if setting up a deb mirror. 51802871c91SPatrick Williamsmirror = "" 51902871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 52002871c91SPatrick Williams mirror = f""" 52102871c91SPatrick WilliamsRUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 52202871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 52302871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 52402871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 52502871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 52602871c91SPatrick Williams""" 52702871c91SPatrick Williams 52802871c91SPatrick Williams# Special flags for proxying. 52902871c91SPatrick Williamsproxy_cmd = "" 53002871c91SPatrick Williamsproxy_args = [] 53102871c91SPatrick Williamsif http_proxy: 53202871c91SPatrick Williams proxy_cmd = f""" 53302871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 53402871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 53502871c91SPatrick Williams""" 53602871c91SPatrick Williams proxy_args.extend( 53702871c91SPatrick Williams [ 53802871c91SPatrick Williams "--build-arg", 53902871c91SPatrick Williams f"http_proxy={http_proxy}", 54002871c91SPatrick Williams "--build-arg", 54102871c91SPatrick Williams "https_proxy={https_proxy}", 54202871c91SPatrick Williams ] 54302871c91SPatrick Williams ) 54402871c91SPatrick Williams 54502871c91SPatrick Williams# Create docker image that can run package unit tests 546a18d9c57SPatrick Williamsdockerfile_base = f""" 547a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 54802871c91SPatrick Williams 54902871c91SPatrick Williams{mirror} 55002871c91SPatrick Williams 55102871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 55202871c91SPatrick Williams 55302871c91SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/" 55402871c91SPatrick Williams 55502871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 55602871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 55702871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 558*50837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 559*50837436SPatrick Williams ( apt-get install ubuntu-dbgsym-keyring || \ 560*50837436SPatrick Williams ( apt-get install -yy dirmngr && \ 561*50837436SPatrick Williams apt-key adv --keyserver keyserver.ubuntu.com \ 562*50837436SPatrick Williams --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 56302871c91SPatrick Williams 56402871c91SPatrick Williams# Parse the current repo list into a debug repo list 56502871c91SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 56602871c91SPatrick Williams 56702871c91SPatrick Williams# Remove non-existent debug repos 56802871c91SPatrick WilliamsRUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 56902871c91SPatrick Williams 57002871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 57102871c91SPatrick Williams 57202871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 57302871c91SPatrick Williams gcc-10 \ 57402871c91SPatrick Williams g++-10 \ 57502871c91SPatrick Williams libc6-dbg \ 57602871c91SPatrick Williams libc6-dev \ 57702871c91SPatrick Williams libtool \ 57802871c91SPatrick Williams bison \ 57902871c91SPatrick Williams libdbus-1-dev \ 58002871c91SPatrick Williams flex \ 58102871c91SPatrick Williams cmake \ 58202871c91SPatrick Williams python3 \ 58302871c91SPatrick Williams python3-dev\ 58402871c91SPatrick Williams python3-yaml \ 58502871c91SPatrick Williams python3-mako \ 58602871c91SPatrick Williams python3-pip \ 58702871c91SPatrick Williams python3-setuptools \ 58802871c91SPatrick Williams python3-git \ 58902871c91SPatrick Williams python3-socks \ 59002871c91SPatrick Williams pkg-config \ 59102871c91SPatrick Williams autoconf \ 59202871c91SPatrick Williams autoconf-archive \ 59302871c91SPatrick Williams libsystemd-dev \ 59402871c91SPatrick Williams systemd \ 59502871c91SPatrick Williams libssl-dev \ 59602871c91SPatrick Williams libevdev-dev \ 59702871c91SPatrick Williams libevdev2-dbgsym \ 59802871c91SPatrick Williams libjpeg-dev \ 59902871c91SPatrick Williams libpng-dev \ 60002871c91SPatrick Williams ninja-build \ 60102871c91SPatrick Williams sudo \ 60202871c91SPatrick Williams curl \ 60302871c91SPatrick Williams git \ 60402871c91SPatrick Williams dbus \ 60502871c91SPatrick Williams iputils-ping \ 60602871c91SPatrick Williams clang-10 \ 60702871c91SPatrick Williams clang-format-10 \ 60802871c91SPatrick Williams clang-tidy-10 \ 60902871c91SPatrick Williams clang-tools-10 \ 61002871c91SPatrick Williams shellcheck \ 61102871c91SPatrick Williams npm \ 61202871c91SPatrick Williams iproute2 \ 61302871c91SPatrick Williams libnl-3-dev \ 61402871c91SPatrick Williams libnl-genl-3-dev \ 61502871c91SPatrick Williams libconfig++-dev \ 61602871c91SPatrick Williams libsnmp-dev \ 61702871c91SPatrick Williams valgrind \ 61802871c91SPatrick Williams valgrind-dbg \ 61902871c91SPatrick Williams libpam0g-dev \ 62002871c91SPatrick Williams xxd \ 62102871c91SPatrick Williams libi2c-dev \ 62202871c91SPatrick Williams wget \ 62302871c91SPatrick Williams libldap2-dev \ 62402871c91SPatrick Williams libprotobuf-dev \ 62502871c91SPatrick Williams libperlio-gzip-perl \ 62602871c91SPatrick Williams libjson-perl \ 62702871c91SPatrick Williams protobuf-compiler \ 62802871c91SPatrick Williams libgpiod-dev \ 62902871c91SPatrick Williams device-tree-compiler \ 63002871c91SPatrick Williams cppcheck \ 63102871c91SPatrick Williams libpciaccess-dev \ 63202871c91SPatrick Williams libmimetic-dev \ 63302871c91SPatrick Williams libxml2-utils \ 63402871c91SPatrick Williams libxml-simple-perl 63502871c91SPatrick Williams 63602871c91SPatrick WilliamsRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \ 63702871c91SPatrick Williams --slave /usr/bin/g++ g++ /usr/bin/g++-10 \ 63802871c91SPatrick Williams --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \ 63902871c91SPatrick Williams --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \ 64002871c91SPatrick Williams --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10 64102871c91SPatrick Williams 64202871c91SPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \ 64302871c91SPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \ 64402871c91SPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \ 64502871c91SPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \ 64602871c91SPatrick Williams --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py 64702871c91SPatrick Williams 648*50837436SPatrick Williams""" 649*50837436SPatrick Williams 650*50837436SPatrick Williamsif is_automated_ci_build: 651*50837436SPatrick Williams dockerfile_base += f""" 652*50837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 653*50837436SPatrick Williams# to re-run `apt-get update` daily. 654*50837436SPatrick WilliamsRUN echo {timestamp()} 655*50837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 656*50837436SPatrick Williams 657*50837436SPatrick Williams""" 658*50837436SPatrick Williams 659*50837436SPatrick Williamsdockerfile_base += f""" 66002871c91SPatrick WilliamsRUN pip3 install inflection 66102871c91SPatrick WilliamsRUN pip3 install pycodestyle 66202871c91SPatrick WilliamsRUN pip3 install jsonschema 66302871c91SPatrick WilliamsRUN pip3 install meson==0.54.3 66402871c91SPatrick WilliamsRUN pip3 install protobuf 665a18d9c57SPatrick Williams""" 66602871c91SPatrick Williams 667a18d9c57SPatrick Williams# Build the stage docker images. 668a18d9c57SPatrick Williamsdocker_base_img_name = docker_img_tagname("base", dockerfile_base) 669b16f3e20SPatrick Williamsdocker_img_build("base", docker_base_img_name, dockerfile_base) 670a18d9c57SPatrick Williamspkg_generate_packages() 67102871c91SPatrick Williams 672a18d9c57SPatrick Williamsdockerfile = f""" 67302871c91SPatrick Williams# Build the final output image 674a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 675a18d9c57SPatrick Williams{pkg_copycmds()} 67602871c91SPatrick Williams 67702871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 67802871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 67902871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 68002871c91SPatrick WilliamsRUN echo '{depcache}' > /tmp/depcache 68102871c91SPatrick Williams 68202871c91SPatrick Williams# Final configuration for the workspace 68302871c91SPatrick WilliamsRUN grep -q {gid} /etc/group || groupadd -g {gid} {username} 68402871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 68502871c91SPatrick WilliamsRUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 68602871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 68702871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 68802871c91SPatrick Williams 68902871c91SPatrick Williams{proxy_cmd} 69002871c91SPatrick Williams 69102871c91SPatrick WilliamsRUN /bin/bash 69202871c91SPatrick Williams""" 69302871c91SPatrick Williams 694a18d9c57SPatrick Williams# Do the final docker build 695b16f3e20SPatrick Williamsdocker_img_build("final", docker_image_name, dockerfile) 696